From 89853ab7bee0bace4a813b98be6ec05379d43f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 18 Aug 2026 11:14:26 +0300 Subject: [PATCH 01/17] MDEV-40756 Incorrect multi-batch recovery of file size file_name_t::page0_lsn: Keep track of the last applied recv_sys_t::parse_page0() so that a multi-batch recovery will not reset the file to a smaller size. Reviewed by: Thirunarayanan Balathandayuthapani (cherry picked from commit 8f00e6caca633c783140db86d3a48a96de67cf38) --- storage/innobase/log/log0recv.cc | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 736c226cac4fc..690128f5685c9 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -642,6 +642,9 @@ struct file_name_t { /** Log sequence number of a FILE_CREATE record, or 0 */ lsn_t create_lsn = 0; + /** Last FSP_SIZE or FSP_SPACE_FLAGS change, or 0 */ + lsn_t page0_lsn = 0; + /** FSP_SIZE of tablespace */ uint32_t size = 0; @@ -3098,12 +3101,19 @@ void recv_sys_t::parse_page0(const page_id_t id, const byte *b, f{flags ? mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + b) : file_name_t::initial_flags}; recv_spaces_t::iterator it= recv_spaces.find(space_id); - if (it != recv_spaces.end() && !it->second.space) + if (it != recv_spaces.end()) { - if (size) - it->second.size= s; - if (flags) - it->second.flags= f; + if (it->second.page0_lsn > lsn) + return; + it->second.page0_lsn= lsn; + + if (!it->second.space) + { + if (size) + it->second.size= s; + if (flags) + it->second.flags= f; + } } fil_space_set_recv_size_and_flags(space_id, s, f); } From b625be27ca0a0b24730dd19d37d8bbf85917156c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 24 Aug 2026 16:59:14 +0300 Subject: [PATCH 02/17] MDEV-14992 BACKUP SERVER The following SQL statements will be introduced: BACKUP SERVER TO '/path/to/directory' [ 1 CONCURRENT ]; BACKUP SERVER WITH [ 1 CONCURRENT ] 'command'; In place of the 1, any positive number of threads may be specified. For the first variant, '/path/to' must exist and '/path/to/directory' must be compatible with secure_file_priv and not exist; that is where the backup will be written to. For the second variant, 'command' must be the name of a script or command that will be executed in a child process. The standard input of that command will be in a format that is compatible with GNU tar --format=oldgnu (and also BSD tar variants that are also part of Microsoft Windows and Apple macOS). The command is expected to optionally compress and encrypt the stream and redirect it to a file on a local or a remote server. The BACKUP SERVER WITH will append an additional argument, a positive base-ten number in ASCII, starting with 1, to identify the current thread. In this way, each concurrent stream can write a separate file. The backup or the first stream will contain a file backup.cnf, which includes parameters needed for restoring the backup. Currently, these are innodb_log_recovery_start and innodb_log_recovery_target. If innodb_log_recovery_target>0, InnoDB will be in read-only mode, not allowing any writes to persistent files other than via the log application. To restore a streaming backup made with BACKUP SERVER WITH, an empty directory needs to be created and all streams be extracted there using the standard tar utility of the operating system, optionally after undoing any encryption or compression that had been added by the backup command. Then, the backup is prepared or MariaDB server started up on the extracted directory, similar to as if the BACKUP SERVER TO statement had been used. Note: The parameter innodb_log_recovery_start in backup.cnf is STRICTLY NECESSARY TO AVOID CORRUPTION! By default, InnoDB crash recovery starts from the latest available log checkpoint. However, for restoring a backup, recovery must start from the checkpoint that was the latest when the backup was started. Starting recovery from a possible later checkpoint will result in a corrupted database! The following will be implemented separately: MDEV-39061 mariadb-backup compatible wrapper script for BACKUP SERVER MDEV-40163 Partial backup and restore MDEV-39091 Back up ENGINE=RocksDB MDEV-39092 Less blocking backup of ENGINE=Aria The implementation introduces a basic driver Sql_cmd_backup, storage engine interfaces, and basic copying of the storage engines InnoDB, Aria, MyISAM, MERGE (MyISAM), Archive, CSV. aria_backup_end(): A crude prototype that copies non-InnoDB files. Scans and copies the data directories in a single thread, while everything is locked. This will be refactored in MDEV-39092. backup_target: A structured data type to represent a target directory. On Microsoft Windows, we must use directory paths because there is no variant of CopyFileEx() that would work on file handles. backup_sink: Wraps a per-thread output stream as well as storage engine specific context. handlerton::backup_start(), handlerton::backup_end(): Invoked at the start or end of a backup phase, in the thread that executes a BACKUP SERVER statement. handlerton::backup_step(): A backup step that can be invoked from multiple threads concurrently, between the execution of the corresponding handlerton::backup_start() and handlerton::backup_end() of the same phase. copy_entire_file(): A file copying service for POSIX systems. copy_mmap(): A zero-copy alternative to backup::copy(), to copy from a memory-mapped buffer. copy_file_range_try(): A wrapper for Linux copy_file_range(2), which may fail with EOPNOTSUPP or EXDEV and thus require a fallback to copy_mmap() or backup::copy(). backup::copy(): A partial or sparse file-copying service. On other platforms than FreeBSD or Microsoft Windows, there are shortcut alternatives to this. Note: On Linux we never invoke sendfile(2) for copying between files, because can be much slower than the alternatives. backup_stream_append_plain(): Equivalent to backup::copy(), but appending to a stream. On Linux, this uses sendfile(2), which assumes that the source data will not be changed before the data has been consumed from the pipe. backup_stream_append_async(): A variant of backup_stream_append_plain() where the source file region is guaranteed to be immutable after the call returns. We must not use zero-copy mmap(2) or Linux sendfile(2) for copying data files that may be modified in place, because it could introduce a race condition between a page write that runs concurrently with a child process that is reading the data from the pipe. backup_stream_zeropad(): Zero-pad the last tar block if needed. InnoDB_backup::context: Backup context, attached to backup_sink so that context can continue to exist between the time a BACKUP SERVER releases all locks and another BACKUP SERVER starts executing, with innodb_backup pointing to the new backup, while the old backup is still being finished. InnoDB_backup::queue: Collection of tablespace IDs and payload sizes at the start of the backup, and the log_sys.first_lsn of log files that have to be included in the backup. If any data file is created or extended while the backup is executing, we must have the corresponding write-ahead-log entries that we are copying since the latest checkpoint that was completed when the backup started. If any tablespaces are deleted during the backup, we may or may not copy them, and the application of a FILE_DELETE record will remove them. Similarly, applying FILE_RENAME or FILE_CREATE records will rename or create files during recovery as needed. log_sys.backup: Whether BACKUP SERVER is in progress. The purpose of this is to make BACKUP SERVER prevent the concurrent execution of SET GLOBAL innodb_log_archive=OFF or SET GLOBAL innodb_log_file_size when innodb_log_archive=OFF. log_sys.archived_checkpoint: Keep track of the earliest available checkpoint, corresponding to log_sys.archived_lsn. This reflects SET GLOBAL innodb_log_recovery_start (which is settable now), for incremental backup. fil_system.have_all_spaces: Whether all tablespace metadata is guaranteed to be known. To speed up startup, InnoDB does not normally open all tablespace files. fil_space_t::create_lsn: Change to Atomic_relaxed and use this to indicate tablespace creation LSN, in addition to indicate undo tablespace rebuild LSN. fil_space_t::backup_end: The first page number that is not being backed up (by default 0, to indicate that no backup is in progress). fil_space_t::BACKUP_BATCH_SIZE: The number of preceding pages that will be covered by fil_space_t::backup_end. This is the unit of "page range locking" during InnoDB backup. buf_page_t::write_fix_try(), buf_page_t::write_unfix_try(): Try to set or unset a fake "write fix" on a page, to prevent concurrent flush() during a backup batch. The atomic operations may run concurrently with set_reinit() and set_freed(). The fake "write fix" does not prevent any concurrent read or write of the page data in the buffer pool; it only blocks writes to the underlying data file. buf_page_t::flush(): Atomically test and set write fix, and skip the operation if the fake "write fix" was set. buf_page_t::set_freed(), buf_page_t::set_reinit(): Employ a compare-and-exchange loop to accommodate for the "write fix". innodb_backup_batch_wait(): Look up any pages that we are about to back up. For any dirty pages, invoke buf_page_t::write_fix_try() to try to set a fake "write fix" lock-free. If the page is currently write-fixed between buf_page_t::flush() and buf_page_t::write_complete(), acquire and release a page U-latch to wait for the conflicting write to complete. InnoDB_backup::backup_batch_start(), InnoDB_backup::backup_batch_stop(): Adjust fil_space_t::backup_end and fake "write fix" of dirty pages to protect the copying of a range of pages from the underlying file. log_t::backup_start(): If we were running with innodb_log_archive=ON, ensure that the latest file is a valid recovery starting point. That is, wait for the latest log checkpoint to be within the file. buf_flush_list_space(): Check for concurrent backup before writing each page. This is inefficient, but this function may be invoked from multiple threads concurrently, and it cannot be changed easily, especially for fil_crypt_thread(). fil_ibd_create(): Set fil_space_t::create_lsn after the file has been created. dict_load_tablespaces(): Determine the size of each file if upgrade==true. Backup depends on that. buf_dblwr_t::begin(), buf_dblwr_t::end(), buf_dblwr_t::size(): Accessors to allow BACKUP SERVER to skip the contents of the doublewrite buffer in the system tablespace. It is only useful for crash recovery in case a data page had been incompletely written by the time the server was killed. If the server is killed during a backup, the backup will be incomplete and unusable anyway. Furthermore, the page range locking makes page writes and backup mutually exclusive. --- libmysqld/CMakeLists.txt | 1 + mysql-test/collections/buildbot_suites.bat | 1 + mysql-test/main/backup_server.result | 6 + mysql-test/main/backup_server.test | 10 + mysql-test/main/backup_server_locking.result | 17 + mysql-test/main/backup_server_locking.test | 31 + mysql-test/main/grant_backup_server.result | 27 + mysql-test/main/grant_backup_server.test | 29 + mysql-test/main/mysqld--help.result | 2 +- mysql-test/mariadb-test-run.pl | 1 + .../suite/backup/backup_innodb,debug.rdiff | 16 + .../suite/backup/backup_innodb.combinations | 4 + mysql-test/suite/backup/backup_innodb.opt | 1 + mysql-test/suite/backup/backup_innodb.result | 62 + mysql-test/suite/backup/backup_innodb.test | 108 ++ .../suite/backup/backup_stream,debug.rdiff | 16 + .../suite/backup/backup_stream.combinations | 4 + mysql-test/suite/backup/backup_stream.result | 35 + mysql-test/suite/backup/backup_stream.test | 85 + mysql-test/suite/backup/suite.pm | 19 + .../perfschema/r/max_program_zero.result | 2 +- .../suite/perfschema/r/ortho_iter.result | 2 +- .../perfschema/r/privilege_table_io.result | 2 +- .../r/start_server_disable_idle.result | 2 +- .../r/start_server_disable_stages.result | 2 +- .../r/start_server_disable_statements.result | 2 +- .../start_server_disable_transactions.result | 2 +- .../r/start_server_disable_waits.result | 2 +- .../perfschema/r/start_server_innodb.result | 2 +- .../r/start_server_low_index.result | 2 +- .../r/start_server_low_table_lock.result | 2 +- .../r/start_server_no_account.result | 2 +- .../r/start_server_no_cond_class.result | 2 +- .../r/start_server_no_cond_inst.result | 2 +- .../r/start_server_no_file_class.result | 2 +- .../r/start_server_no_file_inst.result | 2 +- .../perfschema/r/start_server_no_host.result | 2 +- .../perfschema/r/start_server_no_index.result | 2 +- .../perfschema/r/start_server_no_mdl.result | 2 +- .../r/start_server_no_memory_class.result | 2 +- .../r/start_server_no_mutex_class.result | 2 +- .../r/start_server_no_mutex_inst.result | 2 +- ..._server_no_prepared_stmts_instances.result | 2 +- .../r/start_server_no_rwlock_class.result | 2 +- .../r/start_server_no_rwlock_inst.result | 2 +- .../r/start_server_no_setup_actors.result | 2 +- .../r/start_server_no_setup_objects.result | 2 +- .../r/start_server_no_socket_class.result | 2 +- .../r/start_server_no_socket_inst.result | 2 +- .../r/start_server_no_stage_class.result | 2 +- .../r/start_server_no_stages_history.result | 2 +- ...start_server_no_stages_history_long.result | 2 +- .../start_server_no_statements_history.result | 2 +- ...t_server_no_statements_history_long.result | 2 +- .../r/start_server_no_table_hdl.result | 2 +- .../r/start_server_no_table_inst.result | 2 +- .../r/start_server_no_table_lock.result | 2 +- .../r/start_server_no_thread_class.result | 2 +- .../r/start_server_no_thread_inst.result | 2 +- ...tart_server_no_transactions_history.result | 2 +- ...server_no_transactions_history_long.result | 2 +- .../perfschema/r/start_server_no_user.result | 2 +- .../r/start_server_no_waits_history.result | 2 +- .../start_server_no_waits_history_long.result | 2 +- .../perfschema/r/start_server_off.result | 2 +- .../suite/perfschema/r/start_server_on.result | 2 +- .../r/start_server_variables.result | 2 +- .../r/statement_program_lost_inst.result | 2 +- .../suite/sys_vars/r/sysvars_innodb.result | 2 +- sql/CMakeLists.txt | 1 + sql/handler.h | 40 + sql/mysqld.cc | 9 +- sql/mysqld.h | 4 +- sql/sql_backup.cc | 1038 +++++++++++ sql/sql_backup.h | 53 + sql/sql_backup_interface.h | 305 ++++ sql/sql_base.cc | 2 +- sql/sql_base.h | 2 +- sql/sql_command.h | 1 + sql/sql_parse.cc | 4 +- sql/sql_yacc.yy | 28 +- sql/sys_vars.inl | 2 + storage/innobase/CMakeLists.txt | 2 + storage/innobase/btr/btr0sea.cc | 1 - storage/innobase/buf/buf0buf.cc | 126 +- storage/innobase/buf/buf0dblwr.cc | 2 +- storage/innobase/buf/buf0flu.cc | 134 +- storage/innobase/buf/buf0lru.cc | 11 +- storage/innobase/dict/dict0load.cc | 30 +- storage/innobase/fil/fil0fil.cc | 5 +- storage/innobase/handler/backup_innodb.cc | 1585 +++++++++++++++++ storage/innobase/handler/backup_innodb.h | 58 + storage/innobase/handler/ha_innodb.cc | 54 +- storage/innobase/include/buf0buf.h | 27 +- storage/innobase/include/buf0dblwr.h | 7 + storage/innobase/include/fil0fil.h | 30 +- storage/innobase/include/log0log.h | 62 +- storage/innobase/log/log0log.cc | 78 +- storage/innobase/log/log0recv.cc | 11 +- storage/innobase/mtr/mtr0mtr.cc | 69 +- storage/innobase/os/os0file.cc | 10 +- storage/maria/CMakeLists.txt | 1 + storage/maria/ha_maria.cc | 18 + storage/maria/ma_backup_server.c | 379 ++++ storage/maria/ma_backup_server.h | 65 + 105 files changed, 4585 insertions(+), 242 deletions(-) create mode 100644 mysql-test/main/backup_server.result create mode 100644 mysql-test/main/backup_server.test create mode 100644 mysql-test/main/backup_server_locking.result create mode 100644 mysql-test/main/backup_server_locking.test create mode 100644 mysql-test/main/grant_backup_server.result create mode 100644 mysql-test/main/grant_backup_server.test create mode 100644 mysql-test/suite/backup/backup_innodb,debug.rdiff create mode 100644 mysql-test/suite/backup/backup_innodb.combinations create mode 100644 mysql-test/suite/backup/backup_innodb.opt create mode 100644 mysql-test/suite/backup/backup_innodb.result create mode 100644 mysql-test/suite/backup/backup_innodb.test create mode 100644 mysql-test/suite/backup/backup_stream,debug.rdiff create mode 100644 mysql-test/suite/backup/backup_stream.combinations create mode 100644 mysql-test/suite/backup/backup_stream.result create mode 100644 mysql-test/suite/backup/backup_stream.test create mode 100644 mysql-test/suite/backup/suite.pm create mode 100644 sql/sql_backup.cc create mode 100644 sql/sql_backup.h create mode 100644 sql/sql_backup_interface.h create mode 100644 storage/innobase/handler/backup_innodb.cc create mode 100644 storage/innobase/handler/backup_innodb.h create mode 100644 storage/maria/ma_backup_server.c create mode 100644 storage/maria/ma_backup_server.h diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 80b202aa4c03e..59f386a86edae 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -166,6 +166,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/opt_hints.cc ../sql/opt_hints.h ../sql/opt_trace_ddl_info.cc ../sql/opt_trace_ddl_info.h ../sql/sql_path.cc + ../sql/sql_backup.cc ${GEN_SOURCES} ${MYSYS_LIBWRAP_SOURCE} ) diff --git a/mysql-test/collections/buildbot_suites.bat b/mysql-test/collections/buildbot_suites.bat index e673afd50c926..cde563ac97877 100644 --- a/mysql-test/collections/buildbot_suites.bat +++ b/mysql-test/collections/buildbot_suites.bat @@ -6,6 +6,7 @@ innodb,^ versioning,^ plugins,^ mariabackup,^ +backup,^ roles,^ auth_gssapi,^ mysql_sha2,^ diff --git a/mysql-test/main/backup_server.result b/mysql-test/main/backup_server.result new file mode 100644 index 0000000000000..7a064d7e5fe11 --- /dev/null +++ b/mysql-test/main/backup_server.result @@ -0,0 +1,6 @@ +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; diff --git a/mysql-test/main/backup_server.test b/mysql-test/main/backup_server.test new file mode 100644 index 0000000000000..50cf326d39838 --- /dev/null +++ b/mysql-test/main/backup_server.test @@ -0,0 +1,10 @@ +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory diff --git a/mysql-test/main/backup_server_locking.result b/mysql-test/main/backup_server_locking.result new file mode 100644 index 0000000000000..6e49a09e3e873 --- /dev/null +++ b/mysql-test/main/backup_server_locking.result @@ -0,0 +1,17 @@ +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP STAGE START; +connect backup,localhost,root; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR 70100: Query was interrupted: execution time limit 0.1 sec exceeded +connection default; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't execute the command as you have a BACKUP STAGE active +BACKUP STAGE END; +connection backup; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +disconnect backup; +connection default; diff --git a/mysql-test/main/backup_server_locking.test b/mysql-test/main/backup_server_locking.test new file mode 100644 index 0000000000000..4e769b61b2f20 --- /dev/null +++ b/mysql-test/main/backup_server_locking.test @@ -0,0 +1,31 @@ +--source include/not_embedded.inc +--source include/count_sessions.inc + +--mkdir $MYSQLTEST_VARDIR/some_directory +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE START; +--connect (backup,localhost,root) +--error ER_STATEMENT_TIMEOUT +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +--connection default + +--error ER_BACKUP_LOCK_IS_ACTIVE +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE END; +--connection backup +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--disconnect backup +--connection default + +--rmdir $MYSQLTEST_VARDIR/some_directory + +--source include/wait_until_count_sessions.inc diff --git a/mysql-test/main/grant_backup_server.result b/mysql-test/main/grant_backup_server.result new file mode 100644 index 0000000000000..a4d85292dee45 --- /dev/null +++ b/mysql-test/main/grant_backup_server.result @@ -0,0 +1,27 @@ +CREATE USER user1@localhost IDENTIFIED BY ''; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON test.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT RELOAD ON test.* TO user1@localhost; +ERROR HY000: Incorrect usage of DB GRANT and GLOBAL PRIVILEGES +GRANT RELOAD ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the SELECT privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +disconnect con1; +connection default; +DROP USER user1@localhost; diff --git a/mysql-test/main/grant_backup_server.test b/mysql-test/main/grant_backup_server.test new file mode 100644 index 0000000000000..726abb19908bb --- /dev/null +++ b/mysql-test/main/grant_backup_server.test @@ -0,0 +1,29 @@ +--source include/not_embedded.inc +CREATE USER user1@localhost IDENTIFIED BY ''; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON test.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +--error ER_WRONG_USAGE +GRANT RELOAD ON test.* TO user1@localhost; +GRANT RELOAD ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +--disconnect con1 +--connection default +DROP USER user1@localhost; diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result index 4e6593e67af42..93a89adcfef96 100644 --- a/mysql-test/main/mysqld--help.result +++ b/mysql-test/main/mysqld--help.result @@ -2039,7 +2039,7 @@ performance-schema-max-socket-classes 10 performance-schema-max-socket-instances -1 performance-schema-max-sql-text-length 1024 performance-schema-max-stage-classes 170 -performance-schema-max-statement-classes 227 +performance-schema-max-statement-classes 228 performance-schema-max-statement-stack 10 performance-schema-max-table-handles -1 performance-schema-max-table-instances -1 diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 9aadfa5456d61..a0e751d5434d1 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -180,6 +180,7 @@ END main- archive- atomic- + backup- binlog- binlog_encryption- binlog_in_engine- diff --git a/mysql-test/suite/backup/backup_innodb,debug.rdiff b/mysql-test/suite/backup/backup_innodb,debug.rdiff new file mode 100644 index 0000000000000..592e1bf5e57ad --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_innodb.result ++++ backup_innodb,debug.result +@@ -20,7 +20,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER TO 'target_directory' 4 CONCURRENT; ++connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; ++INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; ++connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/backup/backup_innodb.combinations b/mysql-test/suite/backup/backup_innodb.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_innodb.opt b/mysql-test/suite/backup/backup_innodb.opt new file mode 100644 index 0000000000000..0902441fde5cb --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.opt @@ -0,0 +1 @@ +--secure-file-priv=$MYSQLTEST_VARDIR diff --git a/mysql-test/suite/backup/backup_innodb.result b/mysql-test/suite/backup/backup_innodb.result new file mode 100644 index 0000000000000..d93f9b10f9d8c --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.result @@ -0,0 +1,62 @@ +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; +BACKUP SERVER TO '$target_directory'; +ERROR HY000: The MariaDB server is running with the --secure-file-priv option so it cannot execute this statement +BACKUP SERVER TO '$target_directory'; +ROLLBACK; +SELECT * FROM t; +a b +1 +BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH '/bin/false'' at line 1 +BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH 4 CONCURRENT '/bin/false'' at line 1 +BACKUP SERVER WITH '/bin/false'; +ERROR HY000: IO Write error: (...) BACKUP SERVER +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER TO 'target_directory' 4 CONCURRENT; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/some_directory/backup.cnf --datadir=MYSQLTEST_VARDIR/some_directory +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +i +1 +1 +2 +3 +5 +SELECT * FROM at1; +i +2 +3 +5 +7 +DROP TABLE t, at0, at1; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +ERROR 42S02: Table 'test.at0' doesn't exist +SELECT * FROM at1; +ERROR 42S02: Table 'test.at1' doesn't exist +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_innodb.test b/mysql-test/suite/backup/backup_innodb.test new file mode 100644 index 0000000000000..87b56fb7614a2 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.test @@ -0,0 +1,108 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/maybe_debug.inc + +--disable_query_log +call mtr.add_suppression("mariadbd.*: IO Write error:"); +--enable_query_log + +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); + +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; + +--let $target_directory=/tmp/some_directory$MTR_COMBINATION_ARCHIVED +--error ER_OPTION_PREVENTS_STATEMENT +evalp BACKUP SERVER TO '$target_directory'; + +# comment out the following line +# (and replace all "rmdir" with "exec rm -fr" and remove backup_innodb.opt) +# to test cross-filesystem copy +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory + +evalp BACKUP SERVER TO '$target_directory'; +--rmdir $target_directory +ROLLBACK; +# BACKUP SERVER will implicitly commit the current transaction +SELECT * FROM t; + +--error ER_PARSE_ERROR +evalp BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +--error ER_PARSE_ERROR +evalp BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +--replace_regex /\(.*\)/(...)/ +--error ER_IO_WRITE_ERROR +BACKUP SERVER WITH '/bin/false'; + +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +if ($have_debug) { +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +--replace_result $target_directory target_directory +send_eval BACKUP SERVER TO '$target_directory' 4 CONCURRENT; +--connection default +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +SET DEBUG_SYNC='now SIGNAL resume'; +--connection backup +--reap +} +if (!$have_debug) { +--replace_result $target_directory target_directory +eval BACKUP SERVER TO '$target_directory' 4 CONCURRENT; +} + +--disconnect backup +--connection default +ROLLBACK; +SELECT * FROM t; +DELETE FROM t; +DROP TABLE at0, at1; + +if ($MARIADB_UPGRADE_EXE) { +let target_directory=$target_directory; +perl; +open IN, "<", "$ENV{MYSQLTEST_VARDIR}/my.cnf"; +open OUT, ">>", "$ENV{target_directory}/backup.cnf"; +print OUT while (); +close(IN); +close(OUT); +EOF +} +if (!$MARIADB_UPGRADE_EXE) { + --exec cat $MYSQLTEST_VARDIR/my.cnf >> $target_directory/backup.cnf +} +--let $restart_parameters=--defaults-file=$target_directory/backup.cnf --datadir=$target_directory +--source include/restart_mysqld.inc + +SELECT * FROM t; +# A nonzero innodb_log_recovery_target makes InnoDB read-only. +--error ER_OPEN_AS_READONLY +DELETE FROM t; +# Non-InnoDB tables are read-write. +SELECT * FROM at0; +SELECT * FROM at1; +--error ER_OPEN_AS_READONLY +DROP TABLE t, at0, at1; +--error ER_NO_SUCH_TABLE +SELECT * FROM at0; +--error ER_NO_SUCH_TABLE +SELECT * FROM at1; + +--let $restart_parameters= +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; + +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_stream,debug.rdiff b/mysql-test/suite/backup/backup_stream,debug.rdiff new file mode 100644 index 0000000000000..8130727323dec --- /dev/null +++ b/mysql-test/suite/backup/backup_stream,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_stream.result ++++ backup_stream,debug.result +@@ -8,7 +8,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER WITH 2 CONCURRENT 'stream.bat'; ++connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; ++INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; ++connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/backup/backup_stream.combinations b/mysql-test/suite/backup/backup_stream.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_stream.result b/mysql-test/suite/backup/backup_stream.result new file mode 100644 index 0000000000000..e12a2f683c286 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.result @@ -0,0 +1,35 @@ +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +INSERT INTO t SET a=1; +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (1); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (0); +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER WITH 2 CONCURRENT 'stream.bat'; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/backup_stream/backup.cnf --datadir=MYSQLTEST_VARDIR/backup_stream +SELECT * FROM at0; +i +0 +SELECT * FROM at1; +i +1 +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_stream.test b/mysql-test/suite/backup/backup_stream.test new file mode 100644 index 0000000000000..0ad21d49a663c --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.test @@ -0,0 +1,85 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/maybe_debug.inc + +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +INSERT INTO t SET a=1; + +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (1); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (0); + +--remove_files_wildcard $MYSQL_TMP_DIR/. stream.bat +--write_file $MYSQL_TMP_DIR/stream.bat +#!/bin/sh +exec cat > $MYSQL_TMP_DIR/$1.tar +EOF + +--let $script=$MYSQL_TMP_DIR/stream.bat +if (!$MARIADB_UPGRADE_EXE) { +# Because we may run on Linux /dev/shm which may be mounted as noexec, +# we cannot rely on chmod +x, but must explicitly invoke a shell on the script. +--let $script=/bin/sh $script +} +if ($MARIADB_UPGRADE_EXE) +{ +--exec echo "@cat > %MYSQL_TMP_DIR%\%1.tar" > $MYSQL_TMP_DIR/stream.bat +} + +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +if ($have_debug) { +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +--replace_result $script stream.bat +send_eval BACKUP SERVER WITH 2 CONCURRENT '$script'; +--connection default +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +SET DEBUG_SYNC='now SIGNAL resume'; +--connection backup +--reap +} +if (!$have_debug) { +--replace_result $script stream.bat +eval BACKUP SERVER WITH 2 CONCURRENT '$script'; +} + +--remove_file $MYSQL_TMP_DIR/stream.bat +--disconnect backup +--connection default +ROLLBACK; +SELECT * FROM t; +DELETE FROM t; +DROP TABLE at0, at1; + +--let $target_directory=$MYSQLTEST_VARDIR/backup_stream +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory +--mkdir $target_directory +--exec tar xf $MYSQL_TMP_DIR/1.tar -C $target_directory +--exec tar xf $MYSQL_TMP_DIR/2.tar -C $target_directory + +--exec cat $MYSQLTEST_VARDIR/my.cnf >> $target_directory/backup.cnf + +--let $restart_parameters=--defaults-file=$target_directory/backup.cnf --datadir=$target_directory +--source include/restart_mysqld.inc +SELECT * FROM at0; +SELECT * FROM at1; +SELECT * FROM t; +--error ER_OPEN_AS_READONLY +DELETE FROM t; + +--remove_file $MYSQL_TMP_DIR/1.tar +--remove_file $MYSQL_TMP_DIR/2.tar + +--let $restart_parameters= +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; + +--rmdir $target_directory diff --git a/mysql-test/suite/backup/suite.pm b/mysql-test/suite/backup/suite.pm new file mode 100644 index 0000000000000..7d36a22655efa --- /dev/null +++ b/mysql-test/suite/backup/suite.pm @@ -0,0 +1,19 @@ +package My::Suite::Backup; + +@ISA = qw(My::Suite); +use My::Find; +use File::Basename; +use strict; + +return "Not run for embedded server" if $::opt_embedded_server; + +my $have_cat = index(`echo meowl|cat 2>&1`,"meowl") >= 0; +my $have_tar = `tar --version 2>&1` =~ /tar .*\d\.\d/; + +sub skip_combinations { + my %skip; + $skip{'backup_stream.test'} = 'needs cat,tar' unless $have_cat && $have_tar; + %skip; +} + +bless { }; diff --git a/mysql-test/suite/perfschema/r/max_program_zero.result b/mysql-test/suite/perfschema/r/max_program_zero.result index 047643e06988d..a0e486b2af9c0 100644 --- a/mysql-test/suite/perfschema/r/max_program_zero.result +++ b/mysql-test/suite/perfschema/r/max_program_zero.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 1 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/ortho_iter.result b/mysql-test/suite/perfschema/r/ortho_iter.result index 56c22c8453d8e..589704f4056de 100644 --- a/mysql-test/suite/perfschema/r/ortho_iter.result +++ b/mysql-test/suite/perfschema/r/ortho_iter.result @@ -251,7 +251,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/privilege_table_io.result b/mysql-test/suite/perfschema/r/privilege_table_io.result index 0c82be9f05810..b518052613308 100644 --- a/mysql-test/suite/perfschema/r/privilege_table_io.result +++ b/mysql-test/suite/perfschema/r/privilege_table_io.result @@ -57,7 +57,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_idle.result b/mysql-test/suite/perfschema/r/start_server_disable_idle.result index d0665e3bf4c65..12b956d90769c 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_idle.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_idle.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_stages.result b/mysql-test/suite/perfschema/r/start_server_disable_stages.result index 2ef68328144ff..30ce9d12e56a0 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_stages.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_stages.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_statements.result b/mysql-test/suite/perfschema/r/start_server_disable_statements.result index 0ece2a0c52ed1..4bacdbdfedf68 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_statements.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_statements.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result index ededc09aac95d..3a6831eb2c7ff 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_waits.result b/mysql-test/suite/perfschema/r/start_server_disable_waits.result index 23db9362161e4..a864576dfa979 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_waits.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_waits.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_innodb.result b/mysql-test/suite/perfschema/r/start_server_innodb.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_innodb.result +++ b/mysql-test/suite/perfschema/r/start_server_innodb.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_index.result b/mysql-test/suite/perfschema/r/start_server_low_index.result index 11cade0a2132f..dbd36d2eaa92d 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_index.result +++ b/mysql-test/suite/perfschema/r/start_server_low_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result index 484550095202e..fab64f49d45e6 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_account.result b/mysql-test/suite/perfschema/r/start_server_no_account.result index aab8d3eba9caa..e2cccdba19b43 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_account.result +++ b/mysql-test/suite/perfschema/r/start_server_no_account.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result index 4dfdab9de9f30..44b114013ace2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result index a8d0cbeca3855..27ecb59a40f17 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_class.result b/mysql-test/suite/perfschema/r/start_server_no_file_class.result index fcc01880a7107..5189d36618b05 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result index c56201e7d0a80..533c02383c7b8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_host.result b/mysql-test/suite/perfschema/r/start_server_no_host.result index 662beb3b88a49..e328ea3d3c26a 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_host.result +++ b/mysql-test/suite/perfschema/r/start_server_no_host.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_index.result b/mysql-test/suite/perfschema/r/start_server_no_index.result index ccff0cb113faa..686f430cdc440 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_index.result +++ b/mysql-test/suite/perfschema/r/start_server_no_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mdl.result b/mysql-test/suite/perfschema/r/start_server_no_mdl.result index ebe64409deb0c..c2b6dc3d9eb74 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result index 01a217c534bfd..2a4cdcf8ddd37 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result index 1b3efda5210a9..9b77c7b897c47 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result index 599498915f3f1..3a62653efc18f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result index 73ac1acb9f145..7dd5842809135 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result +++ b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result index 04aa037b960e3..3598c722b9453 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result index e8156711eb3f2..7a60805d2d03f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result index 2d17bd6f49203..76da2883ba6b2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result index 16afee28ee70b..58ac763394a96 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result index 9de0006aa7abc..5a86d51a06231 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 0 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result index bcef0e5c01b05..75ab84f4b3cb8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 0 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result index 1dda39dc79e92..bb750ebd34a26 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 0 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result index 95584521eceb2..6cf3f740fb1be 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result index da6c54b6bbafa..1f2716410a9bc 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result index 09a9a544f5b13..f1078542f69ef 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result index 5ce9874799e8b..a6fc523df2c60 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result index a170fe097fd23..304732cb37927 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 0 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result index 3f009de021d1a..3ef43495d0f22 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 0 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result index db4fe3413106b..20cf1a531ca51 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result index b1b6e614c43ac..e8eb4589fce56 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result index e540f3ce78cbd..4dbf292fc3c10 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result index 80da238ba0ed7..09168c68f9fe8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result index f5d32f0100968..9ec63e991deda 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_user.result b/mysql-test/suite/perfschema/r/start_server_no_user.result index cb249b4e242d3..861b7d897c9c5 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_user.result +++ b/mysql-test/suite/perfschema/r/start_server_no_user.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result index c419aad2995f3..710cef232da58 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result index 0b2bf39ff874f..df4351a8f358f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_off.result b/mysql-test/suite/perfschema/r/start_server_off.result index 17db0395d894b..f3bf7bf7fb15e 100644 --- a/mysql-test/suite/perfschema/r/start_server_off.result +++ b/mysql-test/suite/perfschema/r/start_server_off.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_on.result b/mysql-test/suite/perfschema/r/start_server_on.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_on.result +++ b/mysql-test/suite/perfschema/r/start_server_on.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_variables.result b/mysql-test/suite/perfschema/r/start_server_variables.result index f25f9ab69d1c3..b86116e099cdd 100644 --- a/mysql-test/suite/perfschema/r/start_server_variables.result +++ b/mysql-test/suite/perfschema/r/start_server_variables.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result index 442e212557b55..3e8469b881801 100644 --- a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result +++ b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 2 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 97fd9a7c0ef61..f0834ea3a1e12 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1064,7 +1064,7 @@ NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 18446744073709551615 NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL -READ_ONLY YES +READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_LOG_RECOVERY_TARGET SESSION_VALUE NULL diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index e54e894e1d0fc..fd16d901d64fb 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -163,6 +163,7 @@ SET (SQL_SOURCE grant.cc sql_explain.cc sql_analyze_stmt.cc + sql_backup.cc sql_join_cache.cc create_options.cc multi_range_read.cc opt_histogram_json.cc diff --git a/sql/handler.h b/sql/handler.h index 3ab9e0bcd1a8e..2474b6c52f1ce 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -23,6 +23,7 @@ #include "sql_const.h" #include "sql_basic_types.h" +#include "sql_backup_interface.h" #include "mysqld.h" /* server_id */ #include "optimizer_costs.h" #include "sql_plugin.h" /* plugin_ref, st_plugin_int, plugin */ @@ -1892,9 +1893,48 @@ struct handlerton : public transaction_participant /********************************************************************* backup **********************************************************************/ + + /** BACKUP STAGE START */ void (*prepare_for_backup)(void); + /** BACKUP STAGE END */ void (*end_backup)(void); + /** + Start of a BACKUP SERVER phase, + when no backup_step() or backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_PREPARE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to sink, or nullptr + @retval -1 on failure + */ + void *(*backup_start)(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink); + /** + Process a file that was collected in backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int (*backup_step)(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink); + /** + Finish a phase, once all calls for the current phase are completed. + @param thd current sesssion + @param target backup target + @param phase last phase on which backup_start() was successfully + invoked, or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success + */ + int (*backup_end)(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink); + /********************************************************************** WSREP specific **********************************************************************/ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index ead3f926f8cdf..f4f1cc97568ad 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -621,8 +621,10 @@ time_t server_start_time; char mysql_home[FN_REFLEN], pidfile_name[FN_REFLEN], system_time_zone[30]; char *default_tz_name, *opt_path; char log_error_file[FN_REFLEN], glob_hostname[FN_REFLEN], *opt_log_basename; -char mysql_real_data_home[FN_REFLEN], - lc_messages_dir[FN_REFLEN], reg_ext[FN_EXTLEN], +extern "C" { + char mysql_real_data_home[FN_REFLEN]; +} +char lc_messages_dir[FN_REFLEN], reg_ext[FN_EXTLEN], mysql_charsets_dir[FN_REFLEN], *opt_init_file, *opt_tc_log_file, *opt_ddl_recovery_file; char *lc_messages_dir_ptr= lc_messages_dir, *log_error_file_ptr; @@ -3537,6 +3539,7 @@ SHOW_VAR com_status_vars[]= { {"assign_to_keycache", STMT_STATUS(SQLCOM_ASSIGN_TO_KEYCACHE)}, {"backup", STMT_STATUS(SQLCOM_BACKUP)}, {"backup_lock", STMT_STATUS(SQLCOM_BACKUP_LOCK)}, + {"backup_server", STMT_STATUS(SQLCOM_BACKUP_SERVER)}, {"begin", STMT_STATUS(SQLCOM_BEGIN)}, {"binlog", STMT_STATUS(SQLCOM_BINLOG_BASE64_EVENT)}, {"call_procedure", STMT_STATUS(SQLCOM_CALL)}, @@ -9451,7 +9454,7 @@ fn_format_relative_to_data_home(char * to, const char *name, @retval FALSE The path isn't secure */ -bool is_secure_file_path(char *path) +bool is_secure_file_path(const char *path) { char buff1[FN_REFLEN], buff2[FN_REFLEN]; size_t opt_secure_file_priv_len; diff --git a/sql/mysqld.h b/sql/mysqld.h index f04d2bacaa5bb..59c03d4ee5bdf 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -83,7 +83,7 @@ void unlink_thd(THD *thd); void refresh_status_legacy(THD *thd); void refresh_session_status(THD *thd); void refresh_global_status(); -bool is_secure_file_path(char *path); +bool is_secure_file_path(const char *path); extern void init_net_server_extension(THD *thd); extern void handle_accepted_socket(MYSQL_SOCKET new_sock, MYSQL_SOCKET sock); extern void create_new_thread(CONNECT *connect); @@ -703,7 +703,7 @@ extern const char *mysql_real_data_home_ptr; extern ulong thread_handling; extern "C" MYSQL_PLUGIN_IMPORT char server_version[SERVER_VERSION_LENGTH]; extern char *server_version_ptr; -extern MYSQL_PLUGIN_IMPORT char mysql_real_data_home[]; +extern "C" MYSQL_PLUGIN_IMPORT char mysql_real_data_home[]; extern char mysql_unpacked_real_data_home[]; extern MYSQL_PLUGIN_IMPORT struct system_variables global_system_variables; extern char *my_proxy_protocol_networks; diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc new file mode 100644 index 0000000000000..92b7473cc8a9b --- /dev/null +++ b/sql/sql_backup.cc @@ -0,0 +1,1038 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "mdl.h" +#include "mysys_err.h" +#include "sql_class.h" +#include "sql_backup.h" +#include "sql_backup_interface.h" +#include "sql_parse.h" +#include "my_atomic_wrapper.h" +#include "tpool.h" +#include "aligned.h" + +static bool backup_execute(THD *thd, const char *target, const char *command, + int threads); + +/** + @brief BACKUP SERVER command + + BACKUP SERVER copies the storage of a running server optionally + using multiple CONCURRENT threads, each one writing to the target + directory or its own tar stream. + + BACKUP SERVER TO '/path/to/directory' [ 1 CONCURRENT ]; + BACKUP SERVER WITH [ 1 CONCURRENT ] 'command'; + + The operation is divided into multiple phases; @see backup_phase + + Each phase is divided into "start" and "end", which are invoked + by this thread, and a number of "step" in between that may be + invoked by multiple CONCURRENT threads. + @see handlerton::backup_start + @see handlerton::backup_step + @see handlerton::backup_end + + Most phases are tied with BACKUP STAGE locks; the actual first work phase: + @see BACKUP_PHASE_START + @see MDL_BACKUP_START + + The snapshot for transactional storage engines is determined in: + @see BACKUP_PHASE_NO_COMMIT + @see MDL_BACKUP_WAIT_COMMIT + + Some backup_phase for preparation and clean-up are executed while + not holding any MDL. + + @param thd client connection + @return whether the operation failed +*/ +bool Sql_cmd_backup::execute(THD *thd) +{ + return backup_execute(thd, target, command, threads); +} + +#ifdef _WIN32 +using tpool::pread; +using tpool::pwrite; +#else +# include +/** + Copy a file using a memory mapping. + @tparam stream true=write to a stream, false=pwrite to a file + @param p map to source file + @param out_fd destination + @param o start offset + @param end last offset (exclusive) + @return error code + @retval 0 on success + @retval 1 if a memory mapping failed +*/ +template +static ssize_t mmap_copy(const void *p, int out_fd, uint64_t o, uint64_t end) +{ + size_t c= size_t(end - o); + ssize_t ret; + for (const char *b= static_cast(p) + o;; + b+= ret, o+= uint64_t(ret)) + { + const size_t size{std::min(c, size_t(INT_MAX >> 20 << 20))}; + if (stream) + if ((ret= backup_stream_write(out_fd, b, size))) + break; + ret= stream ? size : pwrite(out_fd, b, size, off_t(o)); + if (ret < 0) + break; + c-= ret; + if (!c) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + return ret; +} +#endif + +/** + Copy a file using positioned reads. + @tparam stream true=write to a stream, false=pwrite to a file + @param in_fd source file + @param out_fd destination + @param o start offset + @param end last offset (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +template +static ssize_t pread_write(backup::handle in_fd, backup_fd out_fd, + uint64_t o, uint64_t end) + noexcept +{ + constexpr size_t READ_WRITE_SIZE= 65536; + char *b= static_cast(aligned_malloc(READ_WRITE_SIZE, 4096)); + if (!b) + return -1; + ssize_t ret; + for (uint64_t count{end - o};; o+= ret) + { + ret= pread(in_fd, b, + ssize_t(std::min(count, READ_WRITE_SIZE)), o); + if (ret > 0) + { + if (!stream) + ret= pwrite(out_fd, b, size_t(ret), o); + else if (backup_stream_write(out_fd, b, size_t(ret))) + { + ret= -1; + break; + } + } + if (ret < 0) + break; + count-= uint64_t(ret); + if (!count) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + aligned_free(b); + return ret; +} + +#ifdef __APPLE__ +/* The inline copy_entire_file() invokes fcopyfile() */ +#elif defined _WIN32 +/* CopyFileEx() should be used */ +#else +/** Copy a file (whole content). +@param src source file descriptor +@param dst target to append src to +@return error code (non-positive) +@retval 0 on success */ +extern "C" int copy_entire_file(int src, int dst) +{ + uint64_t end(lseek(src, 0, SEEK_END)); +#ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(src, 0, 0, POSIX_FADV_SEQUENTIAL); +#endif + int ret; +# ifdef copy_file_shortcut + ret= int(copy_file_shortcut(src, dst, 0, end)); + if (ret == 1) +# endif + { +# ifdef copy_file_mmap + void *p= mmap(nullptr, size_t{end}, PROT_READ, MAP_SHARED, src, 0); + if (p != MAP_FAILED) + { + ret= int(copy_file_mmap(p, dst, 0, end)); + munmap(p, size_t{end}); + } + else +# endif + { + ret= backup::copy(src, dst, 0, end); + } + } +#ifdef POSIX_FADV_DONTNEED + std::ignore= posix_fadvise(src, 0, 0, POSIX_FADV_DONTNEED); +#endif + return ret; +} +#endif + +#if defined __linux__ || defined __FreeBSD__ +using copying_step= ssize_t(int,int,size_t,off_t*); +template +static ssize_t stepwise(int in_fd, int out_fd, off_t offset, off_t end) + noexcept +{ + for (;;) + { + const size_t c{size_t(std::min(end - offset, INT_MAX >> 20 << 20))}; + ssize_t ret= step(in_fd, out_fd, c, &offset); + if (ret < 0) + { + if (nonblocking && errno == EAGAIN) + continue; + return ret; + } + if (offset == end) + return 0; + if (!ret) + return -1; + } +} + +/* Copy between files in a single (type of) file system */ +static inline ssize_t +copy_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return copy_file_range(in_fd, offset, out_fd, offset, count, 0); +} +# define cfr(src,dst,start,end) stepwise(src, dst, start, end) +#endif + +namespace backup { +/** Copy a portion of a file. +@param src source file descriptor +@param dst target to append src to +@param start first offset to copy +@param end last offset to copy (exclusive) +@return error code (non-positive) +@retval 0 on success */ +int copy(handle src, backup_fd dst, uint64_t start, uint64_t end) noexcept +{ + assert(end >= start); +#ifdef __FreeBSD__ + /* On FreeBSD, copy_file_range() without flags just works */ + return int(cfr(src, dst, off_t(start), off_t(end))); +#else + ssize_t ret{pread_write(src, dst, start, end)}; + assert(ret <= 0); + return int(ret); +#endif +} + +/** + Append a file snippet to stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_zeropad() must be invoked. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int append(handle src, backup_fd stream, uint64_t start, uint64_t end) noexcept +{ + assert(stream != backup_sink::NO_STREAM); + /* + It is not safe to send from an mmap(2) on src, because we cannot + guarantee that the receiving end of the pipe has consumed + everything before our caller re-enables writes to this src region. + + On Linux, even for MAP_PRIVATE, the following has been documented: + It is unspecified whether changes made to the file after the + mmap() call are visible in the mapped region. + */ + return int(pread_write(src, stream, start, end)); +} +}; + +extern "C" int backup_stream_append_plain(backup_fd src, backup_fd stream, + uint64_t start, uint64_t end) +{ + /* + On Windows, this invokes native_file_handle::native_file_handle(HANDLE). + + Elsewhere, this should be a tail-call, for example JMP rel32 + (0xe9) on IA-32 or AMD64. This non-inline wrapper exists only + because the basic API target is C, not C++, which is required + because of Windows. + */ + return backup::append(src, stream, start, end); +} + +/** + Zero-pad a the stream to a multiple of 512 bytes. + + @param stream backup stream + @param written least significant bits of the number of payload appended + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_zeropad(backup_fd stream, size_t written) +{ + static constexpr const char zerobuf[511]{'\0'}; + written&= 511; + return written ? backup_stream_write(stream, zerobuf, 512 - written) : 0; +} + +#ifdef __linux__ +/** + Try to copy a portion of a file via copy_file_range(2). + @param src source file descriptor + @param dst target to append src to + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success + @retval 1 if a fallback to copy_mmap() or backup::copy() is needed +*/ +extern "C" +int copy_file_range_try(int src, int dst, uint64_t start, uint64_t end) +{ + assert(end >= start); + ssize_t ret{cfr(src, dst, off_t(start), off_t(end))}; + assert(ret <= 0); + if (ret && (errno == EOPNOTSUPP || errno == EXDEV)) + return 1; + return int(ret); +} +#endif + +#ifdef copy_file_mmap +/** + Copy from a memory mapping to a file. + @param map source file mapping + @param dst target to append map to + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int +copy_mmap(const void *map, int dst, uint64_t start, uint64_t end) +{ + return int(mmap_copy(map, dst, start, end)); +} +#endif + +/** Append to the configuration file. +@param target backup target directory +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +extern "C" int backup_config_append(IF_WIN(const char*, int) target, + const char *config, size_t size) +{ + /* FIXME: append to a pre-created configuration file */ +#ifdef _WIN32 + HANDLE dst; + { + std::string path{target}; + path.append("/backup.cnf"); + dst= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (dst != INVALID_HANDLE_VALUE) + { + BOOL ok; + for (;;) + { + DWORD written; + ok= WriteFile(dst, config, DWORD(size), &written, nullptr); + if (ok || GetLastError() != ERROR_IO_PENDING) + break; + assert(written < DWORD(size)); + config+= written; + size-= size_t(written); + } + if (CloseHandle(dst) & ok) + return 0; + } + } +#else + int dst= openat(target, "backup.cnf", + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + return dst; + ssize_t ret; + for (; (ret= write(dst, config, size)) >= 0; config+= ret, size -= ret) + { + assert(size_t(ret) <= size); + if (!(size-= size_t(ret))) + { + ret= 0; + break; + } + } + if (!(close(dst) | ret)) + return 0; +#endif + my_error(ER_CANT_CREATE_FILE, MYF(0), "backup.cnf", errno); + return -1; +} + +/** backup context */ +struct backup_target_phase +{ + /** target directory or stream */ + backup_target target; + /** current phase of backup */ + backup_phase phase; + /** backup worker state (output stream) */ + backup_sink sink; + /** stream object for streaming backup */ + FILE *stream; + /** handlerton::backup_step return value in multi-threaded operation */ + int ret; + /** engine-specific backup context */ + std::unordered_map &ha_data; +}; + +/** + Inform a storage engine of an upcoming backup by invoking + handlerton::backup_start(BACKUP_PHASE_PREPARE_START) before + acquiring any locks. + @param thd current session + @param plugin storage engine + @return whether the operation failed +*/ +static my_bool backup_preparation(THD *thd, plugin_ref plugin, void*) noexcept +{ + const auto bs= plugin_hton(plugin)->backup_start; + return bs && bs(thd, nullptr, BACKUP_PHASE_PREPARE_START, nullptr); +} + +/** + Invoke handlerton::backup_start() on a storage engine, + when there are no pending handlerton::backup_step() in any thread. + @param thd current session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_start(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + assert(int{t.phase} >= 0 || t.phase == BACKUP_PHASE_FINISH); + if (hton->backup_start) + { + t.sink.ha_data= t.ha_data[hton]; + void *data= hton->backup_start(thd, &t.target, t.phase, &t.sink); + if (data == reinterpret_cast(-1)) + return true; + assert(!t.ha_data[hton] || t.ha_data[hton] == data); + t.ha_data[hton]= data; + } + return false; +} + +/** + Invoke handlerton::backup_end() on a storage engine, + when there are no pending handlerton::backup_step() in any thread. + @param thd current session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_end(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + if (hton->backup_end) + { + t.sink.ha_data= t.ha_data[hton]; + return hton->backup_end(thd, &t.target, t.phase, &t.sink); + } + return false; +} + +/** + Invoke handlerton::backup_step() on a storage engine in a thread + that may or may not be associated with a BACKUP SERVER connection, + between handlerton::backup_start() and handlerton::backup_end() + of the same backup_phase. + @param thd the BACKUP SERVER session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_step(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + assert(int{t.phase} >= 0 || t.phase == BACKUP_PHASE_FINISH); + int res= 0; + if (hton->backup_step) + { + t.sink.ha_data= t.ha_data[hton]; + while ((res= hton->backup_step(thd, &t.target, t.phase, &t.sink))) + if (res < 0) + break; + } + return res != 0; +} + +/** Number of background tasks executing backup_step_callback */ +static Atomic_counter backup_step_callback_pending{0}; + +/** Invoke backup_step() in a background task */ +static void backup_step_callback(void *arg) noexcept +{ + backup_target_phase &t{*static_cast(arg)}; + assert(!t.ret); + t.ret= plugin_foreach_with_mask(nullptr, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, &t); +#ifndef NDEBUG + auto was_pending= +#endif + backup_step_callback_pending--; + assert(was_pending); +} + +/** + Execute all handlerton::backup_step() until completion or failure. + @param thd current connection + @param target_phase backup target and phase + @param threads number of execution threads + @param tp thread pool +*/ +static bool backup_steps(THD *thd, backup_target_phase *target_phase, + int threads, tpool::thread_pool *tp) +{ + assert(!backup_step_callback_pending); + if (threads == 1) + return plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + tpool::task *const tasks= + static_cast(alloca(threads * sizeof *tasks)); + backup_step_callback_pending= threads - 1; + for (int n{threads}; --n; ) + { + target_phase[n].phase= target_phase->phase; + tp->submit_task(new (&tasks[n]) tpool::task{backup_step_callback, + &target_phase[n]}); + } + bool fail= plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + while (backup_step_callback_pending) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + if (fail) + return fail; + + for (int n{threads}; --n; ) + if (target_phase[n].ret) + { + my_error(ER_UNKNOWN_ERROR, MYF(0)); + return true; + } + + return false; +} + +/** + BACKUP SERVER driver + + @param thd client connection + @param target BACKUP SERVER TO directory to be created + @param command BACKUP SERVER WITH command + @param threads number of CONCURRENT threads to employ + @return whether the operation failed +*/ +static bool backup_execute(THD *thd, const char *target, const char *command, + int threads) +{ + assert(!!target == !command); + assert(threads > 0); + + if (check_global_access(thd, RELOAD_ACL) || + check_global_access(thd, SELECT_ACL)) + return true; + + if (!target); + else if (error_if_data_home_dir(target, "BACKUP SERVER TO")) + return true; + else if (!is_secure_file_path(target)) + { + my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0), "--secure-file-priv"); + return true; + } + + if (thd->current_backup_stage != BACKUP_FINISHED) + { + my_error(ER_BACKUP_LOCK_IS_ACTIVE, MYF(0)); + return true; + } + + bool fail{plugin_foreach_with_mask(thd, backup_preparation, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + nullptr)}; + if (fail) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return true; + } + + /* Block concurrent BACKUP SERVER and BACKUP STAGE */ + MDL_request mdl_request; + MDL_REQUEST_INIT(&mdl_request, MDL_key::BACKUP, "", "", MDL_BACKUP_START, + MDL_EXPLICIT); + + if (thd->mdl_context.acquire_lock(&mdl_request, + thd->variables.lock_wait_timeout)) + return true; + + tpool::thread_pool *tp= nullptr; + std::unordered_map ha_data{}; + backup_target_phase *target_phase= static_cast + (alloca(threads * sizeof *target_phase)); + if (threads > 1 && !(tp= tpool::create_thread_pool_generic())) + { + oor: + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + err_exit: + thd->mdl_context.release_lock(mdl_request.ticket); + delete tp; + return true; + } + + if (command) + { + char cmd[1024]; + for (int t{threads}; t; ) + { + if (snprintf(cmd, sizeof cmd, "%s %d", command, t) >= int(sizeof cmd)) + goto oor; + FILE *f= my_popen(cmd, "w"); + if (!f) + { + while (t < threads) + my_pclose(target_phase[t++].stream); + goto oor; + } +#ifdef _WIN32 + HANDLE sink= (HANDLE) _get_osfhandle(_fileno(f)); +#else + int sink= fileno(f); +#endif + new (&target_phase[--t]) + backup_target_phase{backup_target{IF_WIN(nullptr, -1)}, + BACKUP_PHASE_START, backup_sink{sink, nullptr}, f, 0, ha_data}; + } + } + else if (my_mkdir(target, 0755, MYF(MY_WME))) + goto err_exit; + else + { +#ifndef _WIN32 + const int dir{open(target, O_DIRECTORY)}; + if (dir < 0) + { + my_error(EE_CANT_MKDIR, MYF(ME_BELL), target, errno); + goto err_exit; + } +#endif + for (int t{threads}; t; ) + { + new (&target_phase[--t]) + backup_target_phase{backup_target{IF_WIN(target, dir)}, + BACKUP_PHASE_START, + backup_sink{backup_sink::NO_STREAM, nullptr}, nullptr, 0, ha_data}; + } + } + + static_assert(int{MDL_BACKUP_START} + 1 == int{MDL_BACKUP_FLUSH}, ""); + static_assert(int{MDL_BACKUP_START} + 2 == int{MDL_BACKUP_WAIT_FLUSH}, ""); + static_assert(int{MDL_BACKUP_START} + 3 == int{MDL_BACKUP_WAIT_DDL}, ""); + static_assert(int{MDL_BACKUP_START} + 4 == int{MDL_BACKUP_WAIT_COMMIT}, ""); + static_assert(int{BACKUP_PHASE_START} + 1 == + int{BACKUP_PHASE_NO_BEGIN_NON_TRANS}, ""); + static_assert(int{BACKUP_PHASE_START} + 2 == + int{BACKUP_PHASE_NO_DML_NON_TRANS}, ""); + static_assert(int{BACKUP_PHASE_START} + 3 == int{BACKUP_PHASE_NO_DDL}, ""); + static_assert(int{BACKUP_PHASE_START} + 4 == + int{BACKUP_PHASE_NO_COMMIT}, ""); + int phase= int{BACKUP_PHASE_START}; + goto backup_phase_start; + + for (; phase <= int{BACKUP_PHASE_NO_COMMIT}; phase++) + { + assert(!fail); + { + const enum_mdl_type mdl= + enum_mdl_type(phase - int{BACKUP_PHASE_START} + int{MDL_BACKUP_START}); + fail= + thd->mdl_context.upgrade_shared_lock(mdl_request.ticket, mdl, + thd->variables.lock_wait_timeout); + if (fail) + break; + } + backup_phase_start: + target_phase->phase= backup_phase(phase); + fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + if (fail) + break; + fail= backup_steps(thd, target_phase, threads, tp) || + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + if (fail) + break; + } + + /* The final part must not interfere with the use of the server datadir. + Release the locks. */ + thd->mdl_context.release_lock(mdl_request.ticket); + if (!fail) + { + target_phase->phase= BACKUP_PHASE_FINISH; + fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase) || + backup_steps(thd, target_phase, threads, tp); + } + else + { + target_phase->phase= BACKUP_PHASE_ABORT; + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, target_phase); + target_phase->phase= BACKUP_PHASE_FINISH; + } + + fail= + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase) || fail; + delete tp; + + if (command) + { + int pfail{0}; + for (int t= threads; t--; ) + pfail|= my_pclose(target_phase[t].stream); + if (!fail && (fail= !!pfail)) + my_error(ER_NET_ERROR_ON_WRITE, MYF(0)); + } +#ifndef _WIN32 + else + std::ignore= close(target_phase->target.fd); +#endif + + if (!fail) + my_ok(thd); + return fail; +} + +/** + Encode an octal string. + @param start first byte of buffer + @param end first byte after buffer + @param n number to encode +*/ +static void ustar_write_octal(char *start, char *end, uint64_t n) noexcept +{ + for (*--end= '\0'; --end >= start; n>>= 3) + *end= char('0' + (n & 7)); +} + +/** + Encode a quantity in 12 bytes. + @param start first byte of the buffer + @param n number to encode +*/ +static void ustar_write_dozen(char *start, uint64_t n) noexcept +{ + if (n < 1ULL << 33) + ustar_write_octal(start, start + 12, n); + else + { + const uint32_t head{my_htobe32(1U << 31)}; + n= my_htobe64(n); + memcpy(start, &head, 4); + memcpy(start + 4, &n, 8); + } +} + +/** Initialize a ustar block +@param buf GNU tape archiver --format=oldgnu header block +@param name name of the block +@param mode file access mode +@param size physical size of the following block */ +static void ustar_block_init(char *buf, const char *name, mode_t mode, + uint64_t size) noexcept +{ + strncpy(buf, name, 100); + ustar_write_octal(buf + 100, buf + 108, uint64_t(mode)); + ustar_write_octal(buf + 108, buf + 116, 0/* POSIX uid */); + ustar_write_octal(buf + 116, buf + 124, 0/* POSIX gid */); + ustar_write_dozen(buf + 124, size); + /* last modification time */ + ustar_write_octal(buf + 136, buf + 148, 0); + memset(buf + 148, ' ', 9); /* initial block checksum and dummy type */ + memset(buf + 157, '\0', 100); /* name of linked file (unused) */ + memcpy(buf + 257, "ustar ", 8); + strncpy(buf + 265, "root" /* POSIX owner name */, 32); + strncpy(buf + 297, "root" /* POSIX group name */, 512 - 297); + /* The caller will fill in the rest and invoke ustar_block_checksum() */ +} + +/** + Compute and write the POSIX tar block checksum. + @param buf POSIX tar block +*/ +static void ustar_block_checksum(char *buf) noexcept +{ + uint16_t sum{0}; + for (int i{0}; i < 512; i++) + sum+= uint16_t{uint8_t(buf[i])}; + ustar_write_octal(buf + 148, buf + 155, sum); +} + +/** + Write data to a stream. + @param stream backup stream + @param buf source buffer + @param size length of the buffer (usually an integer multiple of 512) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_write(backup_fd stream, const void *buf, + size_t size) +{ +#ifdef _WIN32 + for (DWORD sz= DWORD(size);;) + { + DWORD wrote; + if (WriteFile(stream, buf, sz, &wrote, nullptr)) + { + assert(wrote == sz); + return 0; + } + else if (GetLastError() != ERROR_IO_PENDING) + return -1; + buf= static_cast(buf) + wrote; + sz-= wrote; + } +#else + do + { + ssize_t wrote= write(stream, buf, size); + assert(wrote <= ssize_t(size)); + if (wrote < 0) + { + if (errno == EAGAIN) + continue; + return -1; + } + buf= static_cast(buf) + wrote; + size-= size_t(wrote); + } + while (size); +#endif + return 0; +} + +/** + Copy a prefix of a NUL terminated string to a buffer, NUL-padded. + @param b output buffer + @param s NUL terminated string + @param size size of buf, in bytes +*/ +static inline char *ustar_zeropad(char *b, const char *s, size_t size) noexcept +{ +#if defined __GNUC__ && __GNUC__ >= 8 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wstringop-truncation" +#endif + return strncpy(b, s, size); +#if defined __GNUC__ && __GNUC__ >= 8 +# pragma GCC diagnostic pop +#endif +} + +/** Start streaming a file. +@param stream backup stream +@param name file name +@param mode file access mode +@param size physical length of the file, in bytes +@param chunks payload chunks of a sparse file, or nullptr +@param n_chunks number of chunks; 0 unless sparse file +@return error code (non-positive) +@retval 0 on success */ +extern "C" +int backup_stream_start(backup_fd stream, + const char *name, mode_t mode, uint64_t size, + const struct backup_chunk *chunks, size_t n_chunks) +{ + assert(stream != backup_sink::NO_STREAM); + char buf[512]; + size_t s= strlen(name); + if (s > 100) + { + /* Write a block that contains the full name length, + followed by blocks that contain the full name, in + tar --format=oldgnu */ + ustar_block_init(buf, "././@LongLink", 0644, s); + ustar_block_checksum(buf); + if (int err= backup_stream_write(stream, buf, sizeof buf)) + return err; + const size_t whole{s & ~(sizeof buf)}; + if (whole) + if (int err= backup_stream_write(stream, name, whole)) + return err; + if (s - whole) + { + ustar_zeropad(buf, name + whole, sizeof buf); + if (int err= backup_stream_write(stream, buf, sizeof buf)) + return err; + } + } + + ustar_block_init(buf, name, mode, size); + if (!n_chunks) + buf[156]= '0'; + else + { + buf[156]= 'S'; + char *h= &buf[386]; + if (n_chunks > 4) + return -1; // FIXME; support more chunks + for (size_t i= 0; i < n_chunks; i++, h+= 24) + { + ustar_write_dozen(h, chunks[i].offset); + ustar_write_dozen(h + 12, chunks[i].length); + } + ustar_write_dozen(&buf[0x1e3], chunks[n_chunks - 1].offset + + chunks[n_chunks - 1].length); + } + ustar_block_checksum(buf); + return backup_stream_write(stream, buf, sizeof buf); +} + +/** Append to the configuration file. +@param target backup stream +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +extern "C" int backup_stream_config(backup_fd stream, + const char *config, size_t size) +{ + /* FIXME: append to a pre-created configuration file */ + if (int ret= + backup_stream_start(stream, "backup.cnf", 0644, size, nullptr, 0)) + return ret; + char buf[512]; + const size_t whole{size & ~((sizeof buf) - 1)}; + if (whole) + if (int err= backup_stream_write(stream, config, whole)) + return err; + if (size == whole) + return 0; + ustar_zeropad(buf, config + whole, sizeof buf); + return backup_stream_write(stream, buf, sizeof buf); +} + +#ifndef _WIN32 +# ifdef __linux__ +# include +/** Copy a file to a stream or to a regular file. */ +static inline ssize_t +send_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return sendfile(out_fd, in_fd, offset, count); +} +# endif + +/** + Zero-copy append an immutable file snippet to a stream. + + The caller guarantees that this section of the file will remain + intact until the stream is closed. This guarantee is needed + because the receiving end of the stream pipe might delay consuming + the data, and the operating system might point the pipe buffer + to the block cache for a long time. + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_zeropad() must be invoked. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_append_async(int src, int stream, + uint64_t start, uint64_t end) +{ + assert(stream != backup_sink::NO_STREAM); +# ifdef __linux__ + return int(stepwise(src, stream, off_t(start), off_t(end))); +# else +# if SIZEOF_SIZE_T > 4 + void *p= mmap(nullptr, size_t{end}, PROT_READ, MAP_SHARED, src, 0); + if (p != MAP_FAILED) + { + ssize_t ret= mmap_copy(p, stream, start, end); + munmap(p, size_t{end}); + return int(ret); + } +# endif + return int(pread_write(src, stream, start, end)); +# endif +} +#endif diff --git a/sql/sql_backup.h b/sql/sql_backup.h new file mode 100644 index 0000000000000..2759e0d64b43d --- /dev/null +++ b/sql/sql_backup.h @@ -0,0 +1,53 @@ +/***************************************************************************** +Copyright (c) 2026 MariaDB plc. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +#pragma once + +/** BACKUP SERVER */ +class Sql_cmd_backup : public Sql_cmd +{ + /** target directory, or nullptr when streaming */ + const char *const target{nullptr}; + /** argument of my_popen() for streaming backup, or nullptr */ + const char *const command{nullptr}; + /** number of concurrent threads to use */ + const int threads; + +public: + /** + Constructor. + @param target name of target or scratch directory + @param threads number of concurrent threads to use + */ + Sql_cmd_backup(LEX_CSTRING target, int threads) : + target(target.str), threads(threads) {} + /** + Constructor. + @param threads number of concurrent threads to use + @param command nullptr, or a shell command for handling a backup stream + */ + Sql_cmd_backup(int threads, LEX_CSTRING command) : + command(command.str), threads(threads) {} + ~Sql_cmd_backup()= default; + + bool execute(THD *thd) override; + + enum_sql_command sql_command_code() const override + { + return SQLCOM_BACKUP_SERVER; + } +}; diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h new file mode 100644 index 0000000000000..229b944ac9092 --- /dev/null +++ b/sql/sql_backup_interface.h @@ -0,0 +1,305 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#pragma once +#include + +/** BACKUP SERVER target */ +struct backup_target +{ +#ifdef _WIN32 + /** Target directory path name, or nullptr if streaming */ + const char *path; +#else + /** Target directory descriptor, or -1 if streaming */ + int fd; +#endif +}; + +/** BACKUP SERVER worker specific context */ +struct backup_sink +{ +#ifdef _WIN32 +# ifdef __cplusplus + /** A value indicating an invalid stream */ + static constexpr HANDLE NO_STREAM{INVALID_HANDLE_VALUE}; +# endif + /** Target pipe, or NO_STREAM if path!=nullptr */ + HANDLE stream; +#else +# ifdef __cplusplus + /** A value indicating an invalid file descriptor or stream */ + static constexpr int NO_STREAM{-1}; +# endif + /** Target pipe, or NO_STREAM if copying to a directory */ + int stream; +#endif + /** storage engine context returned by handlerton::backup_start() */ + void *ha_data; +}; + +/** BACKUP SERVER execution phase; @see Sql_cmd_backup::execute() */ +enum backup_phase +{ + /** finish backup, possibly after BACKUP_PHASE_ABORT */ + BACKUP_PHASE_FINISH= -2, + /** abort any operation */ + BACKUP_PHASE_ABORT= -1, + /** preparatory phase executed while holding no locks */ + BACKUP_PHASE_PREPARE_START= 0, + /** initial actual work phase; @see MDL_BACKUP_START */ + BACKUP_PHASE_START, + /** copy while new writes to non-transactional tables are blocked; + @see MDL_BACKUP_FLUSH */ + BACKUP_PHASE_NO_BEGIN_NON_TRANS, + /** copy while any writes to non-transactional tables are blocked; + @see MDL_BACKUP_WAIT_FLUSH */ + BACKUP_PHASE_NO_DML_NON_TRANS, + /** copy files while DDL is blocked; @see MDL_BACKUP_WAIT_DDL */ + BACKUP_PHASE_NO_DDL, + /** determine the logical time of the backup and copy any + remaining files while MDL_BACKUP_WAIT_COMMIT is active; + this is followed by BACKUP_PHASE_FINISH */ + BACKUP_PHASE_NO_COMMIT +}; + +/** A payload chunk in a sparse file that is being streamed */ +struct backup_chunk +{ + /** byte offset of the start of the payload, from the start of the file */ + uint64_t offset; + /** length of the hole */ + uint64_t length; +}; + +/** File descriptor */ +typedef IF_WIN(HANDLE, int) backup_fd; + +#ifdef _WIN32 +/* Use CopyFileEx() to copy entire files */ +#elif defined __APPLE__ +/* You should invoke fclonefileat(2) manually before attempting +copy_entire_file() or backup::copy() */ +# include +# include +# include +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (negative) +@retval 0 on success */ +inline int copy_entire_file(int src, int dst) +{ + return fcopyfile(src, dst, NULL, COPYFILE_ALL | COPYFILE_CLONE); +} +#else +# ifdef __cplusplus +extern "C" +# endif +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (non-positive) +@retval 0 on success */ +int copy_entire_file(int src, int dst); +#endif + +#ifdef __cplusplus +# ifdef _WIN32 +struct native_file_handle; +# endif +namespace backup { + +typedef IF_WIN(native_file_handle, int) handle; + +/** + Copy a portion of a file. + @param src source file descriptor + @param dst target to append src to + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int copy(handle src, backup_fd dst, uint64_t start, uint64_t end) noexcept; + +/** + Append a file snippet to the stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int append(handle src, backup_fd stream, uint64_t start, uint64_t end) + noexcept; +} +#endif + +#if defined _WIN32 || defined __FreeBSD__ +/* There is no special variant of backup::copy(). */ +#else +# if SIZEOF_SIZE_T > 4 +# ifdef __cplusplus +extern "C" +# endif +/** + Copy from a memory mapping to a file. + @param map source file mapping + @param dst target to append map to + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int copy_mmap(const void *map, int dst, uint64_t start, uint64_t end); +# define copy_file_mmap copy_mmap +# endif + +# ifdef __linux__ +# ifdef __cplusplus +extern "C" +# endif +/** + Try to copy a portion of a file via copy_file_range(2). + @param src source file descriptor + @param dst target to append src to + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success + @retval 1 if a fallback to copy_mmap() or backup::copy() is needed +*/ +int copy_file_range_try(int src, int dst, uint64_t start, uint64_t end); +# define copy_file_shortcut copy_file_range_try +# endif +#endif + +#ifdef __cplusplus +extern "C" +#endif +/** Append to the configuration file. +@param target backup target directory +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +int backup_config_append(IF_WIN(const char*, int) target, + const char *config, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** Append to the configuration file. +@param target backup stream +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +int backup_stream_config(backup_fd stream, const char *config, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** Start streaming a file. +@param target backup target +@param name file name +@param mode file access mode +@param size physical length of the file, in bytes +@param chunks payload chunks of a sparse file, or nullptr +@param n_chunks number of chunks; 0 unless sparse file +@return error code (non-positive) +@retval 0 on success */ +int backup_stream_start(backup_fd stream, + const char *name, mode_t mode, uint64_t size, + const struct backup_chunk *chunks, size_t n_chunks); + +#ifdef __cplusplus +extern "C" +#endif +/** + Write data to a stream. + @param stream backup stream + @param buf source buffer + @param size length of the buffer (usually an integer multiple of 512) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_write(backup_fd stream, const void *buf, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** + Append a file snippet to the stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_zeropad() must be invoked. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_append_plain(backup_fd src, backup_fd stream, + uint64_t start, uint64_t end); + +#ifdef __cplusplus +extern "C" +#endif +/** + Zero-pad a the stream to a multiple of 512 bytes. + + @param stream backup stream + @param written least significant bits of the number of payload appended + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_zeropad(backup_fd stream, size_t written); + +#ifdef _WIN32 +# define backup_stream_append_async backup_stream_append_plain +#else +# ifdef __cplusplus +extern "C" +# endif +/** + Zero-copy append an immutable file snippet to a stream. + + The caller guarantees that this section of the file will remain + intact until the stream is closed. This guarantee is needed + because the receiving end of the stream pipe might delay consuming + the data, and the operating system might point the pipe buffer + to the block cache for a long time. + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_zeropad() must be invoked. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_append_async(int src, int stream, + uint64_t start, uint64_t end); +#endif diff --git a/sql/sql_base.cc b/sql/sql_base.cc index bb1f815eea5c1..c29fae7a9b396 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -321,7 +321,7 @@ OPEN_TABLE_LIST *list_open_tables(THD *thd, Close all tables that are not in use in table definition cache */ -void purge_tables() +extern "C" void purge_tables() { /* Force close of all open tables. diff --git a/sql/sql_base.h b/sql/sql_base.h index 77d2b17a1fe5d..0fb4e8cdf00f9 100644 --- a/sql/sql_base.h +++ b/sql/sql_base.h @@ -320,7 +320,7 @@ void close_log_table(THD *thd, Open_tables_backup *backup); bool close_cached_tables(THD *thd, TABLE_LIST *tables, bool wait_for_refresh, ulong timeout); -void purge_tables(); +extern "C" void purge_tables(); bool flush_tables(THD *thd, flush_tables_type flag); void close_all_tables_for_name(THD *thd, TABLE_SHARE *share, ha_extra_function extra, diff --git a/sql/sql_command.h b/sql/sql_command.h index 9c9166706a034..b8903399711f0 100644 --- a/sql/sql_command.h +++ b/sql/sql_command.h @@ -103,6 +103,7 @@ enum enum_sql_command { SQLCOM_SHOW_PACKAGE_BODY_CODE, SQLCOM_BACKUP, SQLCOM_BACKUP_LOCK, SQLCOM_SHOW_CREATE_SERVER, + SQLCOM_BACKUP_SERVER, /* When a command is added here, be sure it's also added in mysqld.cc diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index dc069695b23b6..4b67aed0f6475 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -781,6 +781,7 @@ void init_update_queries(void) sql_command_flags[SQLCOM_DROP_SERVER]|= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP]= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP_LOCK]= CF_AUTO_COMMIT_TRANS; + sql_command_flags[SQLCOM_BACKUP_SERVER]= CF_AUTO_COMMIT_TRANS; /* The following statements can deal with temporary tables, @@ -5899,6 +5900,7 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt) case SQLCOM_CALL: case SQLCOM_REVOKE: case SQLCOM_GRANT: + case SQLCOM_BACKUP_SERVER: if (thd->variables.option_bits & OPTION_IF_EXISTS) lex->create_info.set(DDL_options_st::OPT_IF_EXISTS); DBUG_ASSERT(lex->m_sql_cmd != NULL); @@ -10249,7 +10251,7 @@ int test_if_data_home_dir(const char *dir) if (!dir) DBUG_RETURN(0); - (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH); + (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH|MY_RESOLVE_SYMLINKS); DBUG_RETURN(path_starts_from_data_home_dir(path)); } diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index b2071b1c50b7f..08bf1fb7d84c8 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -50,6 +50,7 @@ #include "sql_alter.h" // Sql_cmd_alter_table* #include "sql_truncate.h" // Sql_cmd_truncate_table #include "sql_admin.h" // Sql_cmd_analyze/Check..._table +#include "sql_backup.h" #include "sql_partition_admin.h" // Sql_cmd_alter_table_*_part. #include "sql_handler.h" // Sql_cmd_handler_* #include "sql_signal.h" @@ -1473,7 +1474,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %type json_on_response %type json_type_constraint -%type json_key_unique_constraint +%type json_key_unique_constraint opt_concurrent %type json_predicate %type field_type field_type_all field_type_all_builtin @@ -15563,6 +15564,31 @@ backup_statements: /* Table list is empty for unlock */ Lex->sql_command= SQLCOM_BACKUP_LOCK; } + | SERVER_SYM TO_SYM TEXT_STRING_sys opt_concurrent + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3, $4); + } + | SERVER_SYM WITH opt_concurrent TEXT_STRING_sys + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3, $4); + } + ; + +opt_concurrent: + /* empty */ + { $$= 1; } + | ulonglong_num CONCURRENT + { + $$= int($1); + if ($1 < 1 || $1 > 256) + { + my_error(ER_DATA_OUT_OF_RANGE, myf(0), "CONCURRENT", + "BACKUP SERVER"); + MYSQL_YYABORT; + } + } ; opt_delete_gtid_domain: diff --git a/sql/sys_vars.inl b/sql/sys_vars.inl index 0f8aa1eb63bf6..8e5f0b984e81e 100644 --- a/sql/sys_vars.inl +++ b/sql/sys_vars.inl @@ -2506,6 +2506,7 @@ public: bool session_update(THD *thd, set_var *var) override; }; +#ifdef HAVE_REPLICATION /* Class for replicate_events_marked_for_skip. We need a custom update function that ensures the slave is stopped when @@ -2647,6 +2648,7 @@ public: } const uchar *global_value_ptr(THD *thd, const LEX_CSTRING *base) const override; }; +#endif /* HAVE_REPLICATION */ /** diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 9e3a23b34ab46..d63751a16af08 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -185,6 +185,8 @@ SET(INNOBASE_SOURCES handler/handler0alter.cc handler/innodb_binlog.cc handler/i_s.cc + handler/backup_innodb.h + handler/backup_innodb.cc ibuf/ibuf0ibuf.cc include/btr0btr.h include/btr0btr.inl diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 49f9a2a96a0ab..8174c3ac9d66a 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1204,7 +1204,6 @@ btr_search_guess_on_hash( } ut_ad(!block->page.is_read_fixed(state)); - ut_ad(!block->page.is_write_fixed(state) || latch_mode == BTR_SEARCH_LEAF); const dict_index_t *block_index= block->index; if (index != block_index && index_id == block_index->id) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 577973db456b0..17be16e2de657 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -2298,7 +2298,7 @@ void buf_page_free(fil_space_t *space, uint32_t page, mtr_t *mtr) if (block->index) btr_search_drop_page_hash_index(block, nullptr); #endif /* BTR_CUR_HASH_ADAPT */ - block->page.set_freed(block->page.state()); + block->page.set_freed(); mtr->memo_push(block, MTR_MEMO_PAGE_X_MODIFY); } @@ -2367,7 +2367,7 @@ buf_page_t *buf_page_get_zip(const page_id_t page_id) noexcept exclusive latch on this block and either in progress or invoking buf_pool_t::corrupted_evict(). - Let us aqcuire and release buf_pool.mutex to ensure that any + Let us acquire and release buf_pool.mutex to ensure that any buf_pool_t::corrupted_evict() will proceed before we reacquire the hash_lock that it could be waiting for. @@ -2501,7 +2501,12 @@ buf_block_t *buf_pool_t::unzip(buf_page_t *b, buf_pool_t::hash_chain &chain) case buf_page_t::REINIT + 1: break; default: - ut_ad(state < buf_page_t::READ_FIX); + /* + There may be a fake "write fix" if a thread is executing + between InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() on this page. + */ + ut_ad(!buf_page_t::is_read_fixed(state)); if (state < buf_page_t::UNFIXED + 1) { @@ -2520,7 +2525,7 @@ buf_block_t *buf_pool_t::unzip(buf_page_t *b, buf_pool_t::hash_chain &chain) goto wait_for_unfix; } - /* Ensure that another buf_page_get_low() or buf_page_t::page_fix() + /* Ensure that another buf_page_get_low() or buf_pool_t::page_fix() will wait for block->page.lock.x_unlock(). buf_relocate() will copy the state from b to block and replace b with block in page_hash. */ b->set_state(buf_page_t::READ_FIX); @@ -2879,7 +2884,17 @@ buf_page_get_gen( if (!nowait) { goto latch_waited; } else { - ut_ad(state < buf_page_t::READ_FIX); + /* If a thread is executing between + InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() + for this page, a fake "write + fix" may exist. We are free to + modify the page in the buffer + pool, but buf_page_t::flush() + will refuse to write it to the + file system. */ + ut_ad(!buf_page_t:: + is_read_fixed(state)); } /* fall through */ case RW_S_LATCH: @@ -2893,8 +2908,7 @@ buf_page_get_gen( } else { not_read_fixed: ut_ad(state > buf_page_t::FREED); - ut_ad(state < buf_page_t::READ_FIX - || state > buf_page_t::WRITE_FIX); + ut_ad(!buf_page_t::is_read_fixed(state)); if (UNIV_UNLIKELY(!block->page.frame && mode == BUF_PEEK_IF_IN_POOL)) { /* The BUF_PEEK_IF_IN_POOL mode is mainly used @@ -2956,7 +2970,8 @@ buf_page_get_gen( break; case RW_SX_LATCH: block->page.lock.u_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); break; default: ut_ad(rw_latch == RW_X_LATCH); @@ -3039,7 +3054,8 @@ buf_block_t *buf_page_optimistic_get(buf_block_t *block, goto fail; else { - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); if (modify_clock != block->modify_clock || block->page.is_freed()) { @@ -3151,6 +3167,37 @@ buf_pool_t::page_hash_table::replace(buf_pool_t::hash_chain &chain, *prev= bpage; } +/** Mark the block as reinitialized in the file. @see set_freed() */ +void buf_page_t::set_reinit() noexcept +{ + /* + Ensure that lock.x_lock() is being held (hopefully, by us). + This blocks a concurrent flush(), protected by lock.u_lock(). + */ + ut_ad(lock.is_write_locked()); + uint32_t s{state()}; + do + { + /* + InnoDB_backup::backup_batch_start() may set fix() and + write_fix_try() on any dirty block to prevent flush() from + writing changes back to the file system. As long as buf_relocate() + will not be invoked, we may safely mark such blocks as REINIT; + write_unfix_try() will account for that. + */ + ut_ad(s < READ_FIX || (s > WRITE_FIX && frame)); + if (!((REINIT ^ s) & LRU_MASK)) + break; + } + /* + fetch_add() or fetch_sub() are not safe here, because this may run + concurrently with write_fix_try() or write_unfix_try(). + */ + while (!zip.fix.compare_exchange_weak(s, REINIT + buf_fix_count(s), + std::memory_order_acquire, + std::memory_order_relaxed)); +} + static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, mtr_t *mtr, buf_block_t *free_block) noexcept @@ -3198,10 +3245,9 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, mysql_mutex_unlock(&buf_pool.mutex); bpage->lock.x_lock(); - const page_id_t id{bpage->id()}; - if (UNIV_UNLIKELY(id != page_id)) + if (UNIV_UNLIKELY(bpage->id() != page_id)) { - ut_ad(id.is_corrupted()); + ut_ad(bpage->id().is_corrupted()); ut_ad(bpage->is_freed()); bpage->unfix(); bpage->lock.x_unlock(); @@ -3209,26 +3255,23 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, } mysql_mutex_lock(&buf_pool.mutex); state= bpage->state(); - ut_ad(!bpage->is_io_fixed(state)); ut_ad(bpage->buf_fix_count(state)); } else state= bpage->state(); - ut_ad(state > buf_page_t::FREED); - ut_ad(state < buf_page_t::READ_FIX); /* In addition to our buffer-fix, there may be another that is held by a concurrent IORequest::read_complete() that had - released the bpage->lock in bpage->read_complete(...) but not + released the bpage->lock in bpage->read_complete(...) but not yet invoked bpage->unfix(). This should only be due to an - asynchronous read-ahead for a page that was actually marked as - freed in the underlying data file. */ - ut_ad(bpage->buf_fix_count(state) <= 2); + unnecessary asynchronous read-ahead for a page that was actually + marked as freed in the underlying data file. - if (state < buf_page_t::UNFIXED) - bpage->set_reinit(buf_page_t::FREED); - else - bpage->set_reinit(state & buf_page_t::LRU_MASK); + Alternatively, another thread may be executing between + InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() on this page. */ + ut_ad(bpage->buf_fix_count(state) <= 2); + ut_ad(state < buf_page_t::READ_FIX || state > buf_page_t::WRITE_FIX); if (UNIV_LIKELY(bpage->frame != nullptr)) { @@ -3242,23 +3285,40 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, } else { + /* + Augment the compressed-only ROW_FORMAT=COMPRESSED page with + an uncompressed page frame. + */ page_hash_latch &hash_lock= buf_pool.page_hash.lock_get(chain); for (;;) { hash_lock.lock(); state= bpage->state(); - if ((state & ~buf_page_t::LRU_MASK) == 1) + if (state == buf_page_t::FREED + 1 || + state == buf_page_t::UNFIXED + 1 || + state == buf_page_t::REINIT + 1) break; - /* Wait for a concurrent IORequest::read_complete() to - invoke bpage->unfix(), for an unnecessary read-ahead of - a freed page. */ - ut_ad((state & ~buf_page_t::LRU_MASK) == 2); + ut_ad(state > buf_page_t::FREED); + /* + Wait for the competing thread to invoke buf_page_t::unfix() + and possibly buf_page_t::write_unfix_try() before that. + */ + ut_ad(bpage->buf_fix_count(state) == 2); hash_lock.unlock(); + if (state >= buf_page_t::READ_FIX) + { + ut_ad(state > buf_page_t::WRITE_FIX); + /* InnoDB_backup::step() may take a long time. */ + mysql_mutex_unlock(&buf_pool.mutex); + std::this_thread::yield(); + mysql_mutex_lock(&buf_pool.mutex); + } } mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_relocate(bpage, &free_block->page); free_block->page.lock.x_lock(); + free_block->page.set_state(buf_page_t::REINIT + 1); buf_flush_relocate_on_flush_list(bpage, &free_block->page); mysql_mutex_unlock(&buf_pool.flush_list_mutex); @@ -3272,7 +3332,7 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, #endif ut_free(bpage); mtr->memo_push(free_block, MTR_MEMO_PAGE_X_FIX); - bpage= &free_block->page; + return free_block; } } else @@ -3283,12 +3343,9 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, #ifdef BTR_CUR_HASH_ADAPT ut_ad(!reinterpret_cast(bpage)->index); #endif - const auto state= bpage->state(); - ut_ad(state >= buf_page_t::FREED); - bpage->set_reinit(state < buf_page_t::UNFIXED ? buf_page_t::FREED - : state & buf_page_t::LRU_MASK); } + bpage->set_reinit(); #ifdef BTR_CUR_HASH_ADAPT if (drop_hash_entry) btr_search_drop_page_hash_index(reinterpret_cast(bpage), @@ -3831,8 +3888,7 @@ void buf_pool_t::validate() noexcept /* do nothing */ break; default: - if (f >= buf_page_t::READ_FIX - && f < buf_page_t::WRITE_FIX) { + if (buf_page_t::is_read_fixed(f)) { /* A read-fixed block is not necessarily in the page_hash yet. */ break; diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index fb4a7bc5d9958..b9afbb158250f 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -752,7 +752,7 @@ void buf_dblwr_t::flush_buffered_writes_completed(const IORequest &request) static_cast(frame))); ut_ad(lsn); ut_ad(lsn >= bpage->oldest_modification()); - if (lsn < e.request.node->space->get_create_lsn()) + if (lsn < e.request.node->space->create_lsn) { /* mtr_t::commit_shrink() must have been invoked between buf_dblwr_t::flush_buffered_writes() and diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 299da45582cd7..df511cef99843 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -35,6 +35,7 @@ Created 11/11/1995 Heikki Tuuri #include "buf0buf.h" #include "buf0checksum.h" #include "buf0dblwr.h" +#include "backup_innodb.h" #include "srv0start.h" #include "page0zip.h" #include "fil0fil.h" @@ -244,7 +245,7 @@ void buf_flush_remove_pages(uint32_t id) noexcept { const auto s= bpage->state(); ut_ad(s >= buf_page_t::REMOVE_HASH); - ut_ad(s < buf_page_t::READ_FIX || s >= buf_page_t::WRITE_FIX); + ut_ad(!buf_page_t::is_read_fixed(s)); buf_page_t *prev= UT_LIST_GET_PREV(list, bpage); const page_id_t bpage_id(bpage->id()); @@ -786,7 +787,7 @@ bool buf_page_t::flush(fil_space_t *space) noexcept ut_ad((space->is_temporary()) == (space == fil_system.temp_space)); ut_ad(space->referenced()); - const auto s= state(); + uint32_t s{state()}; const lsn_t lsn= mach_read_from_8(my_assume_aligned<8> @@ -812,16 +813,40 @@ bool buf_page_t::flush(fil_space_t *space) noexcept return false; } - if (UNIV_UNLIKELY(lsn < space->get_create_lsn())) + if (UNIV_UNLIKELY(lsn < space->create_lsn)) { ut_ad(!space->is_temporary()); ut_ad(!space->is_being_imported()); goto freed; } - ut_d(const auto f=) zip.fix.fetch_add(WRITE_FIX - UNFIXED); - ut_ad(f >= UNFIXED); - ut_ad(f < READ_FIX); + do + { + ut_ad(s >= UNFIXED); + if (UNIV_UNLIKELY(s >= READ_FIX)) + { + /* + A thread must have successfully executed write_fix_try() after + fix(), executing between InnoDB_backup::backup_batch_start() + and InnoDB_backup::backup_batch_stop(). + */ + ut_ad(s > WRITE_FIX); + /* Backup skips the temporary tablespace */ + ut_ad(oldest_modification() > 2); + lock.u_unlock(true); + return false; + } + } + /* + compare_exchange_strong() ensures that the write-fix was set by us + and not a concurrent write_fix_try(). + */ + while (!zip.fix.compare_exchange_strong(s, s + (WRITE_FIX - UNFIXED), + std::memory_order_acquire, + std::memory_order_relaxed)); + + ut_ad(s >= UNFIXED); + ut_ad(s < READ_FIX); ut_ad((space == fil_system.temp_space) ? oldest_modification() == 2 : oldest_modification() > 2); @@ -1002,6 +1027,14 @@ uint32_t fil_space_t::flush_freed(bool writable) noexcept mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex); mysql_mutex_assert_not_owner(&buf_pool.mutex); + /* Note: There is no need to check if another thread is executing + between InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_end(), because we are only overwriting + freed (garbage) pages. If backup reads a torn page, it will also + have copied a corresponding FREE_PAGE record, which would be applied + on recovery. Besides, the freed page should never be reachable from + other pages that are part of the snapshot. */ + const bool punch_hole= chain.start->punch_hole == 1; if (!punch_hole && !srv_immediate_scrub_data_uncompressed) return 0; @@ -1084,7 +1117,7 @@ static ulint buf_flush_try_neighbors(fil_space_t *space, (FIL_PAGE_LSN + (bpage->zip.data ? bpage->zip.data : bpage->frame))); ut_ad(lsn >= bpage->oldest_modification()); - if (UNIV_UNLIKELY(lsn < space->get_create_lsn())) + if (UNIV_UNLIKELY(lsn < space->create_lsn)) { ut_a(!bpage->flush(space)); mysql_mutex_unlock(&buf_pool.mutex); @@ -1228,13 +1261,16 @@ fil_space_t *fil_space_t::get_for_write(uint32_t id) noexcept /** Start writing out pages for a tablespace. @param id tablespace identifier +@param end fil_space_t::backup_page_end() @return tablespace and number of pages written */ -static std::pair buf_flush_space(const uint32_t id) - noexcept +static std::pair +buf_flush_space(const uint32_t id, uint32_t *end) noexcept { - if (fil_space_t *space= fil_space_t::get_for_write(id)) - return {space, space->flush_freed(true)}; - return {nullptr, 0}; + fil_space_t *space= fil_space_t::get_for_write(id); + if (!space) + return {nullptr, 0}; + *end= space->backup_page_end(); + return {space, space->flush_freed(true)}; } struct flush_counters_t @@ -1294,6 +1330,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1343,11 +1380,12 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, if (state < buf_page_t::READ_FIX && bpage->lock.u_lock_try(true)) { - ut_ad(!bpage->is_io_fixed()); + ut_ad(!bpage->is_read_fixed()); /* tolerate write_fix_try() */ switch (bpage->oldest_modification()) { case 2: /* LRU flushing will always evict pages of the temporary tablespace, in buf_page_write_complete(). */ + ut_ad(!bpage->is_io_fixed()); ++n->evicted; break; case 1: @@ -1361,6 +1399,8 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, /* fall through */ case 0: bpage->lock.u_unlock(true); + /* Reload in case write_fix_try() had been undone meanwhile. */ + state= bpage->state(); goto evict; } /* Block is ready for flush. Dispatch an IO request. */ @@ -1374,9 +1414,9 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, mysql_mutex_unlock(&buf_pool.mutex); if (space) space->release(); - auto p= buf_flush_space(space_id); - space= p.first; last_space_id= space_id; + auto p= buf_flush_space(space_id, &backup_page_end); + space= p.first; if (!space) { mysql_mutex_lock(&buf_pool.mutex); @@ -1411,7 +1451,8 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, break; } - if (neighbors && space->is_rotational() && UNIV_LIKELY(!to_withdraw) && + if (neighbors && UNIV_LIKELY(!(to_withdraw | backup_page_end)) && + space->is_rotational() && /* Skip neighbourhood flush from LRU list if we haven't yet reached half of the free page target. */ UT_LIST_GET_LEN(buf_pool.free) * 2 >= free_limit) @@ -1423,10 +1464,17 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, flush: if (UNIV_UNLIKELY(to_withdraw != 0)) to_withdraw= buf_flush_LRU_to_withdraw(to_withdraw, *bpage); - if (bpage->flush(space)) + const uint32_t page{bpage->id().page_no()}; + if (page < backup_page_end && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) + bpage->lock.u_unlock(true); + else if (bpage->flush(space)) + { ++n->flushed; - else - continue; + goto reacquire_mutex; + } + + continue; } goto reacquire_mutex; @@ -1486,6 +1534,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1527,7 +1576,9 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept if (!bpage->lock.u_lock_try(true)) goto skip; - ut_ad(!bpage->is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist; + we will check it in buf_page_t::flush() */ + ut_ad(!bpage->is_read_fixed()); if (bpage->oldest_modification() == 1) { @@ -1558,9 +1609,9 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.mutex); if (space) space->release(); - auto p= buf_flush_space(space_id); - space= p.first; last_space_id= space_id; + auto p= buf_flush_space(space_id, &backup_page_end); + space= p.first; mysql_mutex_lock(&buf_pool.mutex); buf_pool.stat.n_pages_written+= p.second; mysql_mutex_lock(&buf_pool.flush_list_mutex); @@ -1581,9 +1632,17 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); do { - if (neighbors && space->is_rotational()) + if (neighbors && UNIV_LIKELY(!backup_page_end) && + space->is_rotational()) count+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, count, max_n); + else if (page_id.page_no() < backup_page_end && + page_id.page_no() >= + backup_page_end - space->BACKUP_BATCH_SIZE) + { + bpage->lock.u_unlock(true); + continue; + } else if (bpage->flush(space)) ++count; else @@ -1693,6 +1752,7 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept if (written) buf_pool.stat.n_pages_written+= written; } + mysql_mutex_lock(&buf_pool.flush_list_mutex); for (buf_page_t *bpage= UT_LIST_GET_LAST(buf_pool.flush_list); bpage; ) @@ -1735,18 +1795,26 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept acquired= false; goto was_freed; } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + const uint32_t page{bpage->id().page_no()}; + const uint32_t backup_page_end{space->backup_page_end()}; + if (UNIV_UNLIKELY(page < backup_page_end) && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) + { + bpage->lock.u_unlock(true); + skip: + mysql_mutex_lock(&buf_pool.flush_list_mutex); + may_have_skipped= true; + goto done; + } + if (bpage->flush(space)) { ++n_flush; - if (!--max_n_flush) - { - mysql_mutex_lock(&buf_pool.mutex); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - may_have_skipped= true; - goto done; - } mysql_mutex_lock(&buf_pool.mutex); + if (!--max_n_flush) + goto skip; } } @@ -2071,6 +2139,7 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept this->end_lsn= end_lsn; if (!archive) { + archived_checkpoint= checkpoint; archived_lsn= end_lsn; checkpoint_completed: if (resize_log.m_file == log.m_file) @@ -2088,7 +2157,7 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept /* Make the previous archived log file read-only */ #ifdef _WIN32 resize_log.close(); - SetFileAttributesA(get_archive_path().c_str(), + SetFileAttributesA(get_archive_path(get_first_lsn() - capacity()).c_str(), FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_ARCHIVE); #else struct stat st; @@ -2098,9 +2167,10 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept st.st_mode= 0444; if (fchmod(resize_log.m_file, st.st_mode)) my_error(ER_ERROR_ON_CLOSE, MYF(ME_ERROR_LOG), - get_archive_path().c_str(), errno); + get_archive_path(get_first_lsn() - capacity()).c_str(), errno); resize_log.close(); #endif + innodb_backup_checkpoint(); } else goto checkpoint_completed; diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 95e2a6cbe0972..7b8bf9a3e06b9 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1271,20 +1271,15 @@ void buf_LRU_truncate_temp(uint32_t threshold) /* Set the extent descriptor page state as FREED */ for (uint32_t cur_xdes_page= xdes_calc_descriptor_page( 0, fil_system.temp_space->free_limit); - cur_xdes_page >= threshold;) + cur_xdes_page >= threshold; + cur_xdes_page-= uint32_t(srv_page_size)) { mtr_t mtr{nullptr}; mtr.start(); if (buf_block_t* block= buf_page_get_gen( page_id_t(SRV_TMP_SPACE_ID, cur_xdes_page), 0, RW_X_LATCH, nullptr, BUF_PEEK_IF_IN_POOL, &mtr)) - { - uint32_t state= block->page.state(); - ut_ad(state > buf_page_t::UNFIXED); - ut_ad(state < buf_page_t::READ_FIX); - block->page.set_freed(state); - } - cur_xdes_page-= uint32_t(srv_page_size); + block->page.set_freed(); mtr.commit(); } diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index d8976dec66974..49ffe82fcd49c 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -889,6 +889,10 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) dict_sys.lock(SRW_LOCK_CALL); + if (fil_system.have_all_spaces) { + goto done; + } + if (!spaces && !upgrade && !encryption_key_id_exists(FIL_DEFAULT_ENCRYPTION_KEY)) { max_space_id = dict_find_max_space_id(&pcur, &mtr); @@ -934,7 +938,15 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) newly created or rebuilt tables or partitions, but will otherwise ignore the flag. */ - if (fil_space_for_table_exists_in_mem(space_id, flags)) { + fil_space_t* space + = fil_space_for_table_exists_in_mem(space_id, flags); + + if (space) { + if (upgrade) { + space->get_size(); + } + next: + max_space_id = ut_max(max_space_id, space_id); continue; } @@ -959,11 +971,13 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) const bool not_dropped{!rec_get_deleted_flag(rec, 0)}; /* Check that the .ibd file exists. */ - if (fil_ibd_open(space_id, dict_tf_to_fsp_flags(flags), - not_dropped - ? fil_space_t::VALIDATE_NOTHING - : fil_space_t::MAYBE_MISSING, - name, filepath)) { + space = fil_ibd_open( + space_id, dict_tf_to_fsp_flags(flags), + not_dropped + ? fil_space_t::VALIDATE_NOTHING + : fil_space_t::MAYBE_MISSING, + name, filepath); + if (space && (!upgrade || space->get_size())) { } else if (!not_dropped) { } else if (srv_operation == SRV_OPERATION_NORMAL && srv_start_after_restore @@ -980,11 +994,11 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) static_cast(len), field); } - max_space_id = ut_max(max_space_id, space_id); - ut_free(filepath); + goto next; } + fil_system.have_all_spaces = upgrade; done: mtr.commit(); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 47bfde7ebd23d..7973837028ebd 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -2035,6 +2035,7 @@ fil_ibd_create( fil_node_t* node = space->add(path, OS_FILE_CLOSED, size, false, true); space->set_stopped(); mysql_mutex_unlock(&fil_system.mutex); + lsn_t create_lsn; buf_block_t *header[2]; { @@ -2053,9 +2054,10 @@ fil_ibd_create( header[0]->page.lock.x_lock(); header[1]->page.lock.x_lock(); mtr.commit(); + create_lsn = mtr.commit_lsn(); /* Durably write the FILE_CREATE record (as well as records for the fsp_header_init() above before creating the file. */ - log_write_up_to(mtr.commit_lsn(), true); + log_write_up_to(create_lsn, true); } DEBUG_SYNC_C("fil_ibd_create_logged"); @@ -2136,6 +2138,7 @@ fil_ibd_create( mysql_mutex_lock(&fil_system.mutex); space->clear_stopped(); + space->create_lsn = create_lsn; node->handle = file; node->find_metadata(IF_WIN(,true)); if (++fil_system.n_open >= srv_max_n_open_files) { diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc new file mode 100644 index 0000000000000..3cceaaff2d466 --- /dev/null +++ b/storage/innobase/handler/backup_innodb.cc @@ -0,0 +1,1585 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "sql_class.h" +#include "backup_innodb.h" +#include "trx0trx.h" +#include "buf0flu.h" +#include "log0crypt.h" +#include "dict0load.h" +#include +#ifdef __linux__ +# include +# include +#endif + +/** Associate a transaction with the current session +@param thd session +@return InnoDB transaction */ +trx_t *check_trx_exists(THD *thd) noexcept; + +/** Try to write-fix a block. +@param s expected state() +@return new s (right before potentially setting the write-fix); +a write-fix was acquired if !is_freed(s) && !is_io_fixed(s) holds */ +inline uint32_t buf_page_t::write_fix_try(uint32_t s) noexcept +{ + /* The calling thread must hold a fix() */ + ut_ad(s > FREED); + /* + set_freed(), set_reinit() or flush() may run concurrently. + compare_exchange_strong() ensures that the write-fix was set by us. + */ + while (!is_freed(s) && !is_io_fixed(s) && + !zip.fix.compare_exchange_strong(s, s + (WRITE_FIX - UNFIXED), + std::memory_order_acquire, + std::memory_order_relaxed)); + return s; +} + +/** Try to undo a successful write_fix_try(). */ +inline void buf_page_t::write_unfix_try() noexcept +{ + uint32_t s{state()}; + /* set_freed() or set_reinit() may clear the write-fix before or + during this loop. We must use a compare-and-exchange loop. */ + while (is_write_fixed(s) && + !zip.fix.compare_exchange_weak(s, s - (WRITE_FIX - UNFIXED), + std::memory_order_relaxed, + std::memory_order_relaxed)); +} + +/** + Ensure that there are no page writes in progress. + @param end array of fil_space_t::BACKUP_BATCH_SIZE block descriptors + @param space_id tablespace identifier + @param end_page last page number that is being copied + @return pointer to the new end of the array, of write-fixed blocks +*/ +static buf_page_t **innodb_backup_batch_wait(buf_page_t **end, + uint32_t space_id, + uint32_t end_page) noexcept +{ + const page_id_t start + {space_id, end_page & ~(fil_space_t::BACKUP_BATCH_SIZE - 1)}; + ut_ad(end_page - 1 > start.page_no()); + for (page_id_t id{space_id, end_page}; id != start; --id) + { + auto &chain= buf_pool.page_hash.cell_get(id.fold()); + page_hash_latch &hash_lock{buf_pool.page_hash.lock_get(chain)}; + hash_lock.lock_shared(); + buf_page_t *const b= *end= buf_pool.page_hash.get(id, chain); + if (b && b->oldest_modification_acquire() > 2) + { + uint32_t state{b->fix()}; + /* The above buffer-fix froze b in buf_pool.page_hash */ + hash_lock.unlock_shared(); + /* + We found out that this page was dirty, and eventually such pages + must be either freed or written back to the file system. + + The lock-free buf_page_t::write_fix_try() aims to block + concurrent asynchronous buf_page_t::flush(). It will not + block any access to the page in the buffer pool; we are + not holding any page latch. + */ + state= b->write_fix_try(state + 1); + if (UNIV_LIKELY(!b->is_io_fixed(state))) + { + if (UNIV_LIKELY(!b->is_freed(state))) + { + /* Schedule a call of b->write_unfix_try() and b->unfix(). */ + end++; + continue; + } + /* + Freed blocks will not be written back to the file system. + As noted in fil_space_t::flush_freed(), we do allow + concurrent FALLOC_FL_PUNCH_HOLE of PAGE_COMPRESSED pages + and NUL writes by immediate_scrub_data_uncompressed=ON + while the data is being backed up. This should be fine, + because those operations are only overwriting freed + (garbage) data. + */ + } + else if (b->is_write_fixed(state)) + { + /* Wait for buf_page_t::write_complete() */ + b->lock.u_lock(); + ut_ad(!b->is_io_fixed()); + b->lock.u_unlock(); + } + + /* + Any subsequent write to this page will be held up by + fil_space_t::backup_page_end() until + fil_space_t::backup_stop() is invoked by + InnoDB_backup::backup_batch_stop(). + */ + b->unfix(); + } + else + hash_lock.unlock_shared(); + } + return end; +} + +namespace +{ +/** Backup state; protected by log_sys.latch */ +class InnoDB_backup +{ +public: + InnoDB_backup() { mutex.init(); } + ~InnoDB_backup() { mutex.destroy(); } + +private: + /** Backup context */ + struct context + { + /** Start LSN of the first backed up log file */ + const lsn_t first_lsn; + /** Start LSN of the last log file, or LSN_MAX if not determined yet */ + lsn_t max_first_lsn; + /** Final LSN of the backup, or LSN_MAX if not determined yet */ + lsn_t last_lsn; + /** size of the first log file */ + const uint64_t first_size; + /** Checkpoint at the start of the backup */ + const lsn_t checkpoint; + /** Log record pointing to the checkpoint */ + const lsn_t checkpoint_end_lsn; + /** the original state of innodb_log_archive before/after backup */ + const bool archived; + /** whether end() was invoked */ + bool cleaned_up; + /** the start LSN of the last hard-linked file, or 0 */ + std::atomic last_hardlink; + + /** + Note that a log file was hard-linked. + @param lsn start LSN of a hard-linked file + */ + void note_hardlink(lsn_t lsn) noexcept + { + for (lsn_t last= last_hardlink.load(std::memory_order_relaxed); + last < lsn && !last_hardlink. + compare_exchange_weak(last, lsn, + std::memory_order_relaxed, + std::memory_order_relaxed); ) {} + } + + /** Ensure that the last, hard-linked log file is not shared with + the server data directory, by copying it until the final LSN + @param target backup target directory + @param hl last_hardlink + @return error code + @retval 0 on success + */ + ATTRIBUTE_COLD int de_hardlink(const backup_target &target, lsn_t hl) + noexcept + { +#ifdef _WIN32 + std::string src{target.path}; + src.push_back('/'); + std::string dst{src}; + src.append("ib_logfile101"); + log_sys.append_archive_name(dst, hl); + const char *const s_{src.c_str()}, *const d_{dst.c_str()}; + if (!MoveFileEx(d_, s_, 0)) + { + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), d_, s_, errno); + return 1; + } + HANDLE s, d; + for (;;) + { + s= CreateFile(s_, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, + nullptr); + if (s != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), s_, errno); + return 1; + } + d= CreateFile(d_, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (d == INVALID_HANDLE_VALUE) + { + error_return: + my_osmaperr(GetLastError()); + std::ignore= CloseHandle(s); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s_, d_, errno); + return 1; + } +#else + std::string dst; + log_sys.append_archive_name(dst, hl); + const char *const d_{dst.c_str()}; + int d{-1}; + int err= ER_FILE_NOT_FOUND; + int s= openat(target.fd, d_, O_RDONLY); + if (s == -1) + { + error_return: + my_error(err, MYF(ME_ERROR_LOG), d_, errno); + if (s != -1) + std::ignore= close(s); + return 1; + } + err= ER_CANT_DELETE_FILE; + if (unlinkat(target.fd, d_, 0)) + goto error_return; + err= ER_CANT_CREATE_FILE; + d= openat(target.fd, d_, O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (d < 0) + goto error_return; +#endif + const uint64_t end{log_sys.START_OFFSET + last_lsn - hl}; + /* First, extend the file to a valid size. */ +#ifdef _WIN32 + int f; + { + LARGE_INTEGER li; + li.QuadPart= std::max(log_sys.FILE_SIZE_MIN, + (end + 4095) & ~4095ULL); + f= !SetFilePointerEx(d, li, nullptr, FILE_BEGIN) || !SetEndOfFile(d); + } +#else + int f= + ftruncate(d, std::max(log_sys.FILE_SIZE_MIN, + (end + 4095) & ~4095LL)); +#endif + if (!f) + { + const uint64_t begin= log_sys.START_OFFSET + + (hl == first_lsn) * (checkpoint - hl); +#ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(s, begin, end - begin, + POSIX_FADV_SEQUENTIAL); +#endif +#ifdef copy_file_shortcut + if (1 == (f= copy_file_shortcut(s, d, begin, end))) +#endif + f= backup::copy(s, d, begin, end); +#ifdef POSIX_FADV_DONTNEED + std::ignore= posix_fadvise(s, 0, 0, POSIX_FADV_DONTNEED); +#endif + if (!f && hl == first_lsn) + { + uint64_t cp_buf[8]{}; + write_checkpoint_buf(cp_buf, + checkpoint_end_lsn - hl + log_sys.START_OFFSET); + f= write_checkpoint(d, cp_buf); + } + } + if (IF_WIN(!CloseHandle(d), close(d)) | f) + goto error_return; + std::ignore= IF_WIN(CloseHandle(s), close(s)); +#ifdef _WIN32 + if (!DeleteFile(s_)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_DELETE_FILE, MYF(ME_ERROR_LOG), s_, errno); + return 1; + } +#endif + return 0; + } + + /** + Finish a backup. + @param target backup target + @param sink backup worker context + @return error code + @retval 0 on success + */ + int cleanup(const backup_target &target, const backup_sink &sink) noexcept + { + const lsn_t hl{last_hardlink.load(std::memory_order_relaxed)}; + if (hl == LSN_MAX) + return 0; + log_sys.latch.rd_lock(); + const lsn_t current_first_lsn{log_sys.get_first_lsn()}; + log_sys.latch.rd_unlock(); + if (hl == current_first_lsn) + { + ut_ad(sink.stream == sink.NO_STREAM); + if (int fail= de_hardlink(target, hl)) + return fail; + } + return write_config(target, sink); + } + }; + + /** pointer to backup context, or nullptr if no backup is active */ + context *ctx; + + /** the original innodb_log_file_size, or 0 */ + uint64_t old_size; + + /** mutex protecting queue, non_log */ + srw_mutex mutex; + /** collection of files and sizes, followed by any log files to be copied */ + std::vector queue; + /** number of non-log files at the start of the queue */ + size_t non_log; + +public: + /** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @return ctx + @retval -1 on failure + */ + void *init(THD *thd) noexcept + { + log_sys.latch.wr_lock(); + ut_ad(!ctx); + mutex.wr_lock(); + ut_ad(!non_log); + if (!queue.empty()) + /* A new BACKUP SERVER is being invoked before a previous one + had been fully finalized. Clean up any log files. */ + delete_logs(); + mutex.wr_unlock(); + + if (log_sys.backup_start(&old_size, thd)) + { + log_sys.latch.wr_unlock(); + fail: + my_error(ER_OUT_OF_RESOURCES, MYF(ME_ERROR_LOG)); + return reinterpret_cast(-1); + } + + mutex.wr_lock(); + + try + { + lsn_t start_end; + const lsn_t start= +#if 1 /* TODO: for incremental backup, allow the start to be specified */ + log_sys.get_latest_checkpoint(start_end); +#else + log_sys.archived_checkpoint; + start_end= log_sys.archived_lsn; +#endif + ut_ad(start_end >= start); + ut_ad(start >= log_sys.get_first_lsn()); + + ctx= new context{ + log_sys.get_first_lsn(), LSN_MAX, LSN_MAX, log_sys.file_size, + start, start_end, !old_size, false, 0 + }; + + /* Collect all tablespaces that have been created before our + start checkpoint. Newer tablespaces will be recovered by the + innodb_log_archive=ON recovery. + + If a tablespace is deleted before step() is invoked, the file + will not be copied, and a FILE_DELETE record in the log will + ensure correct recovery. + + If a tablespace is renamed between this and end(), the recovery + of a FILE_RENAME record will ensure the correct file name, + no matter which name was used by step(). */ + mysql_mutex_lock(&fil_system.mutex); + for (fil_space_t &space : fil_system.space_list) + if (space.id < SRV_SPACE_ID_UPPER_BOUND && + !space.is_being_imported() && !space.is_stopping() && + space.create_lsn <= start) try + { + /* FIXME: how to initialize create_lsn for old files, to + have efficient incremental backup? + fil_node_t::read_page0() cannot assign it from + FIL_PAGE_LSN because that would not reflect the file + creation but for example allocating or freeing a page. + Perhaps we can read it from page 1 (change buffer bitmap)? + + The easy parts of initializing space->create_lsn are + as follows: + (1) In log_parse_file() when processing FILE_CREATE + (2) In deferred_spaces.create() + (3) In fil_ibd_create() outside recovery */ + queue.emplace_back + (uint64_t{space.id} | + uint64_t{std::min(space.size, space.free_limit)} << 32); + } catch(...) { mysql_mutex_unlock(&fil_system.mutex); throw; } + mysql_mutex_unlock(&fil_system.mutex); + non_log= queue.size(); + } + catch (std::bad_alloc&) { + queue.clear(); + delete ctx; + ctx= nullptr; + log_sys.backup_stop(old_size, thd); + goto fail; + } + + mutex.wr_unlock(); + log_sys.latch.wr_unlock(); + DEBUG_SYNC(thd, "innodb_backup_start"); + return ctx; + } + + /** + Process a file that was collected at init(). + This may be invoked from multiple concurrent threads. + @param target backup target + @param phase backup phase + @param sink backup worker context + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int step(const backup_target &target, backup_phase phase, + const backup_sink &sink) noexcept + { + uint64_t id_limit{0}; + mutex.wr_lock(); + ut_ad(sink.ha_data); + ut_ad(ctx ? ctx == sink.ha_data + : phase == BACKUP_PHASE_FINISH || phase == BACKUP_PHASE_NO_COMMIT); + ut_ad(static_cast(sink.ha_data)->last_lsn == LSN_MAX + ? phase == BACKUP_PHASE_START : !ctx); + const size_t size{queue.size()}, non_log_files{non_log}; + ut_ad(size >= non_log_files); + + if (UNIV_UNLIKELY(!size)) + { + mutex.wr_unlock(); + return 0; + } + + non_log-= size == non_log_files; + id_limit= queue.back(); + queue.pop_back(); + mutex.wr_unlock(); + + if (size > non_log_files) + { + log_sys.latch.rd_lock(); + const lsn_t first{log_sys.get_first_lsn()}; + log_sys.latch.rd_unlock(); + if (UNIV_UNLIKELY(id_limit > first)) + /* Wait for checkpoint_complete(). */ + buf_flush_sync_batch(id_limit, true); + if (replicate(id_limit, target, sink, id_limit < first)) + return -1; + } + else if (fil_space_t *space= fil_space_t::get(uint32_t(id_limit))) + { + ut_ad(phase == BACKUP_PHASE_START); + int res= -1; + uint32_t start{0}, limit{uint32_t(id_limit >> 32)}; +#ifdef _WIN32 + if (sink.stream == sink.NO_STREAM) + { + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain);;) + { + if ((res= backup(target.path, node, start, limit))) + break; + fil_node_t *next= UT_LIST_GET_NEXT(chain, node); + if (!next) + break; + const uint32_t size{node->size}; + start+= size; + if (limit >= size) + limit-= size; + else + limit= 0; + node= next; + } + } + else + { + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain);;) + { + if ((res= stream(sink.stream, node, start, limit))) + break; + fil_node_t *next= UT_LIST_GET_NEXT(chain, node); + if (!next) + break; + const uint32_t size{node->size}; + start+= size; + if (limit >= size) + limit-= size; + else + limit= 0; + node= next; + } + } +#else + int fd; + int (*method)(int, fil_node_t *, uint32_t, uint32_t); + if (sink.stream == sink.NO_STREAM) + { + fd= target.fd; + method= backup; + } + else + { + fd= sink.stream; + method= stream; + } + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain);;) + { +# ifdef HAVE_POSIX_FALLOCATE + if (limit & 3 && !UT_LIST_GET_NEXT(chain, node)) + { + const uint32_t page_size{space->physical_size()}; + if ((limit * page_size) & 4095) + /* os_file_set_size() extends ROW_FORMAT=COMPRESSED files to + multiples of 4096 bytes. There may be up to 3 pages + (of 1024 bytes) that have not been written out yet. + We must cap the limit to the actual file size. */ + limit= + std::min(limit, + uint32_t(os_file_get_size(node->handle) / page_size)); + } +# endif + res= (*method)(fd, node, start, limit); +#ifdef POSIX_FADV_DONTNEED + std::ignore= posix_fadvise(node->handle, 0, 0, POSIX_FADV_DONTNEED); +#endif + if (res) + break; + fil_node_t *next= UT_LIST_GET_NEXT(chain, node); + if (!next) + break; + const uint32_t size{node->size}; + start+= size; + if (limit >= size) + limit-= size; + else + limit= 0; + node= next; + } +#endif + space->release(); + if (res) + return res; + } + + return int(std::min(size_t{std::numeric_limits::max()}, size - 1)); + } + + /** + Determine the logical time of the backup snapshot. + */ + void commit() noexcept + { + log_sys.latch.wr_lock(); + ut_ad(ctx); + ut_ad(ctx->last_lsn == LSN_MAX); + const lsn_t last_lsn{log_sys.get_lsn()}; + lsn_t lsn{log_sys.get_first_lsn()}; + mutex.wr_lock(); + ut_ad(!non_log); + if (queue.empty() || queue.back() != lsn) + { + /* Schedule the remaining log for copying */ + queue.emplace_back(lsn); + const lsn_t next_lsn{lsn + log_sys.capacity()}; + if (next_lsn < last_lsn) + queue.emplace_back(lsn= next_lsn); + } + mutex.wr_unlock(); + ctx->max_first_lsn= lsn; + ctx->last_lsn= last_lsn; + ctx= nullptr; /* unsubscribe to checkpoint_complete() */ + log_sys.latch.wr_unlock(); + } + + /** + Finish copying or finalize the backup. + @param thd current session + @param phase backup phase + @param sink backup worker context + @return error code + @retval 0 on success + */ + int end(THD *thd, backup_phase phase, const backup_sink &sink) noexcept + { + context *const ctx{static_cast(sink.ha_data)}; + if (!ctx /* InnoDB_backup::init() must have failed */ || + ctx->cleaned_up /* aborting after phase=BACKUP_PHASE_NO_COMMIT */) + return 0; + ctx->cleaned_up= true; + if (phase == BACKUP_PHASE_ABORT) + ctx->last_hardlink.store(LSN_MAX, std::memory_order_relaxed); + log_sys.latch.wr_lock(); + ut_ad(!this->ctx || this->ctx == ctx); + this->ctx= nullptr; /* fini() will delete the object */ + ut_ad(!log_sys.resize_in_progress()); + ut_ad(log_sys.archive); + int fail{0}; + if (!old_size) + { + mutex.wr_lock(); + queue.clear(); + non_log= 0; + mutex.wr_unlock(); + } + else + { + log_sys.latch.wr_unlock(); + sql_print_information("stop archiving: " LSN_PF, + log_sys.last_checkpoint_lsn.load()); + /* FIXME: execute this at a later stage, + after MDL_BACKUP_WAIT_COMMIT has been released! + This may wait several seconds for some page flushing! */ + fail= log_sys.backup_stop_archiving(thd); + sql_print_information("stopped archiving: " LSN_PF, + log_sys.last_checkpoint_lsn.load()); + log_sys.latch.wr_lock(); + mutex.wr_lock(); + delete_logs(); + mutex.wr_unlock(); + } + + log_sys.backup_stop(old_size, thd); + return fail; + } + + /** + Clean up after end(). + @param target backup target + @param sink backup worker context + @return error code + @retval 0 on success + */ + int fini(const backup_target &target, const backup_sink &sink) noexcept + { + if (context *ctx{static_cast(sink.ha_data)}) + { + ut_ad(ctx != this->ctx); + int fail{ctx->cleanup(target, sink)}; + delete ctx; + return fail; + } + return 0; + } + + /** + Complete the first checkpoint in a new archive log file. + */ + void checkpoint_complete() noexcept + { + ut_ad(log_sys.latch_have_wr()); + if (ctx) + { + const lsn_t lsn{log_sys.get_first_lsn() - log_sys.capacity()}; + mutex.wr_lock(); + queue.emplace_back(lsn); + mutex.wr_unlock(); + } + } + +private: + /** + Safely start backing up a tablespace file. + @param end array of fil_space_t::BACKUP_BATCH_SIZE block descriptors + @param space tablespace that is being backed up + @param end_page first page not to copy + @return pointer to the new end of the array, of write-fixed blocks + */ + static buf_page_t **backup_batch_start(buf_page_t **end, + fil_space_t *space, uint32_t end_page) + noexcept + { + ut_ad(end_page); + space->backup_start(end_page); + /* Block any writes that might be posted after checking + fil_space_t::backup_page_end(). */ + return innodb_backup_batch_wait(end, space->id, end_page - 1); + } + /** + Stop backing up a tablespace. + @param space tablespace + @param begin first write-fixed block descriptor + @param end end of write-fixed block descriptors + */ + static void backup_batch_stop(fil_space_t *space, + buf_page_t **begin, buf_page_t **end) noexcept + { + space->backup_stop(); + while (begin != end) + { + buf_page_t *b= *begin++; + b->write_unfix_try(); + b->unfix(); + } + } + + /** + Delete unnecessary logs that had been created for backup. + */ + void delete_logs() noexcept + { + ut_ad(log_sys.latch_have_wr()); + ut_ad(old_size); + ut_ad(non_log <= queue.size()); + + const lsn_t first_lsn{log_sys.get_first_lsn()}; + size_t i{non_log}; + non_log= 0; + while (i < queue.size()) + { + const lsn_t lsn{queue[i++]}; + if (lsn != first_lsn) + IF_WIN(DeleteFile,unlink)(log_sys.get_archive_path(lsn).c_str()); + } + queue.clear(); + } + +#ifdef copy_file_shortcut + /** + Try copy_file_shortcut() to back up a persistent InnoDB data file. + @param dst target file handle + @param node InnoDB data file + @param start the page number at the start of the file + @param limit the size of the file before the doublewrite buffer + @param page_size node->space->physical_size() + @param final_limit the size of the file at init(), or 0 if no dblwr + @param blocks descriptor array of fil_space_t::BACKUP_BATCH_SIZE + @return error code (non-positive) + @retval 0 on success + */ + static int copy_file_shortcut_try(int dst, fil_node_t *node, + uint32_t start, uint32_t limit, + uint32_t page_size, uint32_t final_limit, + buf_page_t **blocks) + noexcept + { +# if 0 + return 1; // work around https://github.com/rr-debugger/rr/issues/4059 +# endif + for (uint32_t page{0};;) + { + while (page < limit) + { + start+= fil_space_t::BACKUP_BATCH_SIZE; + buf_page_t **end= backup_batch_start(blocks, node->space, start); + const uint64_t o{uint64_t{page} * page_size}; + page= std::min(limit, page + fil_space_t::BACKUP_BATCH_SIZE); + /* TODO: avoid copying freed page ranges, or pages that were + allocated after the backup started */ + int err{copy_file_shortcut(node->handle, dst, + o, uint64_t{page} * page_size)}; + backup_batch_stop(node->space, blocks, end); + if (err) + return err; + } + + if (final_limit != 0 && page == buf_dblwr.begin()) + { + /* Copy the rest after the doublewrite buffer. */ + limit= final_limit; + page+= buf_dblwr.size(); + } + else + return 0; + } + } +#endif + + /** + Back up a persistent InnoDB data file. + @param target backup target directory + @param node InnoDB data file + @param start the page number at the start of the file + @param limit the size of the file at init() + @return error code (non-positive) + @retval 0 on success + */ + static int backup(IF_WIN(const char *,int) target, fil_node_t *node, + uint32_t start, uint32_t limit) noexcept + { + for (bool tried_mkdir{false};;) + { +#ifdef _WIN32 + std::string path{target}; + path.push_back('/'); + path.append(node->name); + HANDLE f= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (f == INVALID_HANDLE_VALUE) + { + unsigned long err= GetLastError(); + if (err == ERROR_PATH_NOT_FOUND && !tried_mkdir && + node->space->id && !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + path.erase(path.rfind('/')); + if (CreateDirectory(path.c_str(), + my_dir_security_attributes.lpSecurityDescriptor + ? &my_dir_security_attributes : nullptr) || + (err= GetLastError()) == ERROR_ALREADY_EXISTS) + continue; + } + + my_osmaperr(err); + goto fail; + } +#else + int f; +# ifdef __APPLE__ + /* aio::synchronous() in another thread may concurrently invoke + pwrite(2) on node->handle. We assume that both pwrite(2) and + fclonefileat(2) are atomic with respect to each other. Should + this assumption be invalid, some data files in the backup may be + corrupted. This corruption can be fixed by either removing this + special handling, or by implementing file-level locking. */ + f= fclonefileat(node->handle, target, node->name, 0); + if (!f) + break; + switch (errno) { + case ENOENT: + goto try_mkdir; + case ENOTSUP: + break; + default: + goto fail; + } +# endif + f= openat(target, node->name, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (f < 0) + { + if (errno == ENOENT) + { +# ifdef __APPLE__ + try_mkdir: +# endif + if (!tried_mkdir && node->space->id && + !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + const char *sep= strchr(node->name, '/'); + ut_ad(sep); + sep= strchr(sep + 1, '/'); + ut_ad(sep); + std::string dir{node->name, size_t(sep - node->name)}; + if (!mkdirat(target, dir.c_str(), 0777) || errno == EEXIST) + continue; + } + } + goto fail; + } +#endif + const uint32_t page_size{node->space->physical_size()}; + int err{0}; + if (node->size < limit) + limit= node->size; +#ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(node->handle, 0, off_t(limit) * page_size, + POSIX_FADV_SEQUENTIAL); +#endif + /* + For the system tablespace, a minimum size has been configured + which may be larger than the currently used size. Preserve the + original size. + + For other persistent data files, fil_node_t::read_page0() + expects at least 4 * innodb_page_size bytes. Small + ROW_FORMAT=COMPRESSED files may be zero-filled to this size. + */ + const uint64_t min_size= + std::max(uint64_t{FIL_IBD_FILE_INITIAL_SIZE} << srv_page_size_shift, + uint64_t{node->size} * page_size); + if (uint64_t{limit} * page_size < min_size) + { + /* Expand the target file to the minimum size. */ +#ifdef _WIN32 + LARGE_INTEGER li; + li.QuadPart= min_size; + err= !SetFilePointerEx(f, li, nullptr, FILE_BEGIN) || !SetEndOfFile(f); +#else + err= ftruncate(f, min_size); +#endif + if (err) + limit= 0; + } + + const uint32_t final_limit= + node == fil_system.sys_space->chain.start && + buf_dblwr.begin() + buf_dblwr.size() == buf_dblwr.end() && + limit > buf_dblwr.end() + ? limit : 0; + if (final_limit) + limit= buf_dblwr.begin(); + + buf_page_t *blocks[fil_space_t::BACKUP_BATCH_SIZE]; +#ifdef copy_file_shortcut + err= copy_file_shortcut_try(f, node, start, limit, + page_size, final_limit, blocks); + if (err == 1) +#endif + { +#ifdef copy_file_mmap + const size_t c{size_t{final_limit ? final_limit : limit} * page_size}; + void *p= mmap(nullptr, c, PROT_READ, MAP_SHARED, node->handle, 0); + if (p != MAP_FAILED) + { + for (uint32_t page{0};;) + { + while (page < limit) + { + start+= fil_space_t::BACKUP_BATCH_SIZE; + buf_page_t **end= backup_batch_start(blocks, node->space, start); + const uint64_t o{uint64_t{page} * page_size}; + page= std::min(limit, page + fil_space_t::BACKUP_BATCH_SIZE); + /* TODO: avoid copying freed page ranges, or pages that were + allocated after the backup started */ + err= copy_file_mmap(p, f, o, uint64_t{page} * page_size); + backup_batch_stop(node->space, blocks, end); + if (err) + break; + } + + if (final_limit != 0 && !err && page == buf_dblwr.begin()) + { + /* Copy the rest after the doublewrite buffer. */ + limit= final_limit; + page+= buf_dblwr.size(); + } + else + break; + } + munmap(p, c); + } + else +#endif + for (uint32_t page{0};;) + { + while (page < limit) + { + start+= fil_space_t::BACKUP_BATCH_SIZE; + buf_page_t **end= backup_batch_start(blocks, node->space, start); + const uint64_t o{uint64_t{page} * page_size}; + page= std::min(limit, page + fil_space_t::BACKUP_BATCH_SIZE); + /* TODO: avoid copying freed page ranges, or pages that were + allocated after the backup started */ + err= + backup::copy(node->handle, f, o, uint64_t{page} * page_size); + backup_batch_stop(node->space, blocks, end); + if (err) + break; + } + + if (final_limit != 0 && !err && page == buf_dblwr.begin()) + { + /* Copy the rest after the doublewrite buffer. */ + limit= final_limit; + page+= buf_dblwr.size(); + } + else + break; + } + } + + if (IF_WIN(!CloseHandle(f), close(f)) | err) + goto fail; + break; + } + return 0; + fail: + my_error(ER_CANT_CREATE_FILE, MYF(0), node->name, errno); + return -1; + } + + /** + Stream a persistent InnoDB data file. + @param stream backup target stream + @param node InnoDB data file + @param start the page number at the start of the file + @param limit the size of the file at init() + @return error code (non-positive) + @retval 0 on success + */ + static int stream(backup_fd stream, fil_node_t *node, + uint32_t start, uint32_t limit) noexcept + { + const uint32_t page_size{node->space->physical_size()}, + file_size= std::max(std::max(limit, node->size), + (FIL_IBD_FILE_INITIAL_SIZE << srv_page_size_shift) / + page_size); + uint64_t physical_size{uint64_t{limit} * page_size}; + backup_chunk chunk[3]{ + {0, physical_size}, + {uint64_t{file_size} * page_size, 0} + }; + size_t n_chunk= (file_size > limit) * 2; + + if (file_size < limit) + { + limit= file_size; + chunk[0].length= chunk[1].offset; + } + +#ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(node->handle, 0, chunk[0].length, + POSIX_FADV_SEQUENTIAL); +#endif + + if (node == fil_system.sys_space->chain.start && + buf_dblwr.begin() + buf_dblwr.size() == buf_dblwr.end() && + limit > buf_dblwr.end()) + { + n_chunk= 2 + !!n_chunk; + limit= buf_dblwr.begin(); + physical_size-= uint64_t{buf_dblwr.size()} * page_size; + memmove(chunk + 1, chunk, 2 * sizeof *chunk); + chunk[0].length= uint64_t{limit} * page_size; + chunk[1].offset= uint64_t{buf_dblwr.end()} * page_size; + chunk[1].length-= chunk[1].offset; + } + + int err= backup_stream_start(stream, node->name, 0644, + physical_size, chunk, n_chunk); + if (err) + limit= 0; + + uint32_t page{0}; + + loop: + while (page < limit) + { + buf_page_t *blocks[fil_space_t::BACKUP_BATCH_SIZE], **end= blocks; + { + const uint32_t end_page{start + fil_space_t::BACKUP_BATCH_SIZE}; + end= backup_batch_start(end, node->space, end_page); + start= end_page; + } + uint32_t last{std::min(limit, page + fil_space_t::BACKUP_BATCH_SIZE)}; + /* TODO: avoid copying freed page ranges, or pages that were + allocated after the backup started */ + err= backup::append(node->handle, stream, + uint64_t{page} * page_size, + uint64_t{last} * page_size); + page= last; + backup_batch_stop(node->space, blocks, end); + if (err) + goto fail; + } + + if (limit == buf_dblwr.begin() && n_chunk == 3) + { + /* Copy the rest after the doublewrite buffer. */ + page+= buf_dblwr.size(); + limit= page + uint32_t(chunk[1].length >> srv_page_size_shift); + goto loop; + } + + if (err) + fail: + my_error(ER_IO_WRITE_ERROR, MYF(0), errno, strerror(errno), + "BACKUP SERVER"); + return err; + } + +private: + /** + Initialize a checkpoint header buffer pointing to the start of the backup. + @param buf checkpoint buffer + @param c offset of the FILE_CHECKPOINT mini-transaction + */ + static void write_checkpoint_buf(uint64_t *buf, uint64_t c) noexcept + { + ut_ad(c >= log_sys.START_OFFSET); + if (log_sys.is_encrypted()) + log_crypt_write_header(reinterpret_cast(buf), true); + buf[4 * log_sys.is_encrypted()]= my_htobe64(c); + } + + /** Write a checkpoint header pointing to the start of the backup. + @param dst target file + @param buf checkpoint header + @return error code + @retval 0 on success */ + static int write_checkpoint(backup_fd dst, const void *buf) noexcept + { +#ifdef _WIN32 + using tpool::pwrite; +#endif + for (ssize_t o= 0, count= 64; count;) + { + ssize_t ret= + pwrite(dst, static_cast(buf) + o, count, o); + if (ret <= 0 || ret > count) + return -1; + o+= ret; + count-= ret; + } + return 0; + } + +public: + /** Maximum length of the configuration string */ + static constexpr size_t CONFIG_SIZE= + sizeof "[server]\n# checkpoint=" + + sizeof "innodb_log_recovery_start=" + + sizeof "innodb_log_recovery_target=\n" + 45 * 3; + + /** Write the configuration parameters for restoring the backup + @param config buffer for configuration string + @param ctx backup context + @return size of the configuration string */ + static size_t write_config_buf(char *config, const context &ctx) + noexcept + { + ut_ad(ctx.last_lsn != LSN_MAX); + return size_t(snprintf(config, CONFIG_SIZE, + "[server]\n# checkpoint=" LSN_PF "\n" + "innodb_log_recovery_start=" LSN_PF "\n" + "innodb_log_recovery_target=" LSN_PF "\n", + ctx.checkpoint, ctx.checkpoint_end_lsn, + ctx.last_lsn)); + } + + /** Write the configuration parameters for restoring the backup + @param target backup target + @param sink backup worker context + @param ctx backup context + @return error code (non-positive) + @retval 0 on success */ + static int write_config(const backup_target &target, + const backup_sink &sink) noexcept + { + char config[CONFIG_SIZE]; + const size_t size + {write_config_buf(config, *static_cast(sink.ha_data))}; + return sink.stream == sink.NO_STREAM + ? backup_config_append(IF_WIN(target.path, target.fd), config, size) + : backup_stream_config(sink.stream, config, size); + } + + /** + Hard-link (copy) or rename (move) or stream an archive log file. + @param lsn The first LSN in the file + @param target backup target + @param sink backup context + @param old lsn < log_sys.get_first_lsn() + @return error code + @retval 0 on success + */ + static int replicate(lsn_t lsn, + const backup_target &target, + const backup_sink &sink, bool old) noexcept + { + ut_ad(log_get_lsn() >= lsn); + const std::string p{log_sys.get_archive_path(lsn)}; + const char *const path= p.c_str(), *basename= strrchr(path, '/'); + if (!basename) + basename= path; + else + basename++; + context &ctx{*static_cast(sink.ha_data)}; + const bool move{old && !ctx.archived}; + uint64_t cp_buf[8]{}; +#ifdef _WIN32 + ut_ad(!target.path == (sink.stream != sink.NO_STREAM)); + std::string b; + const char *destname= nullptr; + if (!target.path) + goto send_file; + b= target.path; + b.push_back('/'); + b.append(basename); + destname= b.c_str(); + unsigned long err; + if (move) + { + if (!MoveFileEx(path, destname, MOVEFILE_COPY_ALLOWED)) + { + fail: + err= GetLastError(); + got_err: + my_osmaperr(err); + if (target.path) + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, + errno); + else + my_error(ER_IO_WRITE_ERROR, MYF(ME_ERROR_LOG), + errno, strerror(errno), "BACKUP SERVER"); + return -1; + } + + if (lsn < ctx.checkpoint) + { + if (!SetFileAttributes(destname, FILE_ATTRIBUTE_NORMAL)) + goto fail; + HANDLE dh= CreateFile(destname, GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dh == INVALID_HANDLE_VALUE) + goto fail; + if (os_file_set_sparse_win32(dh)) + std::ignore= + os_file_punch_hole(dh, 0, log_sys.START_OFFSET + + ((ctx.checkpoint - lsn) & ~4095ULL)); + write_checkpoint_buf(cp_buf, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + int fail= write_checkpoint(dh, cp_buf); + std::ignore= CloseHandle(dh); + if (fail) + goto fail; + } + return 0; + } + else if (CreateHardLink(destname, path, nullptr)) + { + ctx.note_hardlink(lsn); + return 0; + } + + if ((err= GetLastError()) != ERROR_NOT_SAME_DEVICE) + goto got_err; + /* Hard-linking failed. Try copying with the final name. */ + if (target.path) + { + b= target.path; + b.push_back('/'); + b.append(basename); + destname= b.c_str(); + + if (lsn >= ctx.checkpoint && lsn < ctx.max_first_lsn) + { + /* Copy a middle log file entirely. */ + if (CopyFileEx(path, basename, nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) + return 0; + goto fail; + } + } + + send_file: + HANDLE src; + for (;;) + { + src= CreateFile(path, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, + nullptr); + if (src != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + goto fail; + } + HANDLE dst{sink.stream}; + if (dst == INVALID_HANDLE_VALUE) + { + dst= CreateFile(destname, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dst == INVALID_HANDLE_VALUE) + { + std::ignore= CloseHandle(src); + goto fail; + } + } +#else + if (sink.stream != sink.NO_STREAM); + else if (move + ? !renameat(AT_FDCWD, path, target.fd, basename) + : !linkat(AT_FDCWD, path, target.fd, basename, AT_SYMLINK_FOLLOW)) + { + if (!move) + ctx.note_hardlink(lsn); +# ifdef __linux__ + else if (lsn != ctx.first_lsn); + else if (off_t garbage= (ctx.checkpoint - lsn) & ~4095ULL) + /* Best effort to punch a hole to free up some garbage in + the first file. We do not care about failures. */ + if (!fchmodat(target.fd, basename, 0644, 0)) + { + int dst= openat(target.fd, basename, O_RDWR); + if (dst >= 0) + fallocate(dst, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + log_sys.START_OFFSET, garbage); + close(dst); + std::ignore= fchmodat(target.fd, basename, 0444, 0); + } +# endif + return 0; + } + else if (errno != EXDEV) + { + fail: + if (sink.stream == sink.NO_STREAM) + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, errno); + else + my_error(ER_IO_WRITE_ERROR, MYF(ME_ERROR_LOG), errno, strerror(errno), + "BACKUP SERVER"); + return -1; + } + + const int src{open(path, O_RDONLY)}; + if (src < 0) + goto fail; + if (move && unlink(path)) + { + close_and_fail: + std::ignore= close(src); + goto fail; + } + int dst{sink.stream}; + if (dst < 0) + { + dst= openat(target.fd, basename, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + goto close_and_fail; + } + int err; +#endif + backup_chunk chunks[3], *chunk{chunks}; + *chunk++= {log_sys.START_OFFSET, ctx.last_lsn - lsn}; + if (lsn < ctx.checkpoint) + { + /* Copy the necessary part of the first log file. */ + ut_ad(lsn == ctx.first_lsn); + write_checkpoint_buf(cp_buf, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + chunk[-1]= {0, 512}; + const lsn_t end= + std::min(ctx.last_lsn, lsn + ctx.first_size - log_sys.START_OFFSET); + *chunk++= + {log_sys.START_OFFSET + ctx.checkpoint - lsn, end - ctx.checkpoint}; + chunk->offset= end - lsn; + goto pad_size; + } + else if (lsn < ctx.max_first_lsn) + { + /* Copy a middle log file entirely. */ +#ifdef _WIN32 + ut_ad(dst == sink.stream); + chunk->offset= os_file_get_size(src); +#else + chunk->offset= uint64_t(lseek(src, 0, SEEK_END)); +# ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(src, 0, chunk->offset, POSIX_FADV_SEQUENTIAL); +# endif + if (dst != sink.stream) + { + err= copy_entire_file(src, dst); + goto close_dst; + } +#endif + /* Omit the checkpoint header from the stream. */ + chunk[-1].length= chunk->offset - log_sys.START_OFFSET; + goto stream_file; + } + else + { + ut_ad(ctx.max_first_lsn == lsn); + ut_ad(ctx.last_lsn > lsn); + ut_ad(ctx.last_lsn != LSN_MAX); + ut_ad(chunk[-1].length == ctx.last_lsn - lsn); + chunk->offset= chunk[-1].length; + pad_size: +#ifdef POSIX_FADV_SEQUENTIAL + std::ignore= posix_fadvise(src, chunk[-1].offset, chunk[-1].length, + POSIX_FADV_SEQUENTIAL); +#endif + /* Set the logical size of the file. */ + chunk->offset= + std::max(log_sys.FILE_SIZE_MIN, + (chunk->offset + (log_sys.START_OFFSET + 4095)) & + ~4095ULL); + } + + if (dst == sink.stream) + { + stream_file: + chunk++->length= 0; + const backup_chunk &end{chunk[-2]}; + ut_ad(chunk - chunks == 2 || chunk - chunks == 3); + const size_t cp_size{(size_t(chunk - chunks) & 1) << 9}; + err= backup_stream_start(dst, basename, + 0444 | int{lsn == ctx.max_first_lsn} << 7, + end.length + cp_size, chunks, chunk - chunks); + if (!err && cp_size) + err= backup_stream_write(dst, cp_buf, sizeof cp_buf) || + backup_stream_write(dst, field_ref_zero, cp_size - sizeof cp_buf); + if (!err) + { + err= backup_stream_append_async(src, dst, end.offset, + end.offset + end.length) || + backup_stream_zeropad(dst, size_t(end.length)); + } + } + else + { + /* First, extend the file to a valid size. */ +#ifdef _WIN32 + LARGE_INTEGER li; + li.QuadPart= chunk->offset; + err= !SetFilePointerEx(dst, li, nullptr, FILE_BEGIN) || + !SetEndOfFile(dst) || +#else + err= ftruncate(dst, chunk->offset) || +#endif + backup::copy(src, dst, chunk[-1].offset, chunk[-1].offset + + chunk[-1].length) || + (lsn < ctx.checkpoint && write_checkpoint(dst, cp_buf)); +#ifdef _WIN32 + err|= !CloseHandle(dst); +#else + close_dst: + err|= close(dst); +#endif + } + + if (err | IF_WIN(!CloseHandle(src), close(src))) + goto fail; + + return 0; + } +}; + +/** The backup context; protected by log_sys.latch */ +static InnoDB_backup innodb_backup; +} + +bool log_t::backup_start(uint64_t *old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + ut_ad(!backup); + ut_ad(end_lsn >= last_checkpoint_lsn); + backup= true; + *old_size= 0; + if (archive) + { + if (first_lsn > last_checkpoint_lsn) + { + /* Wait for recovery to be independent from the previous log. */ + mysql_mutex_lock(&buf_pool.flush_list_mutex); + buf_flush_wait(end_lsn, false); + ut_ad(first_lsn <= last_checkpoint_lsn); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + } + return false; + } + const uint64_t old_file_size{file_size}; + latch.wr_unlock(); + const bool fail{set_archive(true, thd, true)}; + latch.wr_lock(); + if (!fail) + { + *old_size= old_file_size; + return false; + } + ut_ad(backup); + backup= false; + const uint64_t new_file_size{file_size}; + latch.wr_unlock(); + if (old_file_size != new_file_size && old_file_size && + resize_start(old_file_size, thd) == RESIZE_STARTED) + resize_finish(thd); + latch.wr_lock(); + return true; +} + +void log_t::backup_stop(uint64_t old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + /* We will be invoked with old_size=0 after a failed backup_start(), + or if innodb_log_archive=ON held during a successful backup_start(). */ + ut_ad(!old_size || !resize_in_progress()); + ut_ad(!old_size || backup); + backup= false; + const uint64_t new_size{file_size}; + latch.wr_unlock(); + if (old_size && old_size != new_size && + resize_start(old_size, thd) == RESIZE_STARTED) + resize_finish(thd); +} + +void *innodb_backup_start(THD *thd, const backup_target *, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_PREPARE_START: + if (!fil_system.have_all_spaces) + { + /* To speed up startup, InnoDB does not normally open all + tablespace files that are pointed to by SYS_TABLES. + InnoDB_backup::init() assumes that the information of all + tablespaces is available, including files that had been created + before the server was started, and never opened in the course of + the current server execution. */ + dict_load_tablespaces(nullptr, true); + ut_ad(fil_system.have_all_spaces); + } + return 0; + case BACKUP_PHASE_START: + return innodb_backup.init(thd); + case BACKUP_PHASE_NO_COMMIT: + innodb_backup.commit(); + /* fall through */ + default: + return sink->ha_data; + } +} + +int innodb_backup_step(THD *, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_START: + case BACKUP_PHASE_NO_COMMIT: + case BACKUP_PHASE_FINISH: + return innodb_backup.step(*target, phase, *sink); + default: + return 0; + } +} + +int innodb_backup_end(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + default: + return 0; + case BACKUP_PHASE_FINISH: + return innodb_backup.fini(*target, *sink); + case BACKUP_PHASE_NO_COMMIT: + case BACKUP_PHASE_ABORT: + return innodb_backup.end(thd, phase, *sink); + } +} + +void innodb_backup_checkpoint() noexcept +{ + innodb_backup.checkpoint_complete(); +} diff --git a/storage/innobase/handler/backup_innodb.h b/storage/innobase/handler/backup_innodb.h new file mode 100644 index 0000000000000..e9393038e5a21 --- /dev/null +++ b/storage/innobase/handler/backup_innodb.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/** + Start of a BACKUP SERVER phase, + when no innodb_backup_step() or innodb_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to sink, or nullptr + @retval -1 on failure +*/ +void *innodb_backup_start(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) + noexcept; + +/** + Process a file that was collected in innodb_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number of files remaining, or negative on error + @retval 0 on completion +*/ +int innodb_backup_step(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current sesssion + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int innodb_backup_end(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Complete the first checkpoint in a new archive log file. +*/ +void innodb_backup_checkpoint() noexcept; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index d09250515aa27..e4f03b4e8521d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -152,6 +152,7 @@ MDL_ticket *get_mdl_ticket(TABLE *table) noexcept; #include "ha_innodb.h" #include "i_s.h" +#include "backup_innodb.h" #include #include @@ -4185,6 +4186,9 @@ static int innodb_init(void* p) = innodb_prepare_commit_versioned; innobase_hton->update_optimizer_costs= innobase_update_optimizer_costs; + innobase_hton->backup_start = innodb_backup_start; + innobase_hton->backup_step = innodb_backup_step; + innobase_hton->backup_end = innodb_backup_end; innobase_hton->binlog_init= innodb_binlog_init; innobase_hton->set_binlog_max_size= ibb_set_max_size; innobase_hton->binlog_write_direct_ordered= @@ -18885,39 +18889,7 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_CANT_CREATE_HANDLER_FILE); break; case log_t::RESIZE_STARTED: - for (timespec abstime;;) - { - if (thd_kill_level(thd)) - { - log_sys.resize_abort(thd); - break; - } - - set_timespec(abstime, 5); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - lsn_t resizing= log_sys.resize_in_progress(); - if (resizing > buf_pool.get_oldest_modification(0)) - { - buf_pool.page_cleaner_wakeup(true); - my_cond_timedwait(&buf_pool.done_flush_list, - &buf_pool.flush_list_mutex.m_mutex, &abstime); - resizing= log_sys.resize_in_progress(); - } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (!resizing || !log_sys.resize_running(thd)) - break; - log_sys.latch.wr_lock(); - while (resizing > log_sys.get_lsn()) - { - ut_ad(!log_sys.is_mmap()); - /* The server is almost idle. Write dummy FILE_CHECKPOINT records - to ensure that the log resizing will complete. */ - mtr_t mtr{nullptr}; - mtr.start(); - mtr.commit_files(log_sys.last_checkpoint_lsn); - } - log_sys.latch.wr_unlock(); - } + log_sys.resize_finish(thd); } } mysql_mutex_lock(&LOCK_global_system_variables); @@ -19774,7 +19746,9 @@ static void innodb_log_archive_update(THD *thd, st_mysql_sys_var*, { /* MDEV-36828 TODO: On failure, report which other setting conflicted with the request */ + mysql_mutex_unlock(&LOCK_global_system_variables); log_sys.set_archive(*static_cast(save), thd); + mysql_mutex_lock(&LOCK_global_system_variables); } static MYSQL_SYSVAR_BOOL(log_archive, log_sys.archive, @@ -19787,10 +19761,20 @@ static MYSQL_SYSVAR_UINT64_T(log_archive_start, innodb_log_archive_start, "initial value of innodb_lsn_archived; 0=auto-detect", nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); +static void innodb_log_recovery_start_update(THD *, st_mysql_sys_var*, + void *, const void *save) noexcept +{ + const lsn_t lsn{*static_cast(save)}; + recv_sys.recovery_start= lsn; + if (lsn && log_sys.archive) + log_sys.archived_checkpoint= lsn; +} + static MYSQL_SYSVAR_UINT64_T(log_recovery_start, recv_sys.recovery_start, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + PLUGIN_VAR_RQCMDARG, "LSN to start recovery from (0=automatic)", - nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); + nullptr, innodb_log_recovery_start_update, + 0, 0, std::numeric_limits::max(), 0); static MYSQL_SYSVAR_UINT64_T(log_recovery_target, recv_sys.rpo, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 3328319fd82b4..33ba10d7acaa3 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -644,6 +644,14 @@ class buf_page_t /** @return whether this block is read fixed */ bool is_read_fixed() const noexcept { return is_read_fixed(state()); } + /** Try to write-fix a block. + @param s expected state() + @return new s (right before potentially setting the write-fix); + a write-fix was acquired if !is_freed(s) && !is_io_fixed(s) holds */ + inline uint32_t write_fix_try(uint32_t s) noexcept; + /** Try to undo a successful write_fix_try(). */ + inline void write_unfix_try() noexcept; + /** @return if this belongs to buf_pool.unzip_LRU */ bool belongs_to_unzip_LRU() const noexcept { return UNIV_LIKELY_NULL(zip.data) && frame; } @@ -653,13 +661,8 @@ class buf_page_t bool is_freed() const noexcept { return is_freed(state()); } bool is_reinit() const { return !(~state() & REINIT); } - void set_reinit(uint32_t prev_state) noexcept - { - ut_ad(prev_state < READ_FIX); - ut_d(const auto s=) zip.fix.fetch_add(REINIT - prev_state); - ut_ad(s > prev_state); - ut_ad(s < prev_state + UNFIXED); - } + /** Mark the block as reinitialized in the file. @see set_freed() */ + void set_reinit() noexcept; uint32_t read_unfix(uint32_t s) noexcept { @@ -671,14 +674,8 @@ class buf_page_t return old_state + (s - READ_FIX); } - void set_freed(uint32_t prev_state, uint32_t count= 0) noexcept - { - ut_ad(lock.is_write_locked()); - ut_ad(prev_state >= UNFIXED); - ut_ad(prev_state < READ_FIX); - ut_d(auto s=) zip.fix.fetch_sub((prev_state & LRU_MASK) - FREED - count); - ut_ad(!((prev_state ^ s) & LRU_MASK)); - } + /** Mark an X-latched block as freed in the tablespace. @see set_reinit() */ + void set_freed() noexcept; inline void set_state(uint32_t s) noexcept; inline void set_corrupt_id() noexcept; diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h index 04d9b5485b165..7f70ccd59b887 100644 --- a/storage/innobase/include/buf0dblwr.h +++ b/storage/innobase/include/buf0dblwr.h @@ -177,6 +177,13 @@ class buf_dblwr_t (id >= block2 && id < block2 + block_size); } + /** the first page number */ + uint32_t begin() const noexcept { return block1.page_no(); } + /** @return the first page number after the doublewrite buffer */ + uint32_t end() const noexcept { return block2.page_no() + block_size; } + /** the size of the doublewrite buffer, in pages */ + uint32_t size() const noexcept { return 2 * block_size; } + /** Wait for flush_buffered_writes() to be fully completed */ void wait_flush_buffered_writes() noexcept { diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 67fa1bfd026f0..7d5467ecaa077 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -408,6 +408,9 @@ struct fil_space_t final /** Whether any corruption of this tablespace has been reported */ mutable std::atomic_flag is_corrupted= ATOMIC_FLAG_INIT; + /** first page number that is not yet being backed up, or 0 */ + std::atomic backup_end{0}; + public: /** mutex to protect freed_ranges and last_freed_lsn */ std::mutex freed_range_mutex; @@ -418,9 +421,11 @@ struct fil_space_t final /** LSN of freeing last page; protected by freed_range_mutex */ lsn_t last_freed_lsn= 0; - /** LSN of undo tablespace creation or 0; protected by latch */ - lsn_t create_lsn= 0; public: + /** LSN of tablespace creation or undo tablespace reinitialization; + protected by fil_system.mutex and (is_stopped() or log_sys.latch) */ + Atomic_relaxed create_lsn{0}; + /** @return whether this is the temporary tablespace */ bool is_temporary() const noexcept { return UNIV_UNLIKELY(id == SRV_TMP_SPACE_ID); } @@ -434,12 +439,6 @@ struct fil_space_t final /** @return whether a page has been freed */ inline bool is_freed(uint32_t page) noexcept; - /** Set create_lsn. */ - inline void set_create_lsn(lsn_t lsn) noexcept; - - /** @return the latest tablespace rebuild LSN, or 0 */ - lsn_t get_create_lsn() const noexcept { return create_lsn; } - /** Apply freed_ranges to the file. @param writable whether the file is writable @return number of pages written or hole-punched */ @@ -1058,6 +1057,19 @@ struct fil_space_t final VALIDATE_IMPORT }; + /** Note that we backing up some pages of the underlying files. + @param last_page the last page that is being backed up (0=stop backup) */ + void backup_start(uint32_t last_page) noexcept + { backup_end.store(last_page, std::memory_order_release); } + /** Note that we are not currently backing up the underlying files. */ + void backup_stop() noexcept { backup_start(0); } + /** @return the first page number that is not being backed up */ + uint32_t backup_page_end() const noexcept + { return backup_end.load(std::memory_order_acquire); } + + /** The size of a backup::copy() batch in pages */ + static constexpr uint32_t BACKUP_BATCH_SIZE{64}; + /** Update the data structures on write completion */ void complete_write() noexcept; @@ -1463,6 +1475,8 @@ struct fil_system_t my_bool buffered; /** whether fdatasync() is needed on data files */ Atomic_relaxed need_unflushed_spaces; + /** whether dict_load_tablespaces(nullptr, true) is unnecessary */ + Atomic_relaxed have_all_spaces; /** Try to enable or disable write-through of data files */ void set_write_through(bool write_through); diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index bb5456f056e2e..eada4ddf19df6 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -221,6 +221,8 @@ struct log_t /** whether !archive log records may have been written with get_sequence_bit()==0 */ bool circular_recovery_from_sequence_bit_0:1; + /** whether we are between backup_start() and backup_stop() */ + bool backup:1; public: /** the default value of log_mmap */ static constexpr bool log_mmap_default= @@ -288,6 +290,8 @@ struct log_t Atomic_relaxed last_checkpoint_lsn; /** The log writer (protected by latch.wr_lock()) */ lsn_t (*writer)() noexcept; + /** the earliest available checkpoint; protected by latch.wr_lock() */ + lsn_t archived_checkpoint; /** end_lsn of the first available checkpoint, or 0; protected by latch.wr_lock() */ lsn_t archived_lsn; @@ -369,11 +373,24 @@ struct log_t RESIZE_NO_CHANGE, RESIZE_IN_PROGRESS, RESIZE_STARTED, RESIZE_FAILED }; +private: /** Start resizing the log and release the exclusive latch. + @param size requested new file_size + @param thd the current thread identifier + @param backup whether the caller is backup_start() or backup_stop() + @return whether the resizing was started successfully */ + resize_start_status resize_start(uint64_t size, void *thd, bool backup) + noexcept; +public: + /** Start resizing the log. @param size requested new file_size @param thd the current thread identifier @return whether the resizing was started successfully */ - resize_start_status resize_start(os_offset_t size, void *thd) noexcept; + resize_start_status resize_start(uint64_t size, void *thd) noexcept + { return resize_start(size, thd, false); } + + /** Wait for the completion of resize_start() == RESIZE_STARTED */ + void resize_finish(THD *thd) noexcept; /** Abort a resize_start() that we started. @param thd thread identifier that had been passed to resize_start() */ @@ -397,10 +414,37 @@ struct log_t resize_write_low(lsn, end, len, seq); } +private: + /** SET GLOBAL innodb_log_archive, or start/stop BACKUP SERVER + @param archive the new value of innodb_log_archive + @param thd SQL connection + @param backup whether the caller is backup_start() or backup_stop() + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd, bool backup) noexcept; +public: /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive - @param thd SQL connection */ - void set_archive(my_bool archive, THD *thd) noexcept; + @param thd SQL connection + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd) noexcept + { return set_archive(archive, thd, false); } + + /** Start BACKUP SERVER. + @param old_size the old file_size, or 0 on failure or when + already running innodb_log_archive=ON + @param thd SQL connection + @return whether the operation failed */ + bool backup_start(uint64_t *old_size, THD *thd) noexcept; + /** Stop log archiving in BACKUP SERVER clean-up + @param thd SQL connection + @return whether the operation failed */ + bool backup_stop_archiving(THD *thd) noexcept + { return set_archive(false, thd, true); } + + /** Stop BACKUP SERVER. + @param old_size the value returned by backup_start() + @param thd SQL connection */ + void backup_stop(uint64_t old_size, THD *thd) noexcept; private: /** Replicate a write to the log. @@ -700,6 +744,18 @@ struct log_t /** @return the first LSN of the log file */ lsn_t get_first_lsn() const noexcept { return first_lsn; } + /** + Determine the latest checkpoint. + @param end LSN leading to the FILE_CHECKPOINT record + @return the latest checkpoint LSN + */ + lsn_t get_latest_checkpoint(lsn_t &end) const noexcept + { + ut_ad(latch_have_any()); + end= end_lsn; + return last_checkpoint_lsn; + } + /** Set the recovered checkpoint. @param lsn log sequence number of the checkpoint @param end_lsn LSN passed to write_checkpoint() diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index f1834bd8f670c..16e7983e705ca 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -635,7 +635,7 @@ void log_t::set_buffered(bool buffered) noexcept } #endif - /** Try to enable or disable durable writes (update log_write_through) */ +/** Try to enable or disable durable writes (update log_write_through) */ void log_t::set_write_through(bool write_through) { if (is_mmap() || high_level_read_only || recv_sys.rpo) @@ -764,9 +764,12 @@ void log_t::header_rewrite(my_bool archive) noexcept /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive -@param thd SQL connection */ -void log_t::set_archive(my_bool archive, THD *thd) noexcept +@param thd SQL connection +@param backup whether the caller is backup_start() or backup_stop() +@return whether the operation failed */ +bool log_t::set_archive(my_bool archive, THD *thd, bool backup) noexcept { + bool fail= false; thd_wait_begin(thd, THD_WAIT_DISKIO); tpool::tpool_wait_begin(); lsn_t wait_lsn; @@ -780,12 +783,20 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept my_printf_error(ER_WRONG_USAGE, "SET GLOBAL innodb_log_file_size is in progress", MYF(0)); + fail: + fail= true; + wait_lsn= 0; break; } if (archive == this->archive) break; - if (thd_kill_level(thd)) - break; + if ((!backup || archive) && thd_kill_level(thd)) + goto fail; + if (!backup && this->backup) + { + my_printf_error(ER_WRONG_USAGE, "BACKUP SERVER is in progress", MYF(0)); + goto fail; + } if (resize_log.is_opened()) { @@ -920,7 +931,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept if (!log.is_opened()) { my_error(ER_ERROR_ON_READ, MYF(0), old_name, errno); - break; + goto fail; } } #endif @@ -945,7 +956,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept { my_error(ER_ERROR_ON_RENAME, MYF(0), old_name, new_name, my_errno); first_lsn= old_first_lsn; - break; + goto fail; } if (archive) @@ -977,14 +988,16 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept thd_wait_end(thd); if (wait_lsn) mtr_flush_ahead(wait_lsn); + return fail; } -/** Start resizing the log and release the exclusive latch. -@param size requested new file_size -@param thd the current thread identifier +/** Start resizing the log. +@param size requested new file_size +@param thd the current thread identifier +@param backup whether the caller is backup_start() or backup_stop() @return whether the resizing was started successfully */ -log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) - noexcept +log_t::resize_start_status log_t::resize_start(uint64_t size, void *thd, + bool backup) noexcept { ut_ad(size >= 4U << 20); ut_ad(!(size & 4095)); @@ -1015,6 +1028,9 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) resize_target= size; } } + else if (!backup && this->backup) + /* backup_start() or backup_stop() is running */ + status= RESIZE_FAILED; else { lsn_t start_lsn; @@ -1116,6 +1132,44 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) return status; } +/** Wait for the completion of resize_start() == RESIZE_STARTED */ +void log_t::resize_finish(THD *thd) noexcept +{ + for (timespec abstime;;) + { + if (thd_kill_level(thd)) + { + resize_abort(thd); + break; + } + + set_timespec(abstime, 5); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + lsn_t resizing= resize_in_progress(); + if (resizing > buf_pool.get_oldest_modification(0)) + { + buf_pool.page_cleaner_wakeup(true); + my_cond_timedwait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); + resizing= resize_in_progress(); + } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + if (!resizing || !resize_running(thd)) + break; + latch.wr_lock(); + while (resizing > get_lsn()) + { + ut_ad(!is_mmap()); + /* The server is almost idle. Write dummy FILE_CHECKPOINT records + to ensure that the log resizing will complete. */ + mtr_t mtr{nullptr}; + mtr.start(); + mtr.commit_files(last_checkpoint_lsn); + } + latch.wr_unlock(); + } +} + /** Abort a resize_start() that we started. */ void log_t::resize_abort(void *thd) noexcept { diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 690128f5685c9..b1136c8fd5758 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1260,6 +1260,7 @@ ATTRIBUTE_COLD static void fil_delete_apply(uint32_t id, const char *name) if (fil_ibd_load(id, name, space) == FIL_LOAD_OK) { ut_ad(space); + deferred_spaces.remove(id); fil_delete_apply(space); return; } @@ -1295,10 +1296,6 @@ static void fil_name_process(const char *name, ulint len, uint32_t space_id, file_name_t& f = p.first->second; ut_ad(!f.space || f.space->id == space_id); auto d = deferred_spaces.find(space_id); - /* deferred_spaces must be a proper subset of recv_spaces. - Both here and in recv_validate_tablespace(), deferred_spaces.add() - may only be invoked for entries that exist in recv_spaces. */ - ut_ad(!d || !p.second); if (deleted) { /* Got FILE_DELETE */ @@ -2159,6 +2156,7 @@ dberr_t recv_sys_t::find_checkpoint() memset_aligned<4096>(const_cast(field_ref_zero), 0, 4096); /* Mark the redo log for upgrading. */ lsn= file_checkpoint= log_sys.last_checkpoint_lsn; + log_sys.archived_checkpoint= lsn; log_sys.set_recovered_lsn(lsn); if (rpo && rpo != lsn) { @@ -2237,7 +2235,8 @@ dberr_t recv_sys_t::find_checkpoint() log_sys.set_recovered_checkpoint(checkpoint_lsn, lsn= end_lsn, field == log_t::CHECKPOINT_1); } - if (!log_sys.last_checkpoint_lsn) + log_sys.archived_checkpoint= log_sys.last_checkpoint_lsn; + if (!log_sys.archived_checkpoint) goto got_no_checkpoint; else if (!log_sys.archived_lsn) log_sys.archived_lsn= lsn; @@ -4228,7 +4227,7 @@ static buf_block_t *recv_recover_page(buf_block_t *block, mtr_t &mtr, /* There have been no operations that modify the page. Any buffered changes will be merged in ibuf_upgrade(). */ ut_ad(!mtr.has_modifications()); - block->page.set_freed(block->page.state()); + block->page.set_freed(); } /* Make sure that committing mtr does not change the modification diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 21e4d1ae1c9ce..8afeacd1f3752 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -313,7 +313,8 @@ void mtr_t::release_unlogged() buf_block_t *block= static_cast(slot.object); ut_d(const auto s=) block->page.unfix(); ut_ad(s >= buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* A fake "write fix" of InnoDB_backup may exist for a latched page */ + ut_ad(!buf_page_t::is_read_fixed(s)); if (slot.type & MTR_MEMO_MODIFY) { @@ -440,7 +441,8 @@ void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) noexcept ut_ad(b->page.id() < end_page_id); ut_d(const auto s= b->page.state()); ut_ad(s > buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* A fake "write fix" of InnoDB_backup may exist for a latched page */ + ut_ad(!buf_page_t::is_read_fixed(s)); ut_ad(mach_read_from_8(b->page.frame + FIL_PAGE_LSN) <= mtr->m_commit_lsn); mach_write_to_8(b->page.frame + FIL_PAGE_LSN, mtr->m_commit_lsn); @@ -492,7 +494,13 @@ void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) noexcept ut_ad(bpage->oldest_modification() < mtr->m_commit_lsn); ut_ad(bpage->id() < end_page_id); ut_ad(s >= buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* If a thread is executing between + InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() for this page, a fake + "write fix" may exist. We are free to modify the page in the + buffer pool, but buf_page_t::flush() will refuse to write it + to the file system. */ + ut_ad(!buf_page_t::is_read_fixed(s)); ut_ad(mach_read_from_8(bpage->frame + FIL_PAGE_LSN) <= mtr->m_commit_lsn); mach_write_to_8(bpage->frame + FIL_PAGE_LSN, mtr->m_commit_lsn); @@ -595,12 +603,34 @@ void mtr_t::rollback_to_savepoint(ulint begin, ulint end) m_memo.erase(m_memo.begin() + begin, m_memo.begin() + end); } -/** Set create_lsn. */ -inline void fil_space_t::set_create_lsn(lsn_t lsn) noexcept +/** Mark an X-latched block as freed in the tablespace. @see set_reinit() */ +void buf_page_t::set_freed() noexcept { - /* Concurrent log_checkpoint_low() must be impossible. */ - ut_ad(latch.have_wr()); - create_lsn= lsn; + /* + Ensure that lock.x_lock() is being held (hopefully, by us). + This blocks a concurrent flush(), protected by lock.u_lock(). + */ + ut_ad(lock.is_write_locked()); + uint32_t s{state()}; + do + { + ut_ad(s >= UNFIXED); + /* + InnoDB_backup::backup_batch_start() may set fix() and + write_fix_try() on any dirty block to prevent flush() from + writing changes back to the file system. We may safely mark such + blocks as FREED; write_unfix_try() will account for that. + */ + ut_ad(s > WRITE_FIX || s < READ_FIX); + } + /* + fetch_sub() is not safe here, because this may run concurrently + with write_fix_try() or write_unfix_try(). + */ + while (!zip.fix.compare_exchange_weak(s, FREED + (s & ~LRU_MASK), + std::memory_order_acquire, + std::memory_order_relaxed) && + !is_freed(s)); } /** Commit a mini-transaction that is shrinking a tablespace. @@ -635,7 +665,7 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) if (space.id == TRX_SYS_SPACE) srv_sys_space.set_last_file_size(file->size); else - space.set_create_lsn(m_commit_lsn); + space.create_lsn= m_commit_lsn; mysql_mutex_unlock(&fil_system.mutex); @@ -674,7 +704,8 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) const page_id_t id{b->page.id()}; const auto s= b->page.state(); ut_ad(s > buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* A fake "write fix" of InnoDB_backup may exist for a latched page */ + ut_ad(!buf_page_t::is_read_fixed(s)); ut_ad(b->page.frame); ut_ad(mach_read_from_8(b->page.frame + FIL_PAGE_LSN) <= m_commit_lsn); ut_ad(!b->page.zip.data); // we no not shrink ROW_FORMAT=COMPRESSED @@ -695,7 +726,7 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) { ut_ad(id.space() == high.space()); if (s >= buf_page_t::UNFIXED) - b->page.set_freed(s); + b->page.set_freed(); if (b->page.oldest_modification() > 1) b->page.reset_oldest_modification(); slot.type= mtr_memo_type_t(slot.type & ~MTR_MEMO_MODIFY); @@ -1542,7 +1573,8 @@ buf_block_t *mtr_t::page_lock(buf_block_t *block, ulint rw_latch) noexcept case RW_SX_LATCH: fix_type= MTR_MEMO_PAGE_SX_FIX; block->page.lock.u_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); break; default: ut_ad(rw_latch == RW_X_LATCH); @@ -1553,7 +1585,8 @@ buf_block_t *mtr_t::page_lock(buf_block_t *block, ulint rw_latch) noexcept page_lock_upgrade(*block); return block; } - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); } done: @@ -1588,11 +1621,13 @@ void mtr_t::upgrade_buffer_fix(ulint savepoint, rw_lock_type_t rw_latch) break; case RW_SX_LATCH: block->page.lock.u_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); break; case RW_X_LATCH: block->page.lock.x_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); } ut_ad(page_id_t(page_get_space_id(block->page.frame), @@ -1784,7 +1819,7 @@ void mtr_t::init(buf_block_t *b) m_freed_space= nullptr; } - b->page.set_reinit(b->page.state() & buf_page_t::LRU_MASK); + b->page.set_reinit(); if (!is_logged()) return; @@ -1849,7 +1884,7 @@ void mtr_t::free(const fil_space_t &space, uint32_t offset) if (block->index) btr_search_drop_page_hash_index(block, nullptr); #endif /* BTR_CUR_HASH_ADAPT */ - block->page.set_freed(block->page.state()); + block->page.set_freed(); } } diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 6494c5e21b96e..4ef86095f78a3 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2017,7 +2017,15 @@ os_file_create_func( ); DWORD create_flag = OPEN_EXISTING; - DWORD share_mode = read_only + /* BACKUP SERVER may invoke CreateHardLink() on a log file that + may concurrently be written to. This is why we must allow + FILE_SHARE_WRITE. This has the side effect that multiple InnoDB + instances may be concurrently started on the same log file. + However, InnoDB will not write any log before it has successfully + opened data files. As long as the multiple instances are also + opening the same InnoDB data files (such as the system tablespace), + they should fail to start up concurrently. */ + DWORD share_mode = read_only || type == OS_LOG_FILE ? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE : FILE_SHARE_READ | FILE_SHARE_DELETE; diff --git a/storage/maria/CMakeLists.txt b/storage/maria/CMakeLists.txt index 9d528f980dc5b..43b950f05d99a 100644 --- a/storage/maria/CMakeLists.txt +++ b/storage/maria/CMakeLists.txt @@ -45,6 +45,7 @@ SET(ARIA_SOURCES ma_init.c ma_open.c ma_extra.c ma_info.c ma_rkey.c ha_maria.h maria_def.h ma_recovery_util.c ma_servicethread.c ma_norec.c ma_crypt.c ma_backup.c + ma_backup_server.c ma_backup_server.h ) IF(APPLE) diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 8f5e47daea728..736ef1e9c49f2 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -23,6 +23,7 @@ #include #include #include "ha_maria.h" +#include "ma_backup_server.h" #include "trnman_public.h" #include "trnman.h" @@ -3728,7 +3729,21 @@ void maria_end_backup() translog_enable_purge(); } +/* A C++ function pointer compatible wrapper of a C function */ +static void * +aria_backup_start_wrap(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) +{ + return aria_backup_start(thd, target, phase, sink); +} +/* A C++ function pointer compatible wrapper of a C function */ +static int +aria_backup_end_wrap(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) +{ + return aria_backup_end(thd, target, phase, sink); +} #define SHOW_MSG_LEN (FN_REFLEN + 20) /** @@ -3942,6 +3957,9 @@ static int ha_maria_init(void *p) maria_hton->prepare_for_backup= maria_prepare_for_backup; maria_hton->end_backup= maria_end_backup; maria_hton->update_optimizer_costs= aria_update_optimizer_costs; + maria_hton->backup_start= aria_backup_start_wrap; + //maria_hton->backup_step= aria_backup_step_wrap; + maria_hton->backup_end= aria_backup_end_wrap; /* TODO: decide if we support Maria being used for log tables */ maria_hton->flags= (HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES | diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c new file mode 100644 index 0000000000000..cef73bcf8b04d --- /dev/null +++ b/storage/maria/ma_backup_server.c @@ -0,0 +1,379 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "maria_def.h" +#include "ma_backup_server.h" +#include "mysqld_error.h" + +#if 1 /* can't #include "sql/table.h" because it is C++ */ +# define tmp_file_prefix "#sql" +# define tmp_file_prefix_length 4 +#endif + +#if 1 /* can't #include "sql/mysqld.h" because it is C++ */ +extern char mysql_real_data_home[]; +#endif + +ATTRIBUTE_COLD ATTRIBUTE_NOINLINE static int dir_error(const char *name) +{ + my_error(ER_CANT_READ_DIR, MYF(0), name, my_errno); + return 1; +} + +/** + Determine if a file may be backed up. + @param file_name candidate file name + @retval FALSE if the file must be excluded + @retval TRUE if the file may be included +*/ +static int is_db_file(const char *file_name) +{ + size_t len= strlen(file_name); + uint32_t suffix; + if (len < 4) + return FALSE; + if (!memcmp(file_name, tmp_file_prefix, tmp_file_prefix_length)) + /* + As noted in MDEV-25854, file names that start with #sql + must be excluded from the backup. For example, a call to + MDL_context::upgrade_shared_lock() in + mysql_inplace_alter_table() could time out, resulting in + cleanup_table_after_inplace_alter() deleting a + #sql-alter*.frm file before we get a chance to copy it. + */ + return FALSE; + memcpy(&suffix, file_name + len - 4, 4); + switch (suffix) { + default: + return len == 6 && !memcmp(file_name, C_STRING_WITH_LEN("db.opt")); +#ifdef WORDS_BIGENDIAN + case 0x2e41524d: /* .ARM ENGINE=ARCHIVE metadata */ + case 0x2e41525a: /* .ARZ ENGINE=ARCHIVE compressed data */ + case 0x2e43534d: /* .CSM ENGINE=CSV metadata */ + case 0x2e435356: /* .CSV ENGINE=CSV data ("comma separated values") */ + case 0x2e4d4144: /* .MAD ENGINE=Aria data heap */ + case 0x2e4d4149: /* .MAI ENGINE=Aria indexes */ + case 0x2e4d5247: /* .MRG ENGINE=MRG_MyISAM */ + case 0x2e4d5944: /* .MYD ENGINE=MyISAM data heap */ + case 0x2e4d5949: /* .MYI ENGINE=MyISAM indexes */ + case 0x2e66726d: /* .frm form (SHOW CREATE TABLE) */ + case 0x2e706172: /* .par PARTITION metadata */ +#else + case 0x4d52412e: /* .ARM ENGINE=ARCHIVE metadata */ + case 0x5a52412e: /* .ARZ ENGINE=ARCHIVE compressed data */ + case 0x4d53432e: /* .CSM ENGINE=CSV metadata */ + case 0x5653432e: /* .CSV ENGINE=CSV data ("comma separated values") */ + case 0x44414d2e: /* .MAD ENGINE=Aria data heap */ + case 0x49414d2e: /* .MAI ENGINE=Aria indexes */ + case 0x47524d2e: /* .MRG ENGINE=MRG_MyISAM */ + case 0x44594d2e: /* .MYD ENGINE=MyISAM data heap */ + case 0x49594d2e: /* .MYI ENGINE=MyISAM indexes */ + case 0x6d72662e: /* .frm form (SHOW CREATE TABLE) */ + case 0x7261702e: /* .par PARTITION metadata */ +#endif + return TRUE; + } +} + +struct Aria_backup_entry +{ + /** directory name */ + const char *dir; + /** file name relative to the directory */ + const char *name; +}; + +/** Backup state */ +struct Aria_backup +{ + /** whether translog_disable_purge() is in effect */ + int translog_purge_disabled; +}; + +static void aria_backup_init(struct Aria_backup *ab) +{ + ab->translog_purge_disabled= TRUE; + translog_disable_purge(); +} + +static void aria_backup_destroy(const struct Aria_backup *ab) +{ + if (ab && ab->translog_purge_disabled) + translog_enable_purge(); +} + +/* + Create directory in the target directory if it does not exist. + Return 0 on success, non-0 on failure. Set errno in case of failure +*/ +static int aria_backup_mkdir(const struct backup_target *target, + const char *name) +{ +#ifdef _WIN32 + if (!target->path) + return 0; + char path[FN_REFLEN]; + if ((int) sizeof path <= + snprintf(path, sizeof path, "%s/%s", target->path, name)) + { + my_error(ER_TOO_LONG_IDENT, MYF(0), name); + return -1; + } + if (CreateDirectory(path, NULL)) + return 0; + DWORD err= GetLastError(); + if (err == ERROR_ALREADY_EXISTS) + return 0; + my_osmaperr(err); +#else + if (target->fd == -1 || likely(!mkdirat(target->fd, name, 0777)) || + errno == EEXIST) + return 0; +#endif + my_error(ER_CANT_CREATE_FILE, MYF(0), name, errno); + return 1; +} + +static int aria_backup_file(const struct backup_target *target, + const struct backup_sink *sink, + const char *dir, const char *name, + size_t dir_prefix, int if_exists) +{ +#ifndef _WIN32 + int src, dst= sink->stream; +#endif + int ret= -1; + char path[FN_REFLEN * 3 / 2]; + if ((int) sizeof path <= snprintf(path, sizeof path, "%s/%s", dir, name)) + { + my_error(ER_TOO_LONG_IDENT, MYF(0), name); + return -1; + } +#ifndef _WIN32 + src= open(path, O_RDONLY); + if (src < 0) + { + my_error(ER_CANT_OPEN_FILE, MYF(0), path, errno); + return ret; + } + if (dst < 0) + { + dst= openat(target->fd, path + dir_prefix, + O_CREAT | O_EXCL | O_WRONLY, 0666); + if (dst < 0) + my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); + else + { + ret= copy_entire_file(src, dst) | close(dst); + if (ret) + write_error: + my_error(ER_ERROR_ON_WRITE, MYF(0), path + dir_prefix, errno); + } + } + else + { + uint64_t end= (uint64_t) lseek(src, 0, SEEK_END); + ret= backup_stream_start(dst, path + dir_prefix, 0644, end, NULL, 0) || + backup_stream_append_plain(src, dst, 0, end) || + backup_stream_zeropad(dst, (size_t) end); + if (ret) + goto write_error; + } + close(src); + return ret; +#else + if (sink->stream == INVALID_HANDLE_VALUE) + { + char dstpath[FN_REFLEN * 3 / 2]; + if ((int) sizeof dstpath <= + snprintf(dstpath, sizeof dstpath, "%s/%s", + target->path, path + dir_prefix)) + my_error(ER_TOO_LONG_IDENT, MYF(0), name); + else if (!CopyFileEx(path, dstpath, NULL, NULL, NULL, + COPY_FILE_NO_BUFFERING)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_CREATE_FILE, MYF(0), dstpath, errno); + } + else + return 0; + } + else + { + LARGE_INTEGER li; + HANDLE src, dst= sink->stream; + for (;;) + { + src= CreateFile(path, GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL); + if (src != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + my_sleep(1000000); + continue; + } + + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), path, errno); + return ret; + } + + ret= !GetFileSizeEx(src, &li) || + backup_stream_start(dst, path + dir_prefix, 0644, li.QuadPart, + NULL, 0) || + backup_stream_append_plain(src, dst, 0, li.QuadPart) || + backup_stream_zeropad(dst, (size_t) li.QuadPart); + (void) CloseHandle(src); + + if (ret) + { + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_WRITE, MYF(0), path + dir_prefix, errno); + } + } + return ret; +#endif +} + +static int aria_backup_dir(const struct backup_target *target, + const struct backup_sink *sink, + const char *dir_name, size_t prefix) +{ + int fail= 0; + char path[FN_REFLEN]; + if ((int) sizeof path <= + snprintf(path, sizeof path, "%s/%s", mysql_real_data_home, dir_name)) + { + my_error(ER_TOO_LONG_IDENT, MYF(0), dir_name); + return -1; + } + else if ((fail= aria_backup_mkdir(target, path + prefix)) != 0) + return fail; + else + { + MY_DIR *dir= my_dir(path, MYF(MY_WANT_STAT)); + if (!dir) + return dir_error(path); + else + { + const struct fileinfo *fi= dir->dir_entry; + const struct fileinfo *const end= fi + dir->number_of_files; + for (; fi < end; fi++) + if (is_db_file(fi->name)) + if ((fail= aria_backup_file(target, sink, path, fi->name, + prefix, 0)) != 0) + break; + my_dirend(dir); + return fail; + } + } +} + +static int aria_backup_scan(const struct backup_target *target, + const struct backup_sink *sink) +{ + int fail= 0; + size_t prefix= strlen(mysql_real_data_home) + 1; + /* Scan the server data directory for data files. */ + MY_DIR *dir= my_dir(mysql_real_data_home, MYF(MY_WANT_STAT)); + if (!dir) + return dir_error(mysql_real_data_home); + else + { + const struct fileinfo *fi= dir->dir_entry; + const struct fileinfo *const end= fi + dir->number_of_files; + for (; fi < end; fi++) + { + if ((fi->mystat->st_mode & S_IFMT) == S_IFDIR) + if ((fail= aria_backup_dir(target, sink, fi->name, prefix)) != 0) + break; + } + my_dirend(dir); + } + if (fail) + return fail; + /* Process the Aria logs. */ + prefix= strlen(maria_data_root) + 1; + fail= aria_backup_file(target, sink, maria_data_root, "aria_log_control", + prefix, 1); + if (fail) + return fail; + translog_flush(translog_get_horizon()); + dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)); + if (!dir) + return dir_error(maria_data_root); + else + { + const struct fileinfo *fi= dir->dir_entry; + const struct fileinfo *const end= fi + dir->number_of_files; + for (; fi < end; fi++) + if ((fail= + !strncmp(fi->name, C_STRING_WITH_LEN("aria_log.")) && + aria_backup_file(target, sink, maria_data_root, fi->name, + prefix, 0)) != 0) + break; + my_dirend(dir); + } + return fail; +} + +void *aria_backup_start(THD *thd, const struct backup_target *target, + enum backup_phase phase, + const struct backup_sink *sink) +{ + switch (phase) { + struct Aria_backup *aria_backup; + case BACKUP_PHASE_PREPARE_START: + return 0; + default: + return sink->ha_data; + case BACKUP_PHASE_NO_COMMIT: + assert(!sink->ha_data); + aria_backup= calloc(1, sizeof *aria_backup); + if (!aria_backup) + return (void*) -1; + aria_backup_init(aria_backup); + return aria_backup; + } +} + +int aria_backup_end(THD *thd, const struct backup_target *target, + enum backup_phase phase, const struct backup_sink *sink) +{ + struct Aria_backup *aria_backup= sink->ha_data; + int ret= 0; + switch (phase) { + extern void purge_tables(void); + case BACKUP_PHASE_NO_COMMIT: + assert(aria_backup); + assert(aria_backup->translog_purge_disabled); + aria_backup->translog_purge_disabled= FALSE; + purge_tables(); // TODO: do not close transactional tables + ret= aria_backup_scan(target, sink); + translog_enable_purge(); + break; + case BACKUP_PHASE_FINISH: + aria_backup_destroy(aria_backup); + free(aria_backup); + break; + default: + break; + } + return ret; +} diff --git a/storage/maria/ma_backup_server.h b/storage/maria/ma_backup_server.h new file mode 100644 index 0000000000000..e021833e9476a --- /dev/null +++ b/storage/maria/ma_backup_server.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#pragma once + +#include "sql_backup_interface.h" + +#ifdef __cplusplus +extern "C" +#endif +/** + Start of a BACKUP SERVER phase, + when no aria_backup_step() or aria_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to backup_target, or nullptr + @retval -1 on failure +*/ +void *aria_backup_start(THD *thd, const struct backup_target *target, + enum backup_phase phase, + const struct backup_sink *sink); + +#ifdef __cplusplus +extern "C" +#endif +/** + Process a file that was collected in aria_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @retval 0 on completion +*/ +int aria_backup_step(THD *thd, const struct backup_target *target, + enum backup_phase phase, const struct backup_sink *sink); + +#ifdef __cplusplus +extern "C" +#endif +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int aria_backup_end(THD *thd, const struct backup_target *target, + enum backup_phase phase, const struct backup_sink *sink); From 208ce0ae56e1ddd990555028b82aabd96909484f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Aug 2026 13:45:43 +0300 Subject: [PATCH 03/17] squash! b625be27ca0a0b24730dd19d37d8bbf85917156c InnoDB_backup::context: Remove the pointer indirection and do not allow two overlapping backup operations. InnoDB_backup::init(): Wait for a possible previous BACKUP SERVER operation to reach the very end of InnoDB_backup::context::cleanup() so that the context can be safely reused. --- storage/innobase/handler/backup_innodb.cc | 244 ++++++++++++++-------- 1 file changed, 153 insertions(+), 91 deletions(-) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index 3cceaaff2d466..ba7a453a850ab 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -139,7 +139,7 @@ static buf_page_t **innodb_backup_batch_wait(buf_page_t **end, namespace { -/** Backup state; protected by log_sys.latch */ +/** Backup state and context; mostly protected by log_sys.latch */ class InnoDB_backup { public: @@ -147,27 +147,36 @@ class InnoDB_backup ~InnoDB_backup() { mutex.destroy(); } private: + enum State { + /** no BACKUP SERVER is in progress */ + IDLE, + /** BACKUP SERVER is active; checkpoint_complete() will fill the queue */ + PROCESSING, + /** BACKUP SERVER is cleaning up, not protected by any MDL */ + CLEANUP + }; + /** Backup context */ struct context { /** Start LSN of the first backed up log file */ - const lsn_t first_lsn; + const lsn_t first_lsn{}; /** Start LSN of the last log file, or LSN_MAX if not determined yet */ - lsn_t max_first_lsn; + lsn_t max_first_lsn{}; /** Final LSN of the backup, or LSN_MAX if not determined yet */ - lsn_t last_lsn; + lsn_t last_lsn{}; /** size of the first log file */ - const uint64_t first_size; + const uint64_t first_size{}; /** Checkpoint at the start of the backup */ - const lsn_t checkpoint; + const lsn_t checkpoint{}; /** Log record pointing to the checkpoint */ - const lsn_t checkpoint_end_lsn; + const lsn_t checkpoint_end_lsn{}; /** the original state of innodb_log_archive before/after backup */ - const bool archived; - /** whether end() was invoked */ - bool cleaned_up; + const bool archived{}; + /** state of the operation; protected by log_sys.latch */ + Atomic_relaxed state{IDLE}; /** the start LSN of the last hard-linked file, or 0 */ - std::atomic last_hardlink; + std::atomic last_hardlink{}; /** Note that a log file was hard-linked. @@ -319,26 +328,33 @@ class InnoDB_backup */ int cleanup(const backup_target &target, const backup_sink &sink) noexcept { + int fail{0}; + ut_ad(state == CLEANUP); const lsn_t hl{last_hardlink.load(std::memory_order_relaxed)}; - if (hl == LSN_MAX) - return 0; - log_sys.latch.rd_lock(); - const lsn_t current_first_lsn{log_sys.get_first_lsn()}; - log_sys.latch.rd_unlock(); - if (hl == current_first_lsn) + if (hl != LSN_MAX) { - ut_ad(sink.stream == sink.NO_STREAM); - if (int fail= de_hardlink(target, hl)) - return fail; + /* abort() had not been invoked for this backup; finish it */ + log_sys.latch.rd_lock(); + const lsn_t current_first_lsn{log_sys.get_first_lsn()}; + log_sys.latch.rd_unlock(); + + if (hl == current_first_lsn) + { + ut_ad(sink.stream == sink.NO_STREAM); + fail= de_hardlink(target, hl); + } + if (!fail) + fail= write_config(target, sink); } - return write_config(target, sink); + state= IDLE; /* unblock init() */ + return fail; } }; - /** pointer to backup context, or nullptr if no backup is active */ - context *ctx; + /** backup context */ + context ctx; - /** the original innodb_log_file_size, or 0 */ + /** the original innodb_log_file_size; 0 if innodb_log_archive was enabled */ uint64_t old_size; /** mutex protecting queue, non_log */ @@ -358,14 +374,18 @@ class InnoDB_backup void *init(THD *thd) noexcept { log_sys.latch.wr_lock(); - ut_ad(!ctx); - mutex.wr_lock(); + while (ctx.state != IDLE) + { + /* A previous BACKUP SERVER has not reached context::cleanup(). */ + log_sys.latch.wr_unlock(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + log_sys.latch.wr_lock(); + } + + ut_d(mutex.wr_lock()); ut_ad(!non_log); - if (!queue.empty()) - /* A new BACKUP SERVER is being invoked before a previous one - had been fully finalized. Clean up any log files. */ - delete_logs(); - mutex.wr_unlock(); + ut_ad(queue.empty()); + ut_d(mutex.wr_unlock()); if (log_sys.backup_start(&old_size, thd)) { @@ -376,6 +396,9 @@ class InnoDB_backup } mutex.wr_lock(); + ut_ad(!non_log); + ut_ad(queue.empty()); + ut_ad(ctx.state == IDLE); try { @@ -390,9 +413,9 @@ class InnoDB_backup ut_ad(start_end >= start); ut_ad(start >= log_sys.get_first_lsn()); - ctx= new context{ + new (&ctx) context{ log_sys.get_first_lsn(), LSN_MAX, LSN_MAX, log_sys.file_size, - start, start_end, !old_size, false, 0 + start, start_end, !old_size, PROCESSING, 0 }; /* Collect all tablespaces that have been created before our @@ -433,8 +456,6 @@ class InnoDB_backup } catch (std::bad_alloc&) { queue.clear(); - delete ctx; - ctx= nullptr; log_sys.backup_stop(old_size, thd); goto fail; } @@ -442,7 +463,7 @@ class InnoDB_backup mutex.wr_unlock(); log_sys.latch.wr_unlock(); DEBUG_SYNC(thd, "innodb_backup_start"); - return ctx; + return &ctx; } /** @@ -459,11 +480,9 @@ class InnoDB_backup { uint64_t id_limit{0}; mutex.wr_lock(); - ut_ad(sink.ha_data); - ut_ad(ctx ? ctx == sink.ha_data - : phase == BACKUP_PHASE_FINISH || phase == BACKUP_PHASE_NO_COMMIT); - ut_ad(static_cast(sink.ha_data)->last_lsn == LSN_MAX - ? phase == BACKUP_PHASE_START : !ctx); + ut_ad(&ctx == sink.ha_data); + ut_ad(ctx.state != IDLE); + ut_ad(ctx.last_lsn != LSN_MAX || phase == BACKUP_PHASE_START); const size_t size{queue.size()}, non_log_files{non_log}; ut_ad(size >= non_log_files); @@ -473,6 +492,7 @@ class InnoDB_backup return 0; } + ut_ad(ctx.state == PROCESSING); non_log-= size == non_log_files; id_limit= queue.back(); queue.pop_back(); @@ -592,8 +612,8 @@ class InnoDB_backup void commit() noexcept { log_sys.latch.wr_lock(); - ut_ad(ctx); - ut_ad(ctx->last_lsn == LSN_MAX); + ut_ad(ctx.state == PROCESSING); + ut_ad(ctx.last_lsn == LSN_MAX); const lsn_t last_lsn{log_sys.get_lsn()}; lsn_t lsn{log_sys.get_first_lsn()}; mutex.wr_lock(); @@ -607,80 +627,120 @@ class InnoDB_backup queue.emplace_back(lsn= next_lsn); } mutex.wr_unlock(); - ctx->max_first_lsn= lsn; - ctx->last_lsn= last_lsn; - ctx= nullptr; /* unsubscribe to checkpoint_complete() */ + ctx.max_first_lsn= lsn; + ctx.last_lsn= last_lsn; log_sys.latch.wr_unlock(); } /** - Finish copying or finalize the backup. + Enable SET GLOBAL innodb_log_archive, innodb_log_file_size. + @param thd current session + */ + ATTRIBUTE_NOINLINE void backup_stop(THD *thd) noexcept + { + ut_ad(!non_log); + ut_ad(ctx.state == PROCESSING); + ctx.state= CLEANUP; /* Unsubscribe from checkpoint_complete() */ + const uint64_t old_size{this->old_size}; + mutex.wr_unlock(); + /* Enable SET GLOBAL innodb_log_archive, innodb_log_file_size */ + log_sys.backup_stop(old_size, thd); + } + + /** + Abort the backup. @param thd current session - @param phase backup phase @param sink backup worker context - @return error code - @retval 0 on success + @return 0 (always) */ - int end(THD *thd, backup_phase phase, const backup_sink &sink) noexcept + ATTRIBUTE_COLD int abort(THD *thd, const backup_sink &sink) noexcept { - context *const ctx{static_cast(sink.ha_data)}; - if (!ctx /* InnoDB_backup::init() must have failed */ || - ctx->cleaned_up /* aborting after phase=BACKUP_PHASE_NO_COMMIT */) + if (!sink.ha_data) + /* init() must have failed; we have nothing to do */ return 0; - ctx->cleaned_up= true; - if (phase == BACKUP_PHASE_ABORT) - ctx->last_hardlink.store(LSN_MAX, std::memory_order_relaxed); - log_sys.latch.wr_lock(); - ut_ad(!this->ctx || this->ctx == ctx); - this->ctx= nullptr; /* fini() will delete the object */ + ut_ad(&ctx == static_cast(sink.ha_data)); + /* inform cleanup() that we will clean up */ + ctx.last_hardlink.store(LSN_MAX, std::memory_order_relaxed); + mutex.wr_lock(); ut_ad(!log_sys.resize_in_progress()); ut_ad(log_sys.archive); - int fail{0}; if (!old_size) { - mutex.wr_lock(); - queue.clear(); + /* + The server was running with innodb_log_archive=ON. + There is nothing to clean up other than the incomplete backup, + which the operator might want to check before deleting. + */ non_log= 0; - mutex.wr_unlock(); + queue.clear(); } else { - log_sys.latch.wr_unlock(); - sql_print_information("stop archiving: " LSN_PF, - log_sys.last_checkpoint_lsn.load()); - /* FIXME: execute this at a later stage, - after MDL_BACKUP_WAIT_COMMIT has been released! - This may wait several seconds for some page flushing! */ - fail= log_sys.backup_stop_archiving(thd); - sql_print_information("stopped archiving: " LSN_PF, - log_sys.last_checkpoint_lsn.load()); + ut_ad(ctx.state == PROCESSING); + mutex.wr_unlock(); + /* Restore innodb_log_archive=OFF as it was before init(). */ + std::ignore= log_sys.backup_stop_archiving(nullptr); log_sys.latch.wr_lock(); mutex.wr_lock(); delete_logs(); - mutex.wr_unlock(); + log_sys.latch.wr_unlock(); } - log_sys.backup_stop(old_size, thd); - return fail; + backup_stop(thd); + return 0; } /** - Clean up after end(). - @param target backup target + Clean up after releasing locks. + @param thd current session @param sink backup worker context @return error code @retval 0 on success */ - int fini(const backup_target &target, const backup_sink &sink) noexcept + void *finish_start(THD *thd, const backup_sink &sink) noexcept { - if (context *ctx{static_cast(sink.ha_data)}) + if (!sink.ha_data) + return nullptr; /* init() must have failed; nothing to do */ + void *ret{&ctx}; + ut_ad(ret == static_cast(sink.ha_data)); + mutex.wr_lock(); + ut_ad(!non_log); + if (ctx.state == CLEANUP) { - ut_ad(ctx != this->ctx); - int fail{ctx->cleanup(target, sink)}; - delete ctx; - return fail; + /* abort() must have been called already */ + ut_ad(queue.empty()); + mutex.wr_unlock(); + return &ctx; } - return 0; + + ut_ad(!log_sys.resize_in_progress()); + ut_ad(log_sys.archive); + + if (old_size) + { + ut_ad(ctx.state == PROCESSING); + /* Restore innodb_log_archive=OFF as it was before init() */ + mutex.wr_unlock(); + if (log_sys.backup_stop_archiving(thd)) + ret= reinterpret_cast(-1); + mutex.wr_lock(); + } + + backup_stop(thd); + return ret; + } + + /** + Clean up after finish_start(). + @param target backup target + @param sink backup worker context + @return error code + @retval 0 on success + */ + int finish_end(const backup_target &target, const backup_sink &sink) noexcept + { + ut_ad(!sink.ha_data || &ctx == static_cast(sink.ha_data)); + return sink.ha_data ? ctx.cleanup(target, sink) : 0; } /** @@ -689,11 +749,12 @@ class InnoDB_backup void checkpoint_complete() noexcept { ut_ad(log_sys.latch_have_wr()); - if (ctx) + if (ctx.state == PROCESSING) { const lsn_t lsn{log_sys.get_first_lsn() - log_sys.capacity()}; mutex.wr_lock(); - queue.emplace_back(lsn); + if (ctx.state == PROCESSING) + queue.emplace_back(lsn); mutex.wr_unlock(); } } @@ -1512,7 +1573,7 @@ bool log_t::backup_start(uint64_t *old_size, THD *thd) noexcept void log_t::backup_stop(uint64_t old_size, THD *thd) noexcept { - ut_ad(latch_have_wr()); + latch.wr_lock(); /* We will be invoked with old_size=0 after a failed backup_start(), or if innodb_log_archive=ON held during a successful backup_start(). */ ut_ad(!old_size || !resize_in_progress()); @@ -1549,6 +1610,8 @@ void *innodb_backup_start(THD *thd, const backup_target *, /* fall through */ default: return sink->ha_data; + case BACKUP_PHASE_FINISH: + return innodb_backup.finish_start(thd, *sink); } } @@ -1572,10 +1635,9 @@ int innodb_backup_end(THD *thd, const backup_target *target, default: return 0; case BACKUP_PHASE_FINISH: - return innodb_backup.fini(*target, *sink); - case BACKUP_PHASE_NO_COMMIT: + return innodb_backup.finish_end(*target, *sink); case BACKUP_PHASE_ABORT: - return innodb_backup.end(thd, phase, *sink); + return innodb_backup.abort(thd, *sink); } } From b80038d965fb652926ee424985c6d699d5d5e8de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Aug 2026 15:38:54 +0300 Subject: [PATCH 04/17] squash! 208ce0ae56e1ddd990555028b82aabd96909484f InnoDB_backup::commit(): Enqueue the remaining log. InnoDB_backup::checkpoint_complete(): If backup is running and commit() has not been called, add each completed innodb_archive_log=ON file to InnoDB_backup::queue. Else, skip or delete, as appropriate. --- storage/innobase/buf/buf0flu.cc | 3 +- storage/innobase/handler/backup_innodb.cc | 69 +++++++++++++---------- 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index df511cef99843..9e90697dd8c33 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -2159,7 +2159,9 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept resize_log.close(); SetFileAttributesA(get_archive_path(get_first_lsn() - capacity()).c_str(), FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_ARCHIVE); + innodb_backup_checkpoint(); #else + innodb_backup_checkpoint(); struct stat st; if (!fstat(resize_log.m_file, &st)) st.st_mode&= 0444; @@ -2170,7 +2172,6 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept get_archive_path(get_first_lsn() - capacity()).c_str(), errno); resize_log.close(); #endif - innodb_backup_checkpoint(); } else goto checkpoint_completed; diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index ba7a453a850ab..aafce206f8960 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -492,7 +492,6 @@ class InnoDB_backup return 0; } - ut_ad(ctx.state == PROCESSING); non_log-= size == non_log_files; id_limit= queue.back(); queue.pop_back(); @@ -511,6 +510,7 @@ class InnoDB_backup } else if (fil_space_t *space= fil_space_t::get(uint32_t(id_limit))) { + ut_ad(ctx.state == PROCESSING); ut_ad(phase == BACKUP_PHASE_START); int res= -1; uint32_t start{0}, limit{uint32_t(id_limit >> 32)}; @@ -612,24 +612,22 @@ class InnoDB_backup void commit() noexcept { log_sys.latch.wr_lock(); + mutex.wr_lock(); + ut_ad(!non_log); ut_ad(ctx.state == PROCESSING); ut_ad(ctx.last_lsn == LSN_MAX); + ut_ad(ctx.max_first_lsn == LSN_MAX); const lsn_t last_lsn{log_sys.get_lsn()}; lsn_t lsn{log_sys.get_first_lsn()}; - mutex.wr_lock(); - ut_ad(!non_log); - if (queue.empty() || queue.back() != lsn) - { - /* Schedule the remaining log for copying */ - queue.emplace_back(lsn); - const lsn_t next_lsn{lsn + log_sys.capacity()}; - if (next_lsn < last_lsn) - queue.emplace_back(lsn= next_lsn); - } - mutex.wr_unlock(); + /* Schedule the remaining log for copying */ + queue.emplace_back(lsn); + const lsn_t next_lsn{lsn + log_sys.capacity()}; + if (next_lsn < last_lsn) + queue.emplace_back(lsn= next_lsn); ctx.max_first_lsn= lsn; ctx.last_lsn= last_lsn; log_sys.latch.wr_unlock(); + mutex.wr_unlock(); } /** @@ -659,11 +657,15 @@ class InnoDB_backup /* init() must have failed; we have nothing to do */ return 0; ut_ad(&ctx == static_cast(sink.ha_data)); - /* inform cleanup() that we will clean up */ - ctx.last_hardlink.store(LSN_MAX, std::memory_order_relaxed); + log_sys.latch.wr_lock(); mutex.wr_lock(); + ut_ad(ctx.state == PROCESSING); ut_ad(!log_sys.resize_in_progress()); ut_ad(log_sys.archive); + + /* inform cleanup() that we will clean up */ + ctx.last_hardlink.store(LSN_MAX, std::memory_order_relaxed); + if (!old_size) { /* @@ -676,14 +678,12 @@ class InnoDB_backup } else { - ut_ad(ctx.state == PROCESSING); + log_sys.latch.wr_unlock(); mutex.wr_unlock(); - /* Restore innodb_log_archive=OFF as it was before init(). */ std::ignore= log_sys.backup_stop_archiving(nullptr); log_sys.latch.wr_lock(); mutex.wr_lock(); delete_logs(); - log_sys.latch.wr_unlock(); } backup_stop(thd); @@ -715,16 +715,18 @@ class InnoDB_backup ut_ad(!log_sys.resize_in_progress()); ut_ad(log_sys.archive); + ut_ad(ctx.state == PROCESSING); - if (old_size) - { - ut_ad(ctx.state == PROCESSING); - /* Restore innodb_log_archive=OFF as it was before init() */ - mutex.wr_unlock(); - if (log_sys.backup_stop_archiving(thd)) - ret= reinterpret_cast(-1); - mutex.wr_lock(); - } + const uint64_t old_size{this->old_size}; + mutex.wr_unlock(); + + /* Restore innodb_log_archive as it was before init() */ + if (old_size != 0 && log_sys.backup_stop_archiving(thd)) + ret= reinterpret_cast(-1); + + log_sys.latch.wr_lock(); + mutex.wr_lock(); + ut_ad(log_sys.get_first_lsn() >= ctx.max_first_lsn); backup_stop(thd); return ret; @@ -753,8 +755,17 @@ class InnoDB_backup { const lsn_t lsn{log_sys.get_first_lsn() - log_sys.capacity()}; mutex.wr_lock(); - if (ctx.state == PROCESSING) - queue.emplace_back(lsn); + if (ctx.state != PROCESSING); + else if (ctx.last_lsn == LSN_MAX) + queue.emplace_back(lsn); /* commit() was not invoked yet */ + else if (lsn > ctx.last_lsn && old_size) + /* + The server was running with innodb_log_archive=OFF, and this + log file covers some changes after the end of the backup. + Let us delete the file straight away, to keep step() and + delete_logs() simple. + */ + IF_WIN(DeleteFile,unlink)(log_sys.get_archive_path(lsn).c_str()); mutex.wr_unlock(); } } @@ -1573,7 +1584,7 @@ bool log_t::backup_start(uint64_t *old_size, THD *thd) noexcept void log_t::backup_stop(uint64_t old_size, THD *thd) noexcept { - latch.wr_lock(); + ut_ad(latch_have_wr()); /* We will be invoked with old_size=0 after a failed backup_start(), or if innodb_log_archive=ON held during a successful backup_start(). */ ut_ad(!old_size || !resize_in_progress()); From 1fd7ee821be6dccc21b5869aa382a2bba99850dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Aug 2026 14:14:46 +0300 Subject: [PATCH 05/17] squash! b625be27ca0a0b24730dd19d37d8bbf85917156c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aria_backup_end(): Removed. aria_backup_start(): A crude single-threaded implementation of copying files that do not belong to ACID storage engines. Most files are copied in BACKUP_PHASE_NO_DDL after flush_tables(thd, FLUSH_NON_TRANS_TABLES) has been invoked. All ENGINE=Aria files (including TRANSACTIONAL=0) are copied in BACKUP_PHASE_NO_COMMIT. Thanks to Andrzej Jarząbek for implementing test cases and suggesting this logic. --- .../backup_ddl_concurrent_verify.result | 53 +++++ .../backup/backup_ddl_concurrent_verify.test | 137 +++++++++++++ .../backup/backup_nonacid_flush_stress.result | 51 +++++ .../backup/backup_nonacid_flush_stress.test | 86 ++++++++ .../backup_sys_stats_not_flushed.result | 34 ++++ .../backup/backup_sys_stats_not_flushed.test | 81 ++++++++ sql/sql_backup.cc | 9 + sql/sql_base.cc | 2 +- sql/sql_base.h | 2 +- storage/maria/ha_maria.cc | 10 - storage/maria/ma_backup_server.c | 186 +++++++++--------- storage/maria/ma_backup_server.h | 30 --- 12 files changed, 542 insertions(+), 139 deletions(-) create mode 100644 mysql-test/suite/backup/backup_ddl_concurrent_verify.result create mode 100644 mysql-test/suite/backup/backup_ddl_concurrent_verify.test create mode 100644 mysql-test/suite/backup/backup_nonacid_flush_stress.result create mode 100644 mysql-test/suite/backup/backup_nonacid_flush_stress.test create mode 100644 mysql-test/suite/backup/backup_sys_stats_not_flushed.result create mode 100644 mysql-test/suite/backup/backup_sys_stats_not_flushed.test diff --git a/mysql-test/suite/backup/backup_ddl_concurrent_verify.result b/mysql-test/suite/backup/backup_ddl_concurrent_verify.result new file mode 100644 index 0000000000000..6c9f12392a430 --- /dev/null +++ b/mysql-test/suite/backup/backup_ddl_concurrent_verify.result @@ -0,0 +1,53 @@ +CREATE TABLE t_myisam (a INT) ENGINE=MyISAM; +INSERT INTO t_myisam VALUES (1), (2), (3); +CREATE TABLE t_aria (a INT) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO t_aria VALUES (1), (2), (3); +CREATE TABLE t_arch (a INT) ENGINE=ARCHIVE; +INSERT INTO t_arch VALUES (1), (2), (3); +CREATE PROCEDURE reader_churn(IN iterations INT) +BEGIN +DECLARE i INT DEFAULT 0; +WHILE i < iterations DO +SELECT SLEEP(0.002) INTO @discard +FROM t_myisam, t_aria, t_arch LIMIT 1; +SET i = i + 1; +END WHILE; +END| +CREATE PROCEDURE ddl_churn(IN iterations INT) +BEGIN +DECLARE i INT DEFAULT 0; +WHILE i < iterations DO +ALTER TABLE t_myisam ADD INDEX ix (a); +ALTER TABLE t_myisam DROP INDEX ix; +ALTER TABLE t_myisam FORCE, ALGORITHM=COPY; +SET i = i + 1; +END WHILE; +END| +connect r1,localhost,root,,; +CALL reader_churn(800); +connect r2,localhost,root,,; +CALL reader_churn(800); +connect d1,localhost,root,,; +CALL ddl_churn(400); +connection default; +connection r1; +connection r2; +connection d1; +connection default; +disconnect r1; +disconnect r2; +disconnect d1; +# restart: --datadir=MYSQLTEST_VARDIR/backup_ddl_concurrent_verify +SELECT COUNT(*) FROM t_myisam; +COUNT(*) +3 +SELECT COUNT(*) FROM t_aria; +COUNT(*) +3 +SELECT COUNT(*) FROM t_arch; +COUNT(*) +3 +# restart +DROP PROCEDURE reader_churn; +DROP PROCEDURE ddl_churn; +DROP TABLE t_myisam, t_aria, t_arch; diff --git a/mysql-test/suite/backup/backup_ddl_concurrent_verify.test b/mysql-test/suite/backup/backup_ddl_concurrent_verify.test new file mode 100644 index 0000000000000..b9ce93ff1147d --- /dev/null +++ b/mysql-test/suite/backup/backup_ddl_concurrent_verify.test @@ -0,0 +1,137 @@ +--source include/have_archive.inc +--source include/have_aria.inc + +# +# Verify a backup that nominally succeeded while DDL was running. +# +# BACKUP SERVER can fail with ER_LOCK_DEADLOCK next to concurrent DDL, which is +# what backup_ddl_deadlock reproduces. Because of that, the other DDL tests +# never get as far as checking the backup they produced. This one tolerates the +# deadlock and retries until a backup completes, so that a backup taken while +# DDL was in progress is actually read back. +# +# Two things could make such a backup wrong: +# +# - flush_tables() skips a table it cannot lock. The deadlock is reported, but +# a plain lock wait timeout is swallowed by its error handler, and the +# fallback that opens an instance of its own asks for the metadata lock with +# a zero timeout. A DDL merely waiting for an exclusive lock is enough to +# make that fail, and in that state the table has not been closed yet, so it +# can be copied while still marked open. +# +# - MDL_BACKUP_ALTER_COPY is not blocked by any backup phase, so an +# ALTER TABLE ... ALGORITHM=COPY keeps writing its #sql- intermediate files +# throughout BACKUP_PHASE_NO_DDL. copy_misc_files() selects files by +# extension only and does not exclude them. +# + +CREATE TABLE t_myisam (a INT) ENGINE=MyISAM; +INSERT INTO t_myisam VALUES (1), (2), (3); + +CREATE TABLE t_aria (a INT) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO t_aria VALUES (1), (2), (3); + +CREATE TABLE t_arch (a INT) ENGINE=ARCHIVE; +INSERT INTO t_arch VALUES (1), (2), (3); + +DELIMITER |; +# Keeps the tables open, so that flush_tables() has no free TABLE instance and +# has to take a metadata lock of its own. +CREATE PROCEDURE reader_churn(IN iterations INT) +BEGIN + DECLARE i INT DEFAULT 0; + WHILE i < iterations DO + SELECT SLEEP(0.002) INTO @discard + FROM t_myisam, t_aria, t_arch LIMIT 1; + SET i = i + 1; + END WHILE; +END| + +# ADD/DROP INDEX rewrites the index file in place; FORCE with ALGORITHM=COPY +# rebuilds through a #sql- intermediate table. Row counts are unaffected. +CREATE PROCEDURE ddl_churn(IN iterations INT) +BEGIN + DECLARE i INT DEFAULT 0; + WHILE i < iterations DO + ALTER TABLE t_myisam ADD INDEX ix (a); + ALTER TABLE t_myisam DROP INDEX ix; + ALTER TABLE t_myisam FORCE, ALGORITHM=COPY; + SET i = i + 1; + END WHILE; +END| +DELIMITER ;| + +--let $target_directory=$MYSQLTEST_VARDIR/backup_ddl_concurrent_verify +--error 0,1 +--rmdir $target_directory + +--connect (r1,localhost,root,,) +--send CALL reader_churn(800) +--connect (r2,localhost,root,,) +--send CALL reader_churn(800) +--connect (d1,localhost,root,,) +--send CALL ddl_churn(400) + +--connection default + +# Do not start before the DDL is actually running, so that the backup that is +# kept is one that overlapped it. +--let $wait_condition= SELECT COUNT(*) FROM information_schema.processlist WHERE info LIKE 'ALTER TABLE t_myisam%' +--source include/wait_condition.inc + +# Retry until one backup gets through, keeping that one. +--let $attempts= 40 +--let $have_backup= 0 +--disable_query_log +while ($attempts) +{ + --error 0,1 + --rmdir $target_directory + --error 0,ER_LOCK_DEADLOCK + --eval BACKUP SERVER TO '$target_directory' + if ($mysql_errno == 0) + { + --let $have_backup= 1 + --let $attempts= 1 + } + dec $attempts; +} +--enable_query_log + +--connection r1 +--reap +--connection r2 +--reap +--connection d1 +--reap + +--connection default +--disconnect r1 +--disconnect r2 +--disconnect d1 + +if (!$have_backup) +{ + --skip every BACKUP SERVER attempt hit ER_LOCK_DEADLOCK +} + +# An ALTER TABLE ... ALGORITHM=COPY that was running during the backup must not +# have left its intermediate table in it. +--list_files $target_directory/test #sql* + +# The tables have to come back without needing repair, so any message about a +# table being crashed or not closed properly fails the test here. +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +SELECT COUNT(*) FROM t_myisam; +SELECT COUNT(*) FROM t_aria; +SELECT COUNT(*) FROM t_arch; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +DROP PROCEDURE reader_churn; +DROP PROCEDURE ddl_churn; +DROP TABLE t_myisam, t_aria, t_arch; +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_nonacid_flush_stress.result b/mysql-test/suite/backup/backup_nonacid_flush_stress.result new file mode 100644 index 0000000000000..de15304712338 --- /dev/null +++ b/mysql-test/suite/backup/backup_nonacid_flush_stress.result @@ -0,0 +1,51 @@ +CREATE TABLE t_myisam (a INT) ENGINE=MyISAM; +INSERT INTO t_myisam VALUES (1), (2), (3); +CREATE TABLE t_aria (a INT) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO t_aria VALUES (1), (2), (3); +CREATE TABLE t_arch (a INT) ENGINE=ARCHIVE; +INSERT INTO t_arch VALUES (1), (2), (3); +CREATE PROCEDURE churn(IN i INT) +BEGIN +WHILE i > 0 DO +SELECT SLEEP(0.002) INTO @discard +FROM t_myisam, t_aria, t_arch LIMIT 1; +SET i = i - 1; +END WHILE; +END| +connect r1,localhost,root,,; +CALL churn(400); +connect r2,localhost,root,,; +CALL churn(400); +connect r3,localhost,root,,; +CALL churn(400); +connect r4,localhost,root,,; +CALL churn(400); +connection default; +connection r1; +connection r2; +connection r3; +connection r4; +connection default; +disconnect r1; +disconnect r2; +disconnect r3; +disconnect r4; +# restart: --datadir=MYSQLTEST_VARDIR/backup_nonacid_purge_stress +SELECT * FROM t_myisam ORDER BY a; +a +1 +2 +3 +SELECT * FROM t_aria ORDER BY a; +a +1 +2 +3 +SELECT * FROM t_arch ORDER BY a; +a +1 +2 +3 +# restart +DROP PROCEDURE churn; +DROP TABLE t_myisam, t_aria, t_arch; diff --git a/mysql-test/suite/backup/backup_nonacid_flush_stress.test b/mysql-test/suite/backup/backup_nonacid_flush_stress.test new file mode 100644 index 0000000000000..ecf258785cfcf --- /dev/null +++ b/mysql-test/suite/backup/backup_nonacid_flush_stress.test @@ -0,0 +1,86 @@ +--source include/have_archive.inc +--source include/have_aria.inc + +# +# Stress test: several readers open and close tables as fast as +# they can while backups run back to back. This exercises the arrival of a +# reader at every point of the flush. + +CREATE TABLE t_myisam (a INT) ENGINE=MyISAM; +INSERT INTO t_myisam VALUES (1), (2), (3); + +CREATE TABLE t_aria (a INT) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO t_aria VALUES (1), (2), (3); + +CREATE TABLE t_arch (a INT) ENGINE=ARCHIVE; +INSERT INTO t_arch VALUES (1), (2), (3); + +# Each iteration opens all three tables, holds them for a few milliseconds and +# closes them again, so the readers cycle through open and closed states many +# times over the course of a backup. +DELIMITER |; +CREATE PROCEDURE churn(IN i INT) +BEGIN + WHILE i > 0 DO + SELECT SLEEP(0.002) INTO @discard + FROM t_myisam, t_aria, t_arch LIMIT 1; + SET i = i - 1; + END WHILE; +END| +DELIMITER ;| + +--let $target_directory=$MYSQLTEST_VARDIR/backup_nonacid_purge_stress +--error 0,1 +--rmdir $target_directory + +--connect (r1,localhost,root,,) +--send CALL churn(400) +--connect (r2,localhost,root,,) +--send CALL churn(400) +--connect (r3,localhost,root,,) +--send CALL churn(400) +--connect (r4,localhost,root,,) +--send CALL churn(400) + +--connection default +--let $i= 6 +while ($i) +{ + --error 0,1 + --rmdir $target_directory + --disable_query_log + --eval BACKUP SERVER TO '$target_directory' + --enable_query_log + dec $i; +} + +--connection r1 +--reap +--connection r2 +--reap +--connection r3 +--reap +--connection r4 +--reap + +--connection default +--disconnect r1 +--disconnect r2 +--disconnect r3 +--disconnect r4 + +# The last backup was taken with the readers still running, so it is only +# complete if every table was closed before its files were copied. +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +SELECT * FROM t_myisam ORDER BY a; +SELECT * FROM t_aria ORDER BY a; +SELECT * FROM t_arch ORDER BY a; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +DROP PROCEDURE churn; +DROP TABLE t_myisam, t_aria, t_arch; +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_sys_stats_not_flushed.result b/mysql-test/suite/backup/backup_sys_stats_not_flushed.result new file mode 100644 index 0000000000000..43d1297c9d4de --- /dev/null +++ b/mysql-test/suite/backup/backup_sys_stats_not_flushed.result @@ -0,0 +1,34 @@ +CREATE TABLE t1 (a INT, KEY(a)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1), (2), (3); +CREATE PROCEDURE fill_stats(IN rows_wanted INT) +BEGIN +DECLARE i INT DEFAULT 0; +WHILE i < rows_wanted DO +INSERT INTO mysql.table_stats (db_name, table_name, cardinality) +VALUES ('test', CONCAT('tbl', i), i); +SET i = i + 1; +END WHILE; +END| +CALL fill_stats(600); +connect backup,localhost,root,,; +SET DEBUG_SYNC='after_purge_tables SIGNAL purged WAIT_FOR go'; +BACKUP SERVER TO 'target_directory'; +connection default; +SET DEBUG_SYNC='now WAIT_FOR purged'; +connect writer,localhost,root,,; +DELETE FROM mysql.table_stats WHERE SLEEP(0.005) = 0; +connection default; +SET DEBUG_SYNC='now SIGNAL go'; +connection writer; +connection backup; +connection default; +disconnect backup; +disconnect writer; +SET DEBUG_SYNC='RESET'; +# restart: --datadir=MYSQLTEST_VARDIR/backup_sys_stats_not_flushed +CHECK TABLE mysql.table_stats; +Table Op Msg_type Msg_text +mysql.table_stats check status OK +# restart +DROP PROCEDURE fill_stats; +DROP TABLE t1; diff --git a/mysql-test/suite/backup/backup_sys_stats_not_flushed.test b/mysql-test/suite/backup/backup_sys_stats_not_flushed.test new file mode 100644 index 0000000000000..9e701ede81fda --- /dev/null +++ b/mysql-test/suite/backup/backup_sys_stats_not_flushed.test @@ -0,0 +1,81 @@ +--source include/have_aria.inc +--source include/have_debug_sync.inc + +# +# Test that non-transactional Aria system tables (stats tables) get copied +# correctly by BACKUP SERVER. The risk point is that as opposed to user tables +# writes to these tables are not blocked in the "no DDL" phase, so they need +# to be flushed and copied in the "no commit" phase. +# BACKUP SERVER succeeds, but a statistics table in the backup was copied while +# it was being written, even though the phase begins by flushing it. +# +# A scenario where mysql.table_stats is written in BACKUP_PHASE_NO_DDL: +# DELETE asks for MDL_SHARED_WRITE, which is below the MDL_SHARED_UPGRADABLE +# threshold in lock_table_names(), so it does not take that function's +# statement-wide MDL_BACKUP_DDL, and the per-table lock its category earns it, +# MDL_BACKUP_SYS_DML, does not conflict with the backup lock until +# MDL_BACKUP_WAIT_COMMIT. +# + +CREATE TABLE t1 (a INT, KEY(a)) ENGINE=MyISAM; +INSERT INTO t1 VALUES (1), (2), (3); + +# Enough rows that the DELETE keeps writing for the whole of the phase. The +# rows have to go in one at a time: a system table cannot be write-locked +# together with another table, so INSERT ... SELECT is rejected. +DELIMITER |; +CREATE PROCEDURE fill_stats(IN rows_wanted INT) +BEGIN + DECLARE i INT DEFAULT 0; + WHILE i < rows_wanted DO + INSERT INTO mysql.table_stats (db_name, table_name, cardinality) + VALUES ('test', CONCAT('tbl', i), i); + SET i = i + 1; + END WHILE; +END| +DELIMITER ;| +CALL fill_stats(600); + +--let $target_directory=$MYSQLTEST_VARDIR/backup_sys_stats_not_flushed +--error 0,1 +--rmdir $target_directory + +--connect (backup,localhost,root,,) +SET DEBUG_SYNC='after_purge_tables SIGNAL purged WAIT_FOR go'; +--replace_result $target_directory target_directory +--send_eval BACKUP SERVER TO '$target_directory' + +--connection default +SET DEBUG_SYNC='now WAIT_FOR purged'; + +--connect (writer,localhost,root,,) +--send DELETE FROM mysql.table_stats WHERE SLEEP(0.005) = 0 + +--connection default +--let $wait_condition= SELECT COUNT(*) FROM information_schema.processlist WHERE state = 'User sleep' AND info LIKE 'DELETE FROM mysql.table_stats%' +--source include/wait_condition.inc + +SET DEBUG_SYNC='now SIGNAL go'; + +--connection writer +--reap +--connection backup +--reap + +--connection default +--disconnect backup +--disconnect writer +SET DEBUG_SYNC='RESET'; + +# The statistics table in the backup has to be usable without repair. +--let $restart_parameters=--datadir=$target_directory +--source include/restart_mysqld.inc + +CHECK TABLE mysql.table_stats; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +DROP PROCEDURE fill_stats; +DROP TABLE t1; +--rmdir $target_directory diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc index 92b7473cc8a9b..04f7bb762648d 100644 --- a/sql/sql_backup.cc +++ b/sql/sql_backup.cc @@ -17,6 +17,7 @@ #include "mdl.h" #include "mysys_err.h" #include "sql_class.h" +#include "sql_base.h" #include "sql_backup.h" #include "sql_backup_interface.h" #include "sql_parse.h" @@ -717,6 +718,14 @@ static bool backup_execute(THD *thd, const char *target, const char *command, if (fail) break; } + + if ((phase == BACKUP_PHASE_NO_DDL || phase == BACKUP_PHASE_NO_COMMIT) && + /* Invoke handler::extra(HA_EXTRA_FLUSH) */ + (fail= flush_tables(thd, phase == BACKUP_PHASE_NO_DDL + ? FLUSH_NON_TRANS_TABLES + : FLUSH_SYS_TABLES))) + break; + backup_phase_start: target_phase->phase= backup_phase(phase); fail= plugin_foreach_with_mask(thd, backup_start, diff --git a/sql/sql_base.cc b/sql/sql_base.cc index c29fae7a9b396..bb1f815eea5c1 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -321,7 +321,7 @@ OPEN_TABLE_LIST *list_open_tables(THD *thd, Close all tables that are not in use in table definition cache */ -extern "C" void purge_tables() +void purge_tables() { /* Force close of all open tables. diff --git a/sql/sql_base.h b/sql/sql_base.h index 0fb4e8cdf00f9..77d2b17a1fe5d 100644 --- a/sql/sql_base.h +++ b/sql/sql_base.h @@ -320,7 +320,7 @@ void close_log_table(THD *thd, Open_tables_backup *backup); bool close_cached_tables(THD *thd, TABLE_LIST *tables, bool wait_for_refresh, ulong timeout); -extern "C" void purge_tables(); +void purge_tables(); bool flush_tables(THD *thd, flush_tables_type flag); void close_all_tables_for_name(THD *thd, TABLE_SHARE *share, ha_extra_function extra, diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 736ef1e9c49f2..b3c3afe13e416 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -3737,14 +3737,6 @@ aria_backup_start_wrap(THD *thd, const backup_target *target, return aria_backup_start(thd, target, phase, sink); } -/* A C++ function pointer compatible wrapper of a C function */ -static int -aria_backup_end_wrap(THD *thd, const backup_target *target, - backup_phase phase, const backup_sink *sink) -{ - return aria_backup_end(thd, target, phase, sink); -} - #define SHOW_MSG_LEN (FN_REFLEN + 20) /** @brief show status handler @@ -3958,8 +3950,6 @@ static int ha_maria_init(void *p) maria_hton->end_backup= maria_end_backup; maria_hton->update_optimizer_costs= aria_update_optimizer_costs; maria_hton->backup_start= aria_backup_start_wrap; - //maria_hton->backup_step= aria_backup_step_wrap; - maria_hton->backup_end= aria_backup_end_wrap; /* TODO: decide if we support Maria being used for log tables */ maria_hton->flags= (HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES | diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c index cef73bcf8b04d..98478b1eda079 100644 --- a/storage/maria/ma_backup_server.c +++ b/storage/maria/ma_backup_server.c @@ -32,42 +32,31 @@ ATTRIBUTE_COLD ATTRIBUTE_NOINLINE static int dir_error(const char *name) return 1; } +typedef int (*name_predicate)(const char *file_name, size_t len); + /** Determine if a file may be backed up. @param file_name candidate file name + @param len strlen(file_name) @retval FALSE if the file must be excluded @retval TRUE if the file may be included */ -static int is_db_file(const char *file_name) +static int is_db_file(const char *file_name, size_t len) { - size_t len= strlen(file_name); uint32_t suffix; - if (len < 4) - return FALSE; - if (!memcmp(file_name, tmp_file_prefix, tmp_file_prefix_length)) - /* - As noted in MDEV-25854, file names that start with #sql - must be excluded from the backup. For example, a call to - MDL_context::upgrade_shared_lock() in - mysql_inplace_alter_table() could time out, resulting in - cleanup_table_after_inplace_alter() deleting a - #sql-alter*.frm file before we get a chance to copy it. - */ - return FALSE; + DBUG_ASSERT(len >= 4); memcpy(&suffix, file_name + len - 4, 4); switch (suffix) { - default: - return len == 6 && !memcmp(file_name, C_STRING_WITH_LEN("db.opt")); #ifdef WORDS_BIGENDIAN case 0x2e41524d: /* .ARM ENGINE=ARCHIVE metadata */ case 0x2e41525a: /* .ARZ ENGINE=ARCHIVE compressed data */ case 0x2e43534d: /* .CSM ENGINE=CSV metadata */ case 0x2e435356: /* .CSV ENGINE=CSV data ("comma separated values") */ - case 0x2e4d4144: /* .MAD ENGINE=Aria data heap */ - case 0x2e4d4149: /* .MAI ENGINE=Aria indexes */ case 0x2e4d5247: /* .MRG ENGINE=MRG_MyISAM */ case 0x2e4d5944: /* .MYD ENGINE=MyISAM data heap */ case 0x2e4d5949: /* .MYI ENGINE=MyISAM indexes */ + case 0x2e545247: /* .TRG trigger definition */ + case 0x2e54524e: /* .TRN trigger name */ case 0x2e66726d: /* .frm form (SHOW CREATE TABLE) */ case 0x2e706172: /* .par PARTITION metadata */ #else @@ -75,16 +64,42 @@ static int is_db_file(const char *file_name) case 0x5a52412e: /* .ARZ ENGINE=ARCHIVE compressed data */ case 0x4d53432e: /* .CSM ENGINE=CSV metadata */ case 0x5653432e: /* .CSV ENGINE=CSV data ("comma separated values") */ - case 0x44414d2e: /* .MAD ENGINE=Aria data heap */ - case 0x49414d2e: /* .MAI ENGINE=Aria indexes */ case 0x47524d2e: /* .MRG ENGINE=MRG_MyISAM */ case 0x44594d2e: /* .MYD ENGINE=MyISAM data heap */ case 0x49594d2e: /* .MYI ENGINE=MyISAM indexes */ + case 0x4752542e: /* .TRG trigger definition */ + case 0x4e52542e: /* .TRN trigger name */ case 0x6d72662e: /* .frm form (SHOW CREATE TABLE) */ case 0x7261702e: /* .par PARTITION metadata */ #endif return TRUE; } + return len == 6 && !memcmp(file_name, C_STRING_WITH_LEN("db.opt")); +} + +/** + Determine if a file is an ENGINE=Aria file. + @param file_name candidate file name + @param len strlen(file_name) + @retval FALSE if the file must be excluded + @retval TRUE if the file may be included +*/ +static int is_ma_file(const char *file_name, size_t len) +{ + uint32_t suffix; + DBUG_ASSERT(len >= 4); + memcpy(&suffix, file_name + len - 4, 4); + switch (suffix) { +#ifdef WORDS_BIGENDIAN + case 0x2e4d4144: /* .MAD ENGINE=Aria data heap */ + case 0x2e4d4149: /* .MAI ENGINE=Aria indexes */ +#else + case 0x44414d2e: /* .MAD ENGINE=Aria data heap */ + case 0x49414d2e: /* .MAI ENGINE=Aria indexes */ +#endif + return TRUE; + } + return FALSE; } struct Aria_backup_entry @@ -95,25 +110,6 @@ struct Aria_backup_entry const char *name; }; -/** Backup state */ -struct Aria_backup -{ - /** whether translog_disable_purge() is in effect */ - int translog_purge_disabled; -}; - -static void aria_backup_init(struct Aria_backup *ab) -{ - ab->translog_purge_disabled= TRUE; - translog_disable_purge(); -} - -static void aria_backup_destroy(const struct Aria_backup *ab) -{ - if (ab && ab->translog_purge_disabled) - translog_enable_purge(); -} - /* Create directory in the target directory if it does not exist. Return 0 on success, non-0 on failure. Set errno in case of failure @@ -253,6 +249,7 @@ static int aria_backup_file(const struct backup_target *target, static int aria_backup_dir(const struct backup_target *target, const struct backup_sink *sink, + name_predicate include_name, const char *dir_name, size_t prefix) { int fail= 0; @@ -274,8 +271,19 @@ static int aria_backup_dir(const struct backup_target *target, { const struct fileinfo *fi= dir->dir_entry; const struct fileinfo *const end= fi + dir->number_of_files; + size_t len; for (; fi < end; fi++) - if (is_db_file(fi->name)) + if ((len= strlen(fi->name)) >= 4 && + /* + As noted in MDEV-25854, file names that start with #sql + must be excluded from the backup. For example, a call to + MDL_context::upgrade_shared_lock() in + mysql_inplace_alter_table() could time out, resulting in + cleanup_table_after_inplace_alter() deleting a + #sql-alter*.frm file before we get a chance to copy it. + */ + memcmp(fi->name, tmp_file_prefix, tmp_file_prefix_length) && + (*include_name)(fi->name, len)) if ((fail= aria_backup_file(target, sink, path, fi->name, prefix, 0)) != 0) break; @@ -285,8 +293,9 @@ static int aria_backup_dir(const struct backup_target *target, } } -static int aria_backup_scan(const struct backup_target *target, - const struct backup_sink *sink) +static int aria_backup_data(const struct backup_target *target, + const struct backup_sink *sink, + name_predicate include_name) { int fail= 0; size_t prefix= strlen(mysql_real_data_home) + 1; @@ -301,34 +310,38 @@ static int aria_backup_scan(const struct backup_target *target, for (; fi < end; fi++) { if ((fi->mystat->st_mode & S_IFMT) == S_IFDIR) - if ((fail= aria_backup_dir(target, sink, fi->name, prefix)) != 0) + if ((fail= aria_backup_dir(target, sink, include_name, + fi->name, prefix)) != 0) break; } my_dirend(dir); } - if (fail) - return fail; - /* Process the Aria logs. */ - prefix= strlen(maria_data_root) + 1; - fail= aria_backup_file(target, sink, maria_data_root, "aria_log_control", - prefix, 1); - if (fail) - return fail; - translog_flush(translog_get_horizon()); - dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)); - if (!dir) - return dir_error(maria_data_root); - else + return fail; +} + +static int aria_backup_logs(const struct backup_target *target, + const struct backup_sink *sink) +{ + size_t prefix= strlen(maria_data_root) + 1; + int fail= aria_backup_file(target, sink, maria_data_root, "aria_log_control", + prefix, 1); + if (!fail) { - const struct fileinfo *fi= dir->dir_entry; - const struct fileinfo *const end= fi + dir->number_of_files; - for (; fi < end; fi++) - if ((fail= - !strncmp(fi->name, C_STRING_WITH_LEN("aria_log.")) && - aria_backup_file(target, sink, maria_data_root, fi->name, - prefix, 0)) != 0) - break; - my_dirend(dir); + MY_DIR *dir; + translog_flush(translog_get_horizon()); + dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)); + if (dir) + { + const struct fileinfo *fi= dir->dir_entry; + const struct fileinfo *const end= fi + dir->number_of_files; + for (; fi < end; fi++) + if ((fail= + !strncmp(fi->name, C_STRING_WITH_LEN("aria_log.")) && + aria_backup_file(target, sink, maria_data_root, fi->name, + prefix, 0)) != 0) + break; + my_dirend(dir); + } } return fail; } @@ -337,43 +350,22 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, enum backup_phase phase, const struct backup_sink *sink) { + DBUG_ASSERT((phase == BACKUP_PHASE_PREPARE_START) ? !sink : !sink->ha_data); switch (phase) { - struct Aria_backup *aria_backup; - case BACKUP_PHASE_PREPARE_START: - return 0; - default: - return sink->ha_data; - case BACKUP_PHASE_NO_COMMIT: - assert(!sink->ha_data); - aria_backup= calloc(1, sizeof *aria_backup); - if (!aria_backup) + int ret; + case BACKUP_PHASE_NO_DDL: + if (aria_backup_data(target, sink, is_db_file)) return (void*) -1; - aria_backup_init(aria_backup); - return aria_backup; - } -} - -int aria_backup_end(THD *thd, const struct backup_target *target, - enum backup_phase phase, const struct backup_sink *sink) -{ - struct Aria_backup *aria_backup= sink->ha_data; - int ret= 0; - switch (phase) { - extern void purge_tables(void); + break; case BACKUP_PHASE_NO_COMMIT: - assert(aria_backup); - assert(aria_backup->translog_purge_disabled); - aria_backup->translog_purge_disabled= FALSE; - purge_tables(); // TODO: do not close transactional tables - ret= aria_backup_scan(target, sink); + translog_disable_purge(); + ret= aria_backup_data(target, sink, is_ma_file) || + aria_backup_logs(target, sink); translog_enable_purge(); - break; - case BACKUP_PHASE_FINISH: - aria_backup_destroy(aria_backup); - free(aria_backup); - break; + if (ret) + return (void*) -1; default: break; } - return ret; + return NULL; } diff --git a/storage/maria/ma_backup_server.h b/storage/maria/ma_backup_server.h index e021833e9476a..a509699f9f645 100644 --- a/storage/maria/ma_backup_server.h +++ b/storage/maria/ma_backup_server.h @@ -33,33 +33,3 @@ extern "C" void *aria_backup_start(THD *thd, const struct backup_target *target, enum backup_phase phase, const struct backup_sink *sink); - -#ifdef __cplusplus -extern "C" -#endif -/** - Process a file that was collected in aria_backup_start(). - @param thd current session - @param target backup target - @param phase last phase on which backup_start() was successfully invoked - @param sink worker context - @retval 0 on completion -*/ -int aria_backup_step(THD *thd, const struct backup_target *target, - enum backup_phase phase, const struct backup_sink *sink); - -#ifdef __cplusplus -extern "C" -#endif -/** - Finish a phase, once all calls for the current phase are completed. - @param thd current session - @param target backup target - @param phase last phase on which backup_start() was successfully invoked, - or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH - @param sink worker context - @return error code - @retval 0 on success -*/ -int aria_backup_end(THD *thd, const struct backup_target *target, - enum backup_phase phase, const struct backup_sink *sink); From 4e71fc51f26ecab8ed556b352fa492ea343429d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Aug 2026 15:14:01 +0300 Subject: [PATCH 06/17] fixup! b625be27ca0a0b24730dd19d37d8bbf85917156c --- storage/innobase/handler/backup_innodb.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc index aafce206f8960..af9d477014196 100644 --- a/storage/innobase/handler/backup_innodb.cc +++ b/storage/innobase/handler/backup_innodb.cc @@ -628,6 +628,12 @@ class InnoDB_backup ctx.last_lsn= last_lsn; log_sys.latch.wr_unlock(); mutex.wr_unlock(); + /* + Ensure that all data will be available to replicate(). Some + might only reside in log_sys.buf. A durable write is not + necessary, because a system crash will make the backup unusable. + */ + log_write_up_to(last_lsn, false); } /** From e84155a7958e9bbdf1ffde788523d41a99a17d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 27 Aug 2026 17:18:09 +0300 Subject: [PATCH 07/17] squash! 1fd7ee821be6dccc21b5869aa382a2bba99850dd Implement multi-threaded non-InnoDB backup. struct Aria_backup: Context for multi-threaded backup. aria_backup_start(): Prepare the context for aria_backup_step(). aria_backup_step(): Copy one file. aria_backup_data(): Copy one data file. We assume that the current working directory is the datadir, which holds for MariaDB Server but not the Embedded Server library. aria_backup_log(): Copy one log file. aria_backup_end(): Finish a copying phase and clean up the context. --- sql/mysqld.cc | 6 +- sql/mysqld.h | 2 +- storage/maria/ha_maria.cc | 18 ++ storage/maria/ma_backup_server.c | 443 +++++++++++++++++++++++-------- storage/maria/ma_backup_server.h | 34 ++- 5 files changed, 380 insertions(+), 123 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index f4f1cc97568ad..696353cefaa26 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -621,10 +621,8 @@ time_t server_start_time; char mysql_home[FN_REFLEN], pidfile_name[FN_REFLEN], system_time_zone[30]; char *default_tz_name, *opt_path; char log_error_file[FN_REFLEN], glob_hostname[FN_REFLEN], *opt_log_basename; -extern "C" { - char mysql_real_data_home[FN_REFLEN]; -} -char lc_messages_dir[FN_REFLEN], reg_ext[FN_EXTLEN], +char mysql_real_data_home[FN_REFLEN], + lc_messages_dir[FN_REFLEN], reg_ext[FN_EXTLEN], mysql_charsets_dir[FN_REFLEN], *opt_init_file, *opt_tc_log_file, *opt_ddl_recovery_file; char *lc_messages_dir_ptr= lc_messages_dir, *log_error_file_ptr; diff --git a/sql/mysqld.h b/sql/mysqld.h index 59c03d4ee5bdf..66745327af711 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -703,7 +703,7 @@ extern const char *mysql_real_data_home_ptr; extern ulong thread_handling; extern "C" MYSQL_PLUGIN_IMPORT char server_version[SERVER_VERSION_LENGTH]; extern char *server_version_ptr; -extern "C" MYSQL_PLUGIN_IMPORT char mysql_real_data_home[]; +extern MYSQL_PLUGIN_IMPORT char mysql_real_data_home[]; extern char mysql_unpacked_real_data_home[]; extern MYSQL_PLUGIN_IMPORT struct system_variables global_system_variables; extern char *my_proxy_protocol_networks; diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index b3c3afe13e416..558fc83d8c3f0 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -3737,6 +3737,22 @@ aria_backup_start_wrap(THD *thd, const backup_target *target, return aria_backup_start(thd, target, phase, sink); } +/* A C++ function pointer compatible wrapper of a C function */ +static int +aria_backup_step_wrap(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) +{ + return aria_backup_step(thd, target, phase, sink); +} + +/* A C++ function pointer compatible wrapper of a C function */ +static int +aria_backup_end_wrap(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) +{ + return aria_backup_end(thd, target, phase, sink); +} + #define SHOW_MSG_LEN (FN_REFLEN + 20) /** @brief show status handler @@ -3950,6 +3966,8 @@ static int ha_maria_init(void *p) maria_hton->end_backup= maria_end_backup; maria_hton->update_optimizer_costs= aria_update_optimizer_costs; maria_hton->backup_start= aria_backup_start_wrap; + maria_hton->backup_step= aria_backup_step_wrap; + maria_hton->backup_end= aria_backup_end_wrap; /* TODO: decide if we support Maria being used for log tables */ maria_hton->flags= (HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES | diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c index 98478b1eda079..d1114e3ef9cca 100644 --- a/storage/maria/ma_backup_server.c +++ b/storage/maria/ma_backup_server.c @@ -22,20 +22,24 @@ # define tmp_file_prefix_length 4 #endif -#if 1 /* can't #include "sql/mysqld.h" because it is C++ */ -extern char mysql_real_data_home[]; -#endif - -ATTRIBUTE_COLD ATTRIBUTE_NOINLINE static int dir_error(const char *name) +/** + Report that a directory cannot be read. + @param name directory name +*/ +ATTRIBUTE_COLD ATTRIBUTE_NOINLINE static void dir_error(const char *name) { my_error(ER_CANT_READ_DIR, MYF(0), name, my_errno); - return 1; } +/** + Predicate for checking if a file must be included in the backup. + @param file_name file base name to check + @param len strlen(file_name) +*/ typedef int (*name_predicate)(const char *file_name, size_t len); /** - Determine if a file may be backed up. + Determine if a non-Aria file may be backed up. @param file_name candidate file name @param len strlen(file_name) @retval FALSE if the file must be excluded @@ -44,7 +48,7 @@ typedef int (*name_predicate)(const char *file_name, size_t len); static int is_db_file(const char *file_name, size_t len) { uint32_t suffix; - DBUG_ASSERT(len >= 4); + assert(len >= 4); memcpy(&suffix, file_name + len - 4, 4); switch (suffix) { #ifdef WORDS_BIGENDIAN @@ -87,7 +91,7 @@ static int is_db_file(const char *file_name, size_t len) static int is_ma_file(const char *file_name, size_t len) { uint32_t suffix; - DBUG_ASSERT(len >= 4); + assert(len >= 4); memcpy(&suffix, file_name + len - 4, 4); switch (suffix) { #ifdef WORDS_BIGENDIAN @@ -102,17 +106,32 @@ static int is_ma_file(const char *file_name, size_t len) return FALSE; } -struct Aria_backup_entry +/** Backup status */ +enum Aria_backup_status { BACKUP_OK, BACKUP_FAIL, TRANSLOG_PURGE_DISABLED }; + +/** Backup state */ +struct Aria_backup { - /** directory name */ - const char *dir; - /** file name relative to the directory */ - const char *name; + /** mutex protecting the data in concurrent aria_backup_step() */ + pthread_mutex_t mutex; + /** status */ + enum Aria_backup_status status; + /** directory iterator */ + MY_DIR *dir; + /** number of consumed dir entries */ + size_t dir_consumed; + /** subdirectory iterator, or NULL if iterating to next entry in dir */ + MY_DIR *subdir; + /** number of consumed subdir entries */ + size_t subdir_consumed; }; -/* - Create directory in the target directory if it does not exist. - Return 0 on success, non-0 on failure. Set errno in case of failure +/** + Create a subdirectory unless we are streaming or it pre-exists. + @param target BACKUP SERVER target + @param name base name of subdirectory to create + @return error code (also errno will be set) + @retval 0 on success (errno might not be touched) */ static int aria_backup_mkdir(const struct backup_target *target, const char *name) @@ -142,22 +161,22 @@ static int aria_backup_mkdir(const struct backup_target *target, return 1; } +/** + Copy a file. + @param target BACKUP SERVER target (possibly, a directory) + @param sink per-thread context (possibly, a stream to write to) + @param path file name + @param dir_prefix length of the path prefix to omit from the backup + @return error code (also errno will be set) + @retval 0 on success (errno might not be touched) +*/ static int aria_backup_file(const struct backup_target *target, const struct backup_sink *sink, - const char *dir, const char *name, - size_t dir_prefix, int if_exists) + const char *path, size_t dir_prefix) { -#ifndef _WIN32 - int src, dst= sink->stream; -#endif int ret= -1; - char path[FN_REFLEN * 3 / 2]; - if ((int) sizeof path <= snprintf(path, sizeof path, "%s/%s", dir, name)) - { - my_error(ER_TOO_LONG_IDENT, MYF(0), name); - return -1; - } #ifndef _WIN32 + int src, dst= sink->stream; src= open(path, O_RDONLY); if (src < 0) { @@ -196,7 +215,7 @@ static int aria_backup_file(const struct backup_target *target, if ((int) sizeof dstpath <= snprintf(dstpath, sizeof dstpath, "%s/%s", target->path, path + dir_prefix)) - my_error(ER_TOO_LONG_IDENT, MYF(0), name); + my_error(ER_TOO_LONG_IDENT, MYF(0), path + dir_prefix); else if (!CopyFileEx(path, dstpath, NULL, NULL, NULL, COPY_FILE_NO_BUFFERING)) { @@ -247,125 +266,315 @@ static int aria_backup_file(const struct backup_target *target, #endif } -static int aria_backup_dir(const struct backup_target *target, - const struct backup_sink *sink, - name_predicate include_name, - const char *dir_name, size_t prefix) +/** + Back up a data file. + @param target BACKUP SERVER target (possibly, a directory) + @param sink per-thread context (possibly, a stream to write to) + @param include_p predicate for files to include + @return number directories or files left; negative on error + @retval 0 on successful completion +*/ +static int aria_backup_data(const struct backup_target *target, + const struct backup_sink *sink, + name_predicate include_p) { - int fail= 0; - char path[FN_REFLEN]; - if ((int) sizeof path <= - snprintf(path, sizeof path, "%s/%s", mysql_real_data_home, dir_name)) + const char *filename= NULL; + char path[FN_REFLEN * 2 + 2]; + struct Aria_backup *const ab= sink->ha_data; + int left= 0; + MY_DIR *dir= ab->dir; + pthread_mutex_lock(&ab->mutex); + assert(dir); + assert(ab->dir_consumed <= dir->number_of_files); + if (ab->status != BACKUP_OK) { - my_error(ER_TOO_LONG_IDENT, MYF(0), dir_name); - return -1; + assert(ab->status == BACKUP_FAIL); + err_exit: + left= -1; + ab->status= BACKUP_FAIL; + } + else if (!ab->subdir) + { + assert(!ab->subdir_consumed); + while (ab->dir_consumed < dir->number_of_files) + { + struct fileinfo *fi= &dir->dir_entry[ab->dir_consumed++]; + if ((fi->mystat->st_mode & S_IFMT) != S_IFDIR) + continue; + else if (aria_backup_mkdir(target, fi->name)); + else if ((ab->subdir= my_dir(fi->name, MYF(MY_WANT_STAT)))) + goto consume_subdir; + else + dir_error(fi->name); + goto err_exit; + } } - else if ((fail= aria_backup_mkdir(target, path + prefix)) != 0) - return fail; else { - MY_DIR *dir= my_dir(path, MYF(MY_WANT_STAT)); - if (!dir) - return dir_error(path); - else + consume_subdir: + assert(ab->dir_consumed > 0); + assert(ab->dir_consumed <= ab->dir->number_of_files); + dir= ab->subdir; + while (ab->subdir_consumed < dir->number_of_files) { - const struct fileinfo *fi= dir->dir_entry; - const struct fileinfo *const end= fi + dir->number_of_files; size_t len; - for (; fi < end; fi++) - if ((len= strlen(fi->name)) >= 4 && - /* - As noted in MDEV-25854, file names that start with #sql - must be excluded from the backup. For example, a call to - MDL_context::upgrade_shared_lock() in - mysql_inplace_alter_table() could time out, resulting in - cleanup_table_after_inplace_alter() deleting a - #sql-alter*.frm file before we get a chance to copy it. - */ - memcmp(fi->name, tmp_file_prefix, tmp_file_prefix_length) && - (*include_name)(fi->name, len)) - if ((fail= aria_backup_file(target, sink, path, fi->name, - prefix, 0)) != 0) - break; - my_dirend(dir); - return fail; + struct fileinfo *fi= &dir->dir_entry[ab->subdir_consumed++]; + if ((fi->mystat->st_mode & S_IFMT) != S_IFREG || + (len= strlen(fi->name)) < 4 || + /* + As noted in MDEV-25854, file names that start with #sql + must be excluded from the backup. For example, a call to + MDL_context::upgrade_shared_lock() in + mysql_inplace_alter_table() could time out, resulting in + cleanup_table_after_inplace_alter() deleting a + #sql-alter*.frm file before we get a chance to copy it. + */ + !memcmp(fi->name, tmp_file_prefix, tmp_file_prefix_length) || + !(*include_p)(fi->name, len)) + continue; + + /* Consume a file name */ + if ((int) sizeof path <= + snprintf(path, sizeof path, "%s/%s", + ab->dir->dir_entry[ab->dir_consumed - 1].name, fi->name)) + { + path[(sizeof path) - 1]= '\0'; + my_error(ER_TOO_LONG_IDENT, MYF(0), path); + goto err_exit; + } + + filename= path; + break; + } + + assert(dir == ab->subdir); + assert(ab->dir_consumed <= ab->dir->number_of_files); + assert(ab->subdir_consumed <= ab->subdir->number_of_files); + left= dir->number_of_files - ab->subdir_consumed; + if (!left) + { + left= ab->dir->number_of_files - ab->dir_consumed; + my_dirend(ab->subdir); + ab->subdir= NULL; + ab->subdir_consumed= 0; } } + + pthread_mutex_unlock(&ab->mutex); + if (filename && aria_backup_file(target, sink, filename, 0)) + return -1; + return left; } -static int aria_backup_data(const struct backup_target *target, - const struct backup_sink *sink, - name_predicate include_name) +/** + Back up an ENGINE=Aria log file. + @param target BACKUP SERVER target (possibly, a directory) + @param sink per-thread context (possibly, a stream to write to) + @return number directories or files left; negative on error + @retval 0 on successful completion +*/ +static int aria_backup_log(const struct backup_target *target, + const struct backup_sink *sink) { - int fail= 0; - size_t prefix= strlen(mysql_real_data_home) + 1; - /* Scan the server data directory for data files. */ - MY_DIR *dir= my_dir(mysql_real_data_home, MYF(MY_WANT_STAT)); - if (!dir) - return dir_error(mysql_real_data_home); + struct Aria_backup *const ab= sink->ha_data; + int left; + const char *filename= NULL; + char path[FN_REFLEN * 2 + 2]; + MY_DIR *dir= ab->dir; + pthread_mutex_lock(&ab->mutex); + assert(dir); + assert(!ab->subdir); + assert(!ab->subdir_consumed); + + if (ab->status != TRANSLOG_PURGE_DISABLED) + { + assert(ab->status == BACKUP_FAIL); + err_exit: + left= -1; + ab->status= BACKUP_FAIL; + } else { - const struct fileinfo *fi= dir->dir_entry; - const struct fileinfo *const end= fi + dir->number_of_files; - for (; fi < end; fi++) + while (ab->dir_consumed < dir->number_of_files) { - if ((fi->mystat->st_mode & S_IFMT) == S_IFDIR) - if ((fail= aria_backup_dir(target, sink, include_name, - fi->name, prefix)) != 0) - break; + struct fileinfo *fi= &dir->dir_entry[ab->dir_consumed++]; + if ((fi->mystat->st_mode & S_IFMT) != S_IFREG || + (strncmp(fi->name, C_STRING_WITH_LEN("aria_log.")) && + strcmp(fi->name, "aria_log_control"))) + continue; + else if ((int) sizeof path <= + snprintf(path, sizeof path, "%s/%s", maria_data_root, fi->name)) + { + path[(sizeof path) - 1]= '\0'; + my_error(ER_TOO_LONG_IDENT, MYF(0), path); + goto err_exit; + } + filename= path; + break; } - my_dirend(dir); + left= ab->dir->number_of_files - ab->dir_consumed; } - return fail; + + pthread_mutex_unlock(&ab->mutex); + if (filename && aria_backup_file(target, sink, filename, + strlen(maria_data_root) + 1)) + return -1; + return left; } -static int aria_backup_logs(const struct backup_target *target, - const struct backup_sink *sink) +/** + Start of a BACKUP SERVER phase, + when no aria_backup_step() or aria_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to backup_target + @retval NULL if no context needs to be created + @retval -1 on failure +*/ +void *aria_backup_start(THD *thd, const struct backup_target *target, + enum backup_phase phase, + const struct backup_sink *sink) { - size_t prefix= strlen(maria_data_root) + 1; - int fail= aria_backup_file(target, sink, maria_data_root, "aria_log_control", - prefix, 1); - if (!fail) - { - MY_DIR *dir; + switch (phase) { + struct Aria_backup* ab; + case BACKUP_PHASE_PREPARE_START: + assert(!sink); + return NULL; + case BACKUP_PHASE_START: + case BACKUP_PHASE_NO_BEGIN_NON_TRANS: + case BACKUP_PHASE_NO_DML_NON_TRANS: + assert(!sink->ha_data); + break; + case BACKUP_PHASE_NO_DDL: + assert(!sink->ha_data); + if (!(ab= calloc(1, sizeof *ab))) + return (void*) -1; + pthread_mutex_init(&ab->mutex, NULL); + opendir: + assert(!ab->dir); + assert(!ab->dir_consumed); + assert(!ab->subdir); + assert(!ab->subdir_consumed); + ab->dir= my_dir(".", MYF(MY_WANT_STAT)); + if (ab->dir) + return ab; + dir_error("."); + pthread_mutex_destroy(&ab->mutex); + free(ab); + return (void*) -1; + case BACKUP_PHASE_NO_COMMIT: translog_flush(translog_get_horizon()); - dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)); - if (dir) + ab= sink->ha_data; + goto opendir; + case BACKUP_PHASE_FINISH: + if (!sink || !sink->ha_data) + break; + ab= sink->ha_data; + assert(!ab->dir); + assert(!ab->dir_consumed); + assert(!ab->subdir); + assert(!ab->subdir_consumed); + if (ab->status != BACKUP_OK) + assert(ab->status == BACKUP_FAIL); + else if (!(ab->dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)))) + { + ab->status= BACKUP_FAIL; + return (void*) -1; + } + else { - const struct fileinfo *fi= dir->dir_entry; - const struct fileinfo *const end= fi + dir->number_of_files; - for (; fi < end; fi++) - if ((fail= - !strncmp(fi->name, C_STRING_WITH_LEN("aria_log.")) && - aria_backup_file(target, sink, maria_data_root, fi->name, - prefix, 0)) != 0) - break; - my_dirend(dir); + ab->status= TRANSLOG_PURGE_DISABLED; + translog_disable_purge(); } + break; + case BACKUP_PHASE_ABORT: + break; } - return fail; + return sink->ha_data; } -void *aria_backup_start(THD *thd, const struct backup_target *target, - enum backup_phase phase, - const struct backup_sink *sink) +/** + Process a file that was collected in aria_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number directories or files left; negative on error + @retval 0 on completion +*/ +int aria_backup_step(THD *thd, const struct backup_target *target, + enum backup_phase phase, const struct backup_sink *sink) { - DBUG_ASSERT((phase == BACKUP_PHASE_PREPARE_START) ? !sink : !sink->ha_data); switch (phase) { - int ret; + case BACKUP_PHASE_PREPARE_START: + assert(!sink); + break; + case BACKUP_PHASE_START: + case BACKUP_PHASE_NO_BEGIN_NON_TRANS: + case BACKUP_PHASE_NO_DML_NON_TRANS: + assert(!sink->ha_data); + break; case BACKUP_PHASE_NO_DDL: - if (aria_backup_data(target, sink, is_db_file)) - return (void*) -1; + return aria_backup_data(target, sink, is_db_file); + case BACKUP_PHASE_NO_COMMIT: + return aria_backup_data(target, sink, is_ma_file); + case BACKUP_PHASE_FINISH: + return aria_backup_log(target, sink); + case BACKUP_PHASE_ABORT: break; + } + return 0; +} + +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int aria_backup_end(THD *thd, const struct backup_target *target, + enum backup_phase phase, const struct backup_sink *sink) +{ + switch (phase) { + struct Aria_backup* ab; + case BACKUP_PHASE_PREPARE_START: + assert(!sink); + break; + case BACKUP_PHASE_START: + case BACKUP_PHASE_NO_BEGIN_NON_TRANS: + case BACKUP_PHASE_NO_DML_NON_TRANS: + assert(!sink->ha_data); + break; + case BACKUP_PHASE_NO_DDL: case BACKUP_PHASE_NO_COMMIT: - translog_disable_purge(); - ret= aria_backup_data(target, sink, is_ma_file) || - aria_backup_logs(target, sink); - translog_enable_purge(); - if (ret) - return (void*) -1; - default: + ab= sink->ha_data; + if (ab) + { + assert(!ab->subdir); + my_dirend(ab->dir); + ab->dir= NULL; + ab->dir_consumed= 0; + } break; + case BACKUP_PHASE_ABORT: + break; + case BACKUP_PHASE_FINISH: + ab= sink->ha_data; + if (!ab) + break; + if (ab->status == TRANSLOG_PURGE_DISABLED) + translog_enable_purge(); + my_dirend(ab->dir); + my_dirend(ab->subdir); + pthread_mutex_destroy(&ab->mutex); + free(ab); } - return NULL; + + return 0; } diff --git a/storage/maria/ma_backup_server.h b/storage/maria/ma_backup_server.h index a509699f9f645..6be76b4560f63 100644 --- a/storage/maria/ma_backup_server.h +++ b/storage/maria/ma_backup_server.h @@ -27,9 +27,41 @@ extern "C" @param target backup target @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) @param sink worker context - @return backup context object to be attached to backup_target, or nullptr + @return backup context object to be attached to backup_target + @retval NULL if no context needs to be created @retval -1 on failure */ void *aria_backup_start(THD *thd, const struct backup_target *target, enum backup_phase phase, const struct backup_sink *sink); + +#ifdef __cplusplus +extern "C" +#endif +/** + Process a file that was collected in aria_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number directories or files left; negative on error + @retval 0 on successful completion +*/ +int aria_backup_step(THD *thd, const struct backup_target *target, + enum backup_phase phase, const struct backup_sink *sink); + +#ifdef __cplusplus +extern "C" +#endif +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int aria_backup_end(THD *thd, const struct backup_target *target, + enum backup_phase phase, const struct backup_sink *sink); From 5a7f49d3cc912326b3dc5f4e98c8bee928adec02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 27 Aug 2026 17:34:42 +0300 Subject: [PATCH 08/17] fixup! e84155a7958e9bbdf1ffde788523d41a99a17d84 --- storage/maria/ma_backup_server.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c index d1114e3ef9cca..697ae0595e4c4 100644 --- a/storage/maria/ma_backup_server.c +++ b/storage/maria/ma_backup_server.c @@ -281,7 +281,7 @@ static int aria_backup_data(const struct backup_target *target, const char *filename= NULL; char path[FN_REFLEN * 2 + 2]; struct Aria_backup *const ab= sink->ha_data; - int left= 0; + size_t left= 0; MY_DIR *dir= ab->dir; pthread_mutex_lock(&ab->mutex); assert(dir); @@ -290,7 +290,7 @@ static int aria_backup_data(const struct backup_target *target, { assert(ab->status == BACKUP_FAIL); err_exit: - left= -1; + left= (size_t) -1; ab->status= BACKUP_FAIL; } else if (!ab->subdir) @@ -363,7 +363,7 @@ static int aria_backup_data(const struct backup_target *target, pthread_mutex_unlock(&ab->mutex); if (filename && aria_backup_file(target, sink, filename, 0)) return -1; - return left; + return (int) left; } /** @@ -377,7 +377,7 @@ static int aria_backup_log(const struct backup_target *target, const struct backup_sink *sink) { struct Aria_backup *const ab= sink->ha_data; - int left; + size_t left; const char *filename= NULL; char path[FN_REFLEN * 2 + 2]; MY_DIR *dir= ab->dir; @@ -390,7 +390,7 @@ static int aria_backup_log(const struct backup_target *target, { assert(ab->status == BACKUP_FAIL); err_exit: - left= -1; + left= (size_t) -1; ab->status= BACKUP_FAIL; } else @@ -419,7 +419,7 @@ static int aria_backup_log(const struct backup_target *target, if (filename && aria_backup_file(target, sink, filename, strlen(maria_data_root) + 1)) return -1; - return left; + return (int) left; } /** From c909702dfcd2261a73cd0ca6a6ce5703ddda411a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Aug 2026 06:32:37 +0300 Subject: [PATCH 09/17] squash! b625be27ca0a0b24730dd19d37d8bbf85917156c pread_write(): On 64-bit systems, allocate a buffer of up to 1 MiB backup::append(): Invoke a zero-copy shortcut for initial part, to be drained by non-zero-copy write of pipe_size. --- sql/sql_backup.cc | 52 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc index 04f7bb762648d..b97ed1bdcbcee 100644 --- a/sql/sql_backup.cc +++ b/sql/sql_backup.cc @@ -125,18 +125,26 @@ static ssize_t mmap_copy(const void *p, int out_fd, uint64_t o, uint64_t end) */ template static ssize_t pread_write(backup::handle in_fd, backup_fd out_fd, - uint64_t o, uint64_t end) - noexcept + uint64_t o, uint64_t end) noexcept { - constexpr size_t READ_WRITE_SIZE= 65536; - char *b= static_cast(aligned_malloc(READ_WRITE_SIZE, 4096)); + assert(end >= o); + uint64_t count{end - o}; +#if SIZEOF_SIZE_T < 8 + size_t size{65536}; +#else + constexpr size_t max_size{1 << 20}; + /* An integer multiple of 4096 bytes, up to max_size */ + size_t size{count >= max_size ? max_size : (size_t(count) + 4095) & 4095}; +#endif + char *b= static_cast(aligned_malloc(size, 4096)); if (!b) return -1; ssize_t ret; - for (uint64_t count{end - o};; o+= ret) + for (;; o+= ret) { - ret= pread(in_fd, b, - ssize_t(std::min(count, READ_WRITE_SIZE)), o); + if (count < size) + size= count; + ret= pread(in_fd, b, ssize_t(size), o); if (ret > 0) { if (!stream) @@ -276,6 +284,7 @@ int copy(handle src, backup_fd dst, uint64_t start, uint64_t end) noexcept int append(handle src, backup_fd stream, uint64_t start, uint64_t end) noexcept { assert(stream != backup_sink::NO_STREAM); + assert(end >= start); /* It is not safe to send from an mmap(2) on src, because we cannot guarantee that the receiving end of the pipe has consumed @@ -285,9 +294,36 @@ int append(handle src, backup_fd stream, uint64_t start, uint64_t end) noexcept It is unspecified whether changes made to the file after the mmap() call are visible in the mapped region. */ +#ifndef _WIN32 +# ifdef __linux__ + const int pipe_size{fcntl(stream, F_GETPIPE_SZ)}; +# elif defined __FreeBSD__ || defined __APPLE__ + // https://unix.stackexchange.com/questions/11946/how-big-is-the-pipe-buffer + constexpr int pipe_size{65535}; +# else + constexpr int pipe_size{0}; +# endif + if (pipe_size > 0) + { + /* + It is safe to invoke the zero-copy path if we can guarantee that + the recipient will drain the buffer before the final + nonzero-copy path completes. We assume that this is guaranteed + by writing at least pipe_size bytes via the nonzero-copy path + after this shortcut. + */ + const uint64_t fast_end{start + uint64_t(pipe_size)}; + if (fast_end < end) + { + if (int ret= backup_stream_append_async(src, stream, start, fast_end)) + return ret; + start= fast_end; + } + } +#endif return int(pread_write(src, stream, start, end)); } -}; +} extern "C" int backup_stream_append_plain(backup_fd src, backup_fd stream, uint64_t start, uint64_t end) From 9b196734f997f1a7336c22b5d5e370c9d1bbe9a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Aug 2026 06:36:31 +0300 Subject: [PATCH 10/17] fixup! e84155a7958e9bbdf1ffde788523d41a99a17d84 --- storage/maria/ma_backup_server.c | 141 +++++++++++++++++++++++-------- 1 file changed, 106 insertions(+), 35 deletions(-) diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c index 697ae0595e4c4..53a4396585d94 100644 --- a/storage/maria/ma_backup_server.c +++ b/storage/maria/ma_backup_server.c @@ -116,6 +116,10 @@ struct Aria_backup pthread_mutex_t mutex; /** status */ enum Aria_backup_status status; +#ifndef _WIN32 + /** directory file descriptor */ + int dirfd; +#endif /** directory iterator */ MY_DIR *dir; /** number of consumed dir entries */ @@ -161,23 +165,21 @@ static int aria_backup_mkdir(const struct backup_target *target, return 1; } +#ifndef _WIN32 /** Copy a file. @param target BACKUP SERVER target (possibly, a directory) @param sink per-thread context (possibly, a stream to write to) + @param dirfd source directory @param path file name - @param dir_prefix length of the path prefix to omit from the backup @return error code (also errno will be set) @retval 0 on success (errno might not be touched) */ static int aria_backup_file(const struct backup_target *target, const struct backup_sink *sink, - const char *path, size_t dir_prefix) + int dirfd, const char *path) { - int ret= -1; -#ifndef _WIN32 - int src, dst= sink->stream; - src= open(path, O_RDONLY); + int ret= -1, src= openat(dirfd, path, O_RDONLY), dst= sink->stream; if (src < 0) { my_error(ER_CANT_OPEN_FILE, MYF(0), path, errno); @@ -185,8 +187,7 @@ static int aria_backup_file(const struct backup_target *target, } if (dst < 0) { - dst= openat(target->fd, path + dir_prefix, - O_CREAT | O_EXCL | O_WRONLY, 0666); + dst= openat(target->fd, path, O_CREAT | O_EXCL | O_WRONLY, 0666); if (dst < 0) my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); else @@ -194,13 +195,13 @@ static int aria_backup_file(const struct backup_target *target, ret= copy_entire_file(src, dst) | close(dst); if (ret) write_error: - my_error(ER_ERROR_ON_WRITE, MYF(0), path + dir_prefix, errno); + my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); } } else { uint64_t end= (uint64_t) lseek(src, 0, SEEK_END); - ret= backup_stream_start(dst, path + dir_prefix, 0644, end, NULL, 0) || + ret= backup_stream_start(dst, path, 0644, end, NULL, 0) || backup_stream_append_plain(src, dst, 0, end) || backup_stream_zeropad(dst, (size_t) end); if (ret) @@ -208,7 +209,22 @@ static int aria_backup_file(const struct backup_target *target, } close(src); return ret; +} #else +/** + Copy a file. + @param target BACKUP SERVER target (possibly, a directory) + @param sink per-thread context (possibly, a stream to write to) + @param path file name + @param dir_prefix length of the path prefix to omit from the backup + @return error code (also errno will be set) + @retval 0 on success (errno might not be touched) +*/ +static int aria_backup_file(const struct backup_target *target, + const struct backup_sink *sink, + const char *path, size_t dir_prefix) +{ + int ret= -1; if (sink->stream == INVALID_HANDLE_VALUE) { char dstpath[FN_REFLEN * 3 / 2]; @@ -263,8 +279,8 @@ static int aria_backup_file(const struct backup_target *target, } } return ret; -#endif } +#endif /** Back up a data file. @@ -361,7 +377,13 @@ static int aria_backup_data(const struct backup_target *target, } pthread_mutex_unlock(&ab->mutex); - if (filename && aria_backup_file(target, sink, filename, 0)) + if (filename && +#ifndef _WIN32 + aria_backup_file(target, sink, ab->dirfd, filename) && +#else + aria_backup_file(target, sink, filename, 0) && +#endif + TRUE) return -1; return (int) left; } @@ -379,7 +401,9 @@ static int aria_backup_log(const struct backup_target *target, struct Aria_backup *const ab= sink->ha_data; size_t left; const char *filename= NULL; +#ifdef _WIN32 char path[FN_REFLEN * 2 + 2]; +#endif MY_DIR *dir= ab->dir; pthread_mutex_lock(&ab->mutex); assert(dir); @@ -389,7 +413,9 @@ static int aria_backup_log(const struct backup_target *target, if (ab->status != TRANSLOG_PURGE_DISABLED) { assert(ab->status == BACKUP_FAIL); +#ifdef _WIN32 err_exit: +#endif left= (size_t) -1; ab->status= BACKUP_FAIL; } @@ -402,6 +428,9 @@ static int aria_backup_log(const struct backup_target *target, (strncmp(fi->name, C_STRING_WITH_LEN("aria_log.")) && strcmp(fi->name, "aria_log_control"))) continue; +#ifndef _WIN32 + filename= fi->name; +#else else if ((int) sizeof path <= snprintf(path, sizeof path, "%s/%s", maria_data_root, fi->name)) { @@ -410,14 +439,21 @@ static int aria_backup_log(const struct backup_target *target, goto err_exit; } filename= path; +#endif break; } left= ab->dir->number_of_files - ab->dir_consumed; } pthread_mutex_unlock(&ab->mutex); - if (filename && aria_backup_file(target, sink, filename, - strlen(maria_data_root) + 1)) + if (filename && +#ifndef _WIN32 + aria_backup_file(target, sink, ab->dirfd, filename) && +#else + aria_backup_file(target, sink, filename, + strlen(maria_data_root) + 1) && +#endif + TRUE) return -1; return (int) left; } @@ -452,22 +488,36 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, if (!(ab= calloc(1, sizeof *ab))) return (void*) -1; pthread_mutex_init(&ab->mutex, NULL); - opendir: assert(!ab->dir); assert(!ab->dir_consumed); assert(!ab->subdir); assert(!ab->subdir_consumed); - ab->dir= my_dir(".", MYF(MY_WANT_STAT)); - if (ab->dir) - return ab; - dir_error("."); +#ifndef _WIN32 + if ((ab->dirfd= open(mysql_data_home, O_DIRECTORY)) >= 0) + { +#endif + ab->dir= my_dir(mysql_data_home, MYF(MY_WANT_STAT)); + if (ab->dir) + return ab; +#ifndef _WIN32 + close(ab->dirfd); + ab->dirfd= -1; + } +#endif + dir_error(mysql_data_home); pthread_mutex_destroy(&ab->mutex); free(ab); return (void*) -1; case BACKUP_PHASE_NO_COMMIT: translog_flush(translog_get_horizon()); +#ifndef NDEBUG ab= sink->ha_data; - goto opendir; +#endif + assert(ab->dir); + assert(!ab->dir_consumed); + assert(!ab->subdir); + assert(!ab->subdir_consumed); + break; case BACKUP_PHASE_FINISH: if (!sink || !sink->ha_data) break; @@ -477,18 +527,27 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, assert(!ab->subdir); assert(!ab->subdir_consumed); if (ab->status != BACKUP_OK) - assert(ab->status == BACKUP_FAIL); - else if (!(ab->dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)))) { - ab->status= BACKUP_FAIL; - return (void*) -1; + assert(ab->status == BACKUP_FAIL); + break; } - else +#ifndef _WIN32 + if ((ab->dirfd= open(maria_data_root, O_DIRECTORY)) >= 0) { - ab->status= TRANSLOG_PURGE_DISABLED; - translog_disable_purge(); +#endif + if ((ab->dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)))) + { + ab->status= TRANSLOG_PURGE_DISABLED; + translog_disable_purge(); + break; + } +#ifndef _WIN32 + close(ab->dirfd); + ab->dirfd= -1; } - break; +#endif + ab->status= BACKUP_FAIL; + return (void*) -1; case BACKUP_PHASE_ABORT: break; } @@ -552,15 +611,24 @@ int aria_backup_end(THD *thd, const struct backup_target *target, assert(!sink->ha_data); break; case BACKUP_PHASE_NO_DDL: + ab= sink->ha_data; + if (!ab) + break; + assert(!ab->subdir); + assert(!ab->subdir_consumed); + /* Rewind the directory for BACKUP_PHASE_NO_COMMIT */ + assert(ab->dir); + ab->dir_consumed= 0; + break; case BACKUP_PHASE_NO_COMMIT: ab= sink->ha_data; - if (ab) - { - assert(!ab->subdir); - my_dirend(ab->dir); - ab->dir= NULL; - ab->dir_consumed= 0; - } + if (!ab) + break; + assert(!ab->subdir); + assert(!ab->subdir_consumed); + my_dirend(ab->dir); + ab->dir= NULL; + ab->dir_consumed= 0; break; case BACKUP_PHASE_ABORT: break; @@ -572,6 +640,9 @@ int aria_backup_end(THD *thd, const struct backup_target *target, translog_enable_purge(); my_dirend(ab->dir); my_dirend(ab->subdir); +#ifndef _WIN32 + close(ab->dirfd); +#endif pthread_mutex_destroy(&ab->mutex); free(ab); } From da840d352e08e765d42abc801fa711b81dc09c8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Aug 2026 09:49:04 +0300 Subject: [PATCH 11/17] fixup! 9b196734f997f1a7336c22b5d5e370c9d1bbe9a6 Use fdopendir(3) and openat(2) on POSIX, to fix libmysqld. FIXME: Microsoft Windows is broken --- storage/maria/ma_backup_server.c | 259 ++++++++++++++++++++++++++----- 1 file changed, 219 insertions(+), 40 deletions(-) diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c index 53a4396585d94..20c50ebba1ad2 100644 --- a/storage/maria/ma_backup_server.c +++ b/storage/maria/ma_backup_server.c @@ -22,6 +22,11 @@ # define tmp_file_prefix_length 4 #endif +#ifndef _WIN32 +# include +# include +#endif + /** Report that a directory cannot be read. @param name directory name @@ -119,7 +124,15 @@ struct Aria_backup #ifndef _WIN32 /** directory file descriptor */ int dirfd; -#endif + /** subdirectory file descriptor */ + int subdirfd; + /** directory stream */ + DIR *dir; + /** the readdir(dir) for which subdir was opened */ + const struct dirent *d; + /** subdirectory stream, or NULL if iterating to next entry in dir */ + DIR *subdir; +#else /** directory iterator */ MY_DIR *dir; /** number of consumed dir entries */ @@ -128,6 +141,7 @@ struct Aria_backup MY_DIR *subdir; /** number of consumed subdir entries */ size_t subdir_consumed; +#endif }; /** @@ -297,20 +311,61 @@ static int aria_backup_data(const struct backup_target *target, const char *filename= NULL; char path[FN_REFLEN * 2 + 2]; struct Aria_backup *const ab= sink->ha_data; - size_t left= 0; - MY_DIR *dir= ab->dir; + int left= 0; +#ifndef _WIN32 + DIR *dir; + struct dirent *d; + struct stat sb; +#else + MY_DIR *dir; +#endif pthread_mutex_lock(&ab->mutex); + dir= ab->dir; assert(dir); +#ifdef _WIN32 assert(ab->dir_consumed <= dir->number_of_files); +#endif + if (ab->status != BACKUP_OK) { assert(ab->status == BACKUP_FAIL); err_exit: - left= (size_t) -1; + left= -1; ab->status= BACKUP_FAIL; } else if (!ab->subdir) { +#ifndef _WIN32 + assert(ab->subdirfd < 0); + while ((d= readdir(dir)) != NULL) + { + switch (d->d_type) { + default: + continue; + case DT_DIR: + break; + case DT_UNKNOWN: + if (fstatat(ab->dirfd, d->d_name, &sb, 0) || + (sb.st_mode & S_IFMT) != S_IFDIR) + continue; + } + if (!aria_backup_mkdir(target, d->d_name)) + { + ab->d= d; + if ((ab->subdirfd= openat(ab->dirfd, d->d_name, O_DIRECTORY)) >= 0) + { + int d= dup(ab->subdirfd); + if ((ab->subdir= fdopendir(d))) + goto consume_subdir; + close(d); + close(ab->subdirfd); + ab->subdirfd= -1; + } + dir_error(d->d_name); + } + goto err_exit; + } +#else assert(!ab->subdir_consumed); while (ab->dir_consumed < dir->number_of_files) { @@ -324,13 +379,69 @@ static int aria_backup_data(const struct backup_target *target, dir_error(fi->name); goto err_exit; } +#endif } else { consume_subdir: +#ifndef _WIN32 + assert(ab->d); + assert(ab->d->d_type == DT_DIR || ab->d->d_type == DT_UNKNOWN); + dir= ab->subdir; + while ((d= readdir(dir)) != NULL) + { + const char *const name= d->d_name; + size_t len; + switch (d->d_type) { + default: + continue; + case DT_REG: + case DT_LNK: + break; + case DT_UNKNOWN: + if (fstatat(ab->subdirfd, name, &sb, 0) || + (sb.st_mode & S_IFMT) != S_IFREG) + continue; + } + if ((len= strlen(name)) < 4 || + /* + As noted in MDEV-25854, file names that start with #sql + must be excluded from the backup. For example, a call to + MDL_context::upgrade_shared_lock() in + mysql_inplace_alter_table() could time out, resulting in + cleanup_table_after_inplace_alter() deleting a + #sql-alter*.frm file before we get a chance to copy it. + */ + !memcmp(name, tmp_file_prefix, tmp_file_prefix_length) || + !(*include_p)(name, len)) + continue; + + /* Consume a file name */ + if ((int) sizeof path <= + snprintf(path, sizeof path, "%s/%s", ab->d->d_name, name)) + { + path[(sizeof path) - 1]= '\0'; + my_error(ER_TOO_LONG_IDENT, MYF(0), path); + goto err_exit; + } + filename= path; + break; + } + + assert(dir == ab->subdir); + left= 1; + if (!d) + { + closedir(ab->subdir); + close(ab->subdirfd); + ab->d= NULL; + ab->subdir= NULL; + ab->subdirfd= -1; + } +#else + dir= ab->subdir; assert(ab->dir_consumed > 0); assert(ab->dir_consumed <= ab->dir->number_of_files); - dir= ab->subdir; while (ab->subdir_consumed < dir->number_of_files) { size_t len; @@ -366,14 +477,15 @@ static int aria_backup_data(const struct backup_target *target, assert(dir == ab->subdir); assert(ab->dir_consumed <= ab->dir->number_of_files); assert(ab->subdir_consumed <= ab->subdir->number_of_files); - left= dir->number_of_files - ab->subdir_consumed; + left= dir->number_of_files > ab->subdir_consumed; if (!left) { - left= ab->dir->number_of_files - ab->dir_consumed; + left= ab->dir->number_of_files > ab->dir_consumed; my_dirend(ab->subdir); ab->subdir= NULL; ab->subdir_consumed= 0; } +#endif } pthread_mutex_unlock(&ab->mutex); @@ -385,7 +497,7 @@ static int aria_backup_data(const struct backup_target *target, #endif TRUE) return -1; - return (int) left; + return left; } /** @@ -399,16 +511,23 @@ static int aria_backup_log(const struct backup_target *target, const struct backup_sink *sink) { struct Aria_backup *const ab= sink->ha_data; - size_t left; + int left; const char *filename= NULL; #ifdef _WIN32 char path[FN_REFLEN * 2 + 2]; + MY_DIR *const dir= ab->dir; +#else + DIR *const dir= ab->dir; #endif - MY_DIR *dir= ab->dir; pthread_mutex_lock(&ab->mutex); assert(dir); assert(!ab->subdir); +#ifdef _WIN32 assert(!ab->subdir_consumed); +#else + assert(ab->subdirfd < 0); + assert(!ab->d); +#endif if (ab->status != TRANSLOG_PURGE_DISABLED) { @@ -416,11 +535,35 @@ static int aria_backup_log(const struct backup_target *target, #ifdef _WIN32 err_exit: #endif - left= (size_t) -1; + left= -1; ab->status= BACKUP_FAIL; } else { +#ifndef _WIN32 + struct dirent *d; + while ((d= readdir(dir)) != NULL) + { + struct stat sb; + switch (d->d_type) { + default: + continue; + case DT_REG: + case DT_LNK: + break; + case DT_UNKNOWN: + if (fstatat(ab->dirfd, d->d_name, &sb, 0) || + (sb.st_mode & S_IFMT) != S_IFREG) + continue; + } + if (strncmp(d->d_name, C_STRING_WITH_LEN("aria_log.")) && + strcmp(d->d_name, "aria_log_control")) + continue; + filename= d->d_name; + break; + } + left= d != NULL; +#else while (ab->dir_consumed < dir->number_of_files) { struct fileinfo *fi= &dir->dir_entry[ab->dir_consumed++]; @@ -428,9 +571,6 @@ static int aria_backup_log(const struct backup_target *target, (strncmp(fi->name, C_STRING_WITH_LEN("aria_log.")) && strcmp(fi->name, "aria_log_control"))) continue; -#ifndef _WIN32 - filename= fi->name; -#else else if ((int) sizeof path <= snprintf(path, sizeof path, "%s/%s", maria_data_root, fi->name)) { @@ -439,10 +579,10 @@ static int aria_backup_log(const struct backup_target *target, goto err_exit; } filename= path; -#endif break; } - left= ab->dir->number_of_files - ab->dir_consumed; + left= ab->dir->number_of_files > ab->dir_consumed; +#endif } pthread_mutex_unlock(&ab->mutex); @@ -455,7 +595,7 @@ static int aria_backup_log(const struct backup_target *target, #endif TRUE) return -1; - return (int) left; + return left; } /** @@ -488,18 +628,18 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, if (!(ab= calloc(1, sizeof *ab))) return (void*) -1; pthread_mutex_init(&ab->mutex, NULL); - assert(!ab->dir); - assert(!ab->dir_consumed); - assert(!ab->subdir); - assert(!ab->subdir_consumed); -#ifndef _WIN32 +#ifdef _WIN32 + ab->dir= my_dir(mysql_data_home, MYF(MY_WANT_STAT)); + if (ab->dir) + return ab; +#else + ab->subdirfd= -1; if ((ab->dirfd= open(mysql_data_home, O_DIRECTORY)) >= 0) { -#endif - ab->dir= my_dir(mysql_data_home, MYF(MY_WANT_STAT)); - if (ab->dir) + int d= dup(ab->dirfd); + if ((ab->dir= fdopendir(d))) return ab; -#ifndef _WIN32 + close(d); close(ab->dirfd); ab->dirfd= -1; } @@ -514,37 +654,55 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, ab= sink->ha_data; #endif assert(ab->dir); - assert(!ab->dir_consumed); assert(!ab->subdir); +#ifndef _WIN32 + assert(!ab->d); + assert(ab->subdirfd == -1); + assert(ab->dirfd >= 0); +#else + assert(!ab->dir_consumed); assert(!ab->subdir_consumed); +#endif break; case BACKUP_PHASE_FINISH: - if (!sink || !sink->ha_data) + if (!sink) break; ab= sink->ha_data; - assert(!ab->dir); - assert(!ab->dir_consumed); - assert(!ab->subdir); - assert(!ab->subdir_consumed); + if (!ab) + break; if (ab->status != BACKUP_OK) { assert(ab->status == BACKUP_FAIL); break; } + assert(!ab->dir); + assert(!ab->subdir); #ifndef _WIN32 + assert(!ab->d); + assert(ab->subdirfd == -1); + assert(ab->dirfd == -1); if ((ab->dirfd= open(maria_data_root, O_DIRECTORY)) >= 0) { -#endif - if ((ab->dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)))) + int d= dup(ab->dirfd); + if ((ab->dir= fdopendir(d))) { ab->status= TRANSLOG_PURGE_DISABLED; translog_disable_purge(); break; } -#ifndef _WIN32 + close(d); close(ab->dirfd); ab->dirfd= -1; } +#else + assert(!ab->dir_consumed); + assert(!ab->subdir_consumed); + if ((ab->dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)))) + { + ab->status= TRANSLOG_PURGE_DISABLED; + translog_disable_purge(); + break; + } #endif ab->status= BACKUP_FAIL; return (void*) -1; @@ -615,20 +773,37 @@ int aria_backup_end(THD *thd, const struct backup_target *target, if (!ab) break; assert(!ab->subdir); - assert(!ab->subdir_consumed); - /* Rewind the directory for BACKUP_PHASE_NO_COMMIT */ assert(ab->dir); + /* Rewind the directory for BACKUP_PHASE_NO_COMMIT */ +#ifndef _WIN32 + assert(ab->dirfd >= 0); + assert(ab->subdirfd == -1); + assert(!ab->d); + rewinddir(ab->dir); +#else + assert(!ab->subdir_consumed); ab->dir_consumed= 0; +#endif break; case BACKUP_PHASE_NO_COMMIT: ab= sink->ha_data; if (!ab) break; + assert(ab->dir); assert(!ab->subdir); +#ifndef _WIN32 + assert(!ab->d); + assert(ab->dirfd >= 0); + assert(ab->subdirfd == -1); + closedir(ab->dir); + close(ab->dirfd); + ab->dirfd= -1; +#else assert(!ab->subdir_consumed); my_dirend(ab->dir); - ab->dir= NULL; ab->dir_consumed= 0; +#endif + ab->dir= NULL; break; case BACKUP_PHASE_ABORT: break; @@ -638,10 +813,14 @@ int aria_backup_end(THD *thd, const struct backup_target *target, break; if (ab->status == TRANSLOG_PURGE_DISABLED) translog_enable_purge(); - my_dirend(ab->dir); - my_dirend(ab->subdir); #ifndef _WIN32 + closedir(ab->dir); + closedir(ab->subdir); close(ab->dirfd); + close(ab->subdirfd); +#else + my_dirend(ab->dir); + my_dirend(ab->subdir); #endif pthread_mutex_destroy(&ab->mutex); free(ab); From 5823f92a3546316ded1df238e60c6e986801d5ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Aug 2026 10:19:05 +0300 Subject: [PATCH 12/17] fixup! da840d352e08e765d42abc801fa711b81dc09c8b --- storage/maria/ma_backup_server.c | 73 +++++++++++--------------------- 1 file changed, 25 insertions(+), 48 deletions(-) diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c index 20c50ebba1ad2..6ab5fc207c018 100644 --- a/storage/maria/ma_backup_server.c +++ b/storage/maria/ma_backup_server.c @@ -122,10 +122,6 @@ struct Aria_backup /** status */ enum Aria_backup_status status; #ifndef _WIN32 - /** directory file descriptor */ - int dirfd; - /** subdirectory file descriptor */ - int subdirfd; /** directory stream */ DIR *dir; /** the readdir(dir) for which subdir was opened */ @@ -336,7 +332,6 @@ static int aria_backup_data(const struct backup_target *target, else if (!ab->subdir) { #ifndef _WIN32 - assert(ab->subdirfd < 0); while ((d= readdir(dir)) != NULL) { switch (d->d_type) { @@ -345,21 +340,19 @@ static int aria_backup_data(const struct backup_target *target, case DT_DIR: break; case DT_UNKNOWN: - if (fstatat(ab->dirfd, d->d_name, &sb, 0) || + if (fstatat(dirfd(dir), d->d_name, &sb, 0) || (sb.st_mode & S_IFMT) != S_IFDIR) continue; } if (!aria_backup_mkdir(target, d->d_name)) { + int dfd= openat(dirfd(dir), d->d_name, O_DIRECTORY); ab->d= d; - if ((ab->subdirfd= openat(ab->dirfd, d->d_name, O_DIRECTORY)) >= 0) + if (dfd >= 0) { - int d= dup(ab->subdirfd); - if ((ab->subdir= fdopendir(d))) + if ((ab->subdir= fdopendir(dfd))) goto consume_subdir; - close(d); - close(ab->subdirfd); - ab->subdirfd= -1; + close(dfd); } dir_error(d->d_name); } @@ -399,7 +392,7 @@ static int aria_backup_data(const struct backup_target *target, case DT_LNK: break; case DT_UNKNOWN: - if (fstatat(ab->subdirfd, name, &sb, 0) || + if (fstatat(dirfd(ab->subdir), name, &sb, 0) || (sb.st_mode & S_IFMT) != S_IFREG) continue; } @@ -433,10 +426,8 @@ static int aria_backup_data(const struct backup_target *target, if (!d) { closedir(ab->subdir); - close(ab->subdirfd); ab->d= NULL; ab->subdir= NULL; - ab->subdirfd= -1; } #else dir= ab->subdir; @@ -491,7 +482,7 @@ static int aria_backup_data(const struct backup_target *target, pthread_mutex_unlock(&ab->mutex); if (filename && #ifndef _WIN32 - aria_backup_file(target, sink, ab->dirfd, filename) && + aria_backup_file(target, sink, dirfd(ab->dir), filename) && #else aria_backup_file(target, sink, filename, 0) && #endif @@ -525,7 +516,6 @@ static int aria_backup_log(const struct backup_target *target, #ifdef _WIN32 assert(!ab->subdir_consumed); #else - assert(ab->subdirfd < 0); assert(!ab->d); #endif @@ -552,7 +542,7 @@ static int aria_backup_log(const struct backup_target *target, case DT_LNK: break; case DT_UNKNOWN: - if (fstatat(ab->dirfd, d->d_name, &sb, 0) || + if (fstatat(dirfd(ab->dir), d->d_name, &sb, 0) || (sb.st_mode & S_IFMT) != S_IFREG) continue; } @@ -588,7 +578,7 @@ static int aria_backup_log(const struct backup_target *target, pthread_mutex_unlock(&ab->mutex); if (filename && #ifndef _WIN32 - aria_backup_file(target, sink, ab->dirfd, filename) && + aria_backup_file(target, sink, dirfd(ab->dir), filename) && #else aria_backup_file(target, sink, filename, strlen(maria_data_root) + 1) && @@ -633,15 +623,14 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, if (ab->dir) return ab; #else - ab->subdirfd= -1; - if ((ab->dirfd= open(mysql_data_home, O_DIRECTORY)) >= 0) { - int d= dup(ab->dirfd); - if ((ab->dir= fdopendir(d))) - return ab; - close(d); - close(ab->dirfd); - ab->dirfd= -1; + int dfd= open(mysql_data_home, O_DIRECTORY); + if (dfd >= 0) + { + if ((ab->dir= fdopendir(dfd))) + return ab; + close(dfd); + } } #endif dir_error(mysql_data_home); @@ -657,8 +646,6 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, assert(!ab->subdir); #ifndef _WIN32 assert(!ab->d); - assert(ab->subdirfd == -1); - assert(ab->dirfd >= 0); #else assert(!ab->dir_consumed); assert(!ab->subdir_consumed); @@ -679,20 +666,18 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, assert(!ab->subdir); #ifndef _WIN32 assert(!ab->d); - assert(ab->subdirfd == -1); - assert(ab->dirfd == -1); - if ((ab->dirfd= open(maria_data_root, O_DIRECTORY)) >= 0) { - int d= dup(ab->dirfd); - if ((ab->dir= fdopendir(d))) + int dfd= open(maria_data_root, O_DIRECTORY); + if (dfd >= 0) { - ab->status= TRANSLOG_PURGE_DISABLED; - translog_disable_purge(); - break; + if ((ab->dir= fdopendir(dfd))) + { + ab->status= TRANSLOG_PURGE_DISABLED; + translog_disable_purge(); + break; + } + close(dfd); } - close(d); - close(ab->dirfd); - ab->dirfd= -1; } #else assert(!ab->dir_consumed); @@ -776,8 +761,6 @@ int aria_backup_end(THD *thd, const struct backup_target *target, assert(ab->dir); /* Rewind the directory for BACKUP_PHASE_NO_COMMIT */ #ifndef _WIN32 - assert(ab->dirfd >= 0); - assert(ab->subdirfd == -1); assert(!ab->d); rewinddir(ab->dir); #else @@ -793,11 +776,7 @@ int aria_backup_end(THD *thd, const struct backup_target *target, assert(!ab->subdir); #ifndef _WIN32 assert(!ab->d); - assert(ab->dirfd >= 0); - assert(ab->subdirfd == -1); closedir(ab->dir); - close(ab->dirfd); - ab->dirfd= -1; #else assert(!ab->subdir_consumed); my_dirend(ab->dir); @@ -816,8 +795,6 @@ int aria_backup_end(THD *thd, const struct backup_target *target, #ifndef _WIN32 closedir(ab->dir); closedir(ab->subdir); - close(ab->dirfd); - close(ab->subdirfd); #else my_dirend(ab->dir); my_dirend(ab->subdir); From f46d1e4f9443b535572948c08f4288a32ee02f3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Aug 2026 11:36:01 +0300 Subject: [PATCH 13/17] fixup! 5823f92a3546316ded1df238e60c6e986801d5ec Use the native FindFirstFileA() and FindNextFile() on Microsoft Windows --- storage/maria/ma_backup_server.c | 199 +++++++++++++++++-------------- 1 file changed, 109 insertions(+), 90 deletions(-) diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c index 6ab5fc207c018..38eafb1ed2d19 100644 --- a/storage/maria/ma_backup_server.c +++ b/storage/maria/ma_backup_server.c @@ -33,7 +33,10 @@ */ ATTRIBUTE_COLD ATTRIBUTE_NOINLINE static void dir_error(const char *name) { - my_error(ER_CANT_READ_DIR, MYF(0), name, my_errno); +#ifdef _WIN32 + my_osmaperr(GetLastError()); +#endif + my_error(ER_CANT_READ_DIR, MYF(0), name, errno); } /** @@ -117,26 +120,28 @@ enum Aria_backup_status { BACKUP_OK, BACKUP_FAIL, TRANSLOG_PURGE_DISABLED }; /** Backup state */ struct Aria_backup { - /** mutex protecting the data in concurrent aria_backup_step() */ - pthread_mutex_t mutex; - /** status */ - enum Aria_backup_status status; #ifndef _WIN32 /** directory stream */ DIR *dir; - /** the readdir(dir) for which subdir was opened */ + /** the readdir(dir) result for which subdir was opened */ const struct dirent *d; /** subdirectory stream, or NULL if iterating to next entry in dir */ DIR *subdir; #else /** directory iterator */ - MY_DIR *dir; - /** number of consumed dir entries */ - size_t dir_consumed; - /** subdirectory iterator, or NULL if iterating to next entry in dir */ - MY_DIR *subdir; - /** number of consumed subdir entries */ - size_t subdir_consumed; + HANDLE dir; + /** subdirectory iterator, or INVALID_HANDLE_VALUE */ + HANDLE subdir; +#endif + /** status */ + enum Aria_backup_status status; + /** mutex protecting d, subdir, status in concurrent aria_backup_step() */ + pthread_mutex_t mutex; +#ifdef _WIN32 + /** FindFirstFileA()/FindNextFile() buffer for dir */ + WIN32_FIND_DATAA d; + /** FindFirstFileA()/FindNextFile() buffer for subdir */ + WIN32_FIND_DATAA sd; #endif }; @@ -309,18 +314,13 @@ static int aria_backup_data(const struct backup_target *target, struct Aria_backup *const ab= sink->ha_data; int left= 0; #ifndef _WIN32 - DIR *dir; struct dirent *d; struct stat sb; + assert(ab->dir); #else - MY_DIR *dir; + assert(ab->dir != INVALID_HANDLE_VALUE); #endif pthread_mutex_lock(&ab->mutex); - dir= ab->dir; - assert(dir); -#ifdef _WIN32 - assert(ab->dir_consumed <= dir->number_of_files); -#endif if (ab->status != BACKUP_OK) { @@ -329,9 +329,10 @@ static int aria_backup_data(const struct backup_target *target, left= -1; ab->status= BACKUP_FAIL; } +#ifndef _WIN32 else if (!ab->subdir) { -#ifndef _WIN32 + DIR *const dir= ab->dir; while ((d= readdir(dir)) != NULL) { switch (d->d_type) { @@ -358,29 +359,13 @@ static int aria_backup_data(const struct backup_target *target, } goto err_exit; } -#else - assert(!ab->subdir_consumed); - while (ab->dir_consumed < dir->number_of_files) - { - struct fileinfo *fi= &dir->dir_entry[ab->dir_consumed++]; - if ((fi->mystat->st_mode & S_IFMT) != S_IFDIR) - continue; - else if (aria_backup_mkdir(target, fi->name)); - else if ((ab->subdir= my_dir(fi->name, MYF(MY_WANT_STAT)))) - goto consume_subdir; - else - dir_error(fi->name); - goto err_exit; - } -#endif } else { consume_subdir: -#ifndef _WIN32 + DIR *const dir= ab->subdir; assert(ab->d); assert(ab->d->d_type == DT_DIR || ab->d->d_type == DT_UNKNOWN); - dir= ab->subdir; while ((d= readdir(dir)) != NULL) { const char *const name= d->d_name; @@ -392,7 +377,7 @@ static int aria_backup_data(const struct backup_target *target, case DT_LNK: break; case DT_UNKNOWN: - if (fstatat(dirfd(ab->subdir), name, &sb, 0) || + if (fstatat(dirfd(dir), name, &sb, 0) || (sb.st_mode & S_IFMT) != S_IFREG) continue; } @@ -425,15 +410,40 @@ static int aria_backup_data(const struct backup_target *target, left= 1; if (!d) { - closedir(ab->subdir); ab->d= NULL; ab->subdir= NULL; + closedir(dir); } + } #else - dir= ab->subdir; - assert(ab->dir_consumed > 0); - assert(ab->dir_consumed <= ab->dir->number_of_files); - while (ab->subdir_consumed < dir->number_of_files) + else if (ab->subdir == INVALID_HANDLE_VALUE) + { + do + { + if (!(ab->d.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) + continue; + + if ((int) sizeof path <= + snprintf(path, sizeof path, "%s/*.*", ab->d.cFileName)) + { + name_too_long: + path[(sizeof path) - 1]= '\0'; + my_error(ER_TOO_LONG_IDENT, MYF(0), path); + } + else if (aria_backup_mkdir(target, ab->d.cFileName)); + else if ((ab->subdir= FindFirstFileA(path, &ab->sd)) != + INVALID_HANDLE_VALUE) + goto consume_subdir; + else + dir_error(path); + goto err_exit; + } + while (FindNextFile(ab->dir, &ab->d)); + } + else + { + consume_subdir: + do { size_t len; struct fileinfo *fi= &dir->dir_entry[ab->subdir_consumed++]; @@ -454,30 +464,21 @@ static int aria_backup_data(const struct backup_target *target, /* Consume a file name */ if ((int) sizeof path <= snprintf(path, sizeof path, "%s/%s", - ab->dir->dir_entry[ab->dir_consumed - 1].name, fi->name)) - { - path[(sizeof path) - 1]= '\0'; - my_error(ER_TOO_LONG_IDENT, MYF(0), path); - goto err_exit; - } + ab->d.cFileName, ab->sd.cFileName)); + goto name_too_long; filename= path; - break; } + while ((left= FindNextFile(ab->subdir, &ab->sd)) && !filename); - assert(dir == ab->subdir); - assert(ab->dir_consumed <= ab->dir->number_of_files); - assert(ab->subdir_consumed <= ab->subdir->number_of_files); - left= dir->number_of_files > ab->subdir_consumed; if (!left) { - left= ab->dir->number_of_files > ab->dir_consumed; - my_dirend(ab->subdir); - ab->subdir= NULL; - ab->subdir_consumed= 0; + FindClose(ab->subdir); + ab->subdir= INVALID_HANDLE_VALUE; + left= FindNextFile(ab->dir, &ab->d); } -#endif } +#endif pthread_mutex_unlock(&ab->mutex); if (filename && @@ -506,17 +507,15 @@ static int aria_backup_log(const struct backup_target *target, const char *filename= NULL; #ifdef _WIN32 char path[FN_REFLEN * 2 + 2]; - MY_DIR *const dir= ab->dir; -#else - DIR *const dir= ab->dir; #endif pthread_mutex_lock(&ab->mutex); - assert(dir); +#ifndef _WIN32 + assert(!ab->d); + assert(ab->dir); assert(!ab->subdir); -#ifdef _WIN32 - assert(!ab->subdir_consumed); #else - assert(!ab->d); + assert(ab->dir != INVALID_HANDLE_VALUE); + assert(ab->subdir == INVALID_HANDLE_VALUE); #endif if (ab->status != TRANSLOG_PURGE_DISABLED) @@ -532,7 +531,7 @@ static int aria_backup_log(const struct backup_target *target, { #ifndef _WIN32 struct dirent *d; - while ((d= readdir(dir)) != NULL) + while ((d= readdir(ab->dir)) != NULL) { struct stat sb; switch (d->d_type) { @@ -619,8 +618,9 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, return (void*) -1; pthread_mutex_init(&ab->mutex, NULL); #ifdef _WIN32 - ab->dir= my_dir(mysql_data_home, MYF(MY_WANT_STAT)); - if (ab->dir) + ab->subdir= INVALID_HANDLE_VALUE; + ab->dir= FindFirstFileA("*.*", &ab->d); + if (ab->dir != INVALID_HANDLE_VALUE) return ab; #else { @@ -662,9 +662,9 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, assert(ab->status == BACKUP_FAIL); break; } +#ifndef _WIN32 assert(!ab->dir); assert(!ab->subdir); -#ifndef _WIN32 assert(!ab->d); { int dfd= open(maria_data_root, O_DIRECTORY); @@ -680,15 +680,22 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, } } #else - assert(!ab->dir_consumed); - assert(!ab->subdir_consumed); - if ((ab->dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)))) + assert(ab->dir == INVALID_HANDLE_VALUE); + assert(ab->subdir == INVALID_HANDLE_VALUE); { - ab->status= TRANSLOG_PURGE_DISABLED; - translog_disable_purge(); - break; + char path[FN_REFLEN * 2 + 2]; + if ((int) sizeof path > + snprintf(path, sizeof path, "%s/aria_log*", maria_data_root) && + (ab->dir= FindFirstFileA(path, &ab->d)) != + INVALID_HANDLE_VALUE) + { + ab->status= TRANSLOG_PURGE_DISABLED; + translog_disable_purge(); + break; + } } #endif + dir_error(maria_data_root); ab->status= BACKUP_FAIL; return (void*) -1; case BACKUP_PHASE_ABORT: @@ -757,32 +764,40 @@ int aria_backup_end(THD *thd, const struct backup_target *target, ab= sink->ha_data; if (!ab) break; - assert(!ab->subdir); - assert(ab->dir); /* Rewind the directory for BACKUP_PHASE_NO_COMMIT */ #ifndef _WIN32 assert(!ab->d); + assert(!ab->subdir); + assert(ab->dir); rewinddir(ab->dir); #else - assert(!ab->subdir_consumed); - ab->dir_consumed= 0; + assert(ab->subdir == INVALID_HANDLE_VALUE); + assert(ab->dir != INVALID_HANDLE_VALUE); + FindClose(ab->dir); + ab->dir= FindFirstFileA("*.*", &ab->d); + if (ab->dir == INVALID_HANDLE_VALUE) + { + dir_error(mysql_data_home); + return -1; + } #endif break; case BACKUP_PHASE_NO_COMMIT: ab= sink->ha_data; if (!ab) break; +#ifndef _WIN32 assert(ab->dir); assert(!ab->subdir); -#ifndef _WIN32 assert(!ab->d); closedir(ab->dir); + ab->dir= NULL; #else - assert(!ab->subdir_consumed); - my_dirend(ab->dir); - ab->dir_consumed= 0; + assert(ab->dir != INVALID_HANDLE_VALUE); + assert(ab->subdir == INVALID_HANDLE_VALUE); + FindClose(ab->dir); + ab->dir= INVALID_HANDLE_VALUE; #endif - ab->dir= NULL; break; case BACKUP_PHASE_ABORT: break; @@ -793,11 +808,15 @@ int aria_backup_end(THD *thd, const struct backup_target *target, if (ab->status == TRANSLOG_PURGE_DISABLED) translog_enable_purge(); #ifndef _WIN32 - closedir(ab->dir); - closedir(ab->subdir); + if (ab->dir) + closedir(ab->dir); + if (ab->subdir) + closedir(ab->subdir); #else - my_dirend(ab->dir); - my_dirend(ab->subdir); + if (ab->dir != INVALID_HANDLE_VALUE) + FindClose(ab->dir); + if (ab->subdir != INVALID_HANDLE_VALUE) + FindClose(ab->subdir); #endif pthread_mutex_destroy(&ab->mutex); free(ab); From 466fb3c8eb87d4f6f04b4ed3af21c7bb66e8ad87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Aug 2026 12:03:27 +0300 Subject: [PATCH 14/17] fixup! f46d1e4f9443b535572948c08f4288a32ee02f3b --- storage/maria/ma_backup_server.c | 38 +++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c index 38eafb1ed2d19..a6ebb5d203e60 100644 --- a/storage/maria/ma_backup_server.c +++ b/storage/maria/ma_backup_server.c @@ -363,10 +363,9 @@ static int aria_backup_data(const struct backup_target *target, else { consume_subdir: - DIR *const dir= ab->subdir; assert(ab->d); assert(ab->d->d_type == DT_DIR || ab->d->d_type == DT_UNKNOWN); - while ((d= readdir(dir)) != NULL) + while ((d= readdir(ab->subdir)) != NULL) { const char *const name= d->d_name; size_t len; @@ -377,7 +376,7 @@ static int aria_backup_data(const struct backup_target *target, case DT_LNK: break; case DT_UNKNOWN: - if (fstatat(dirfd(dir), name, &sb, 0) || + if (fstatat(dirfd(ab->subdir), name, &sb, 0) || (sb.st_mode & S_IFMT) != S_IFREG) continue; } @@ -406,13 +405,12 @@ static int aria_backup_data(const struct backup_target *target, break; } - assert(dir == ab->subdir); left= 1; if (!d) { + closedir(ab->subdir); ab->d= NULL; ab->subdir= NULL; - closedir(dir); } } #else @@ -445,10 +443,11 @@ static int aria_backup_data(const struct backup_target *target, consume_subdir: do { + const char *const name= ab->sd.cFileName; size_t len; - struct fileinfo *fi= &dir->dir_entry[ab->subdir_consumed++]; - if ((fi->mystat->st_mode & S_IFMT) != S_IFREG || - (len= strlen(fi->name)) < 4 || + if (ab->sd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + continue; + if ((len= strlen(name)) < 4 || /* As noted in MDEV-25854, file names that start with #sql must be excluded from the backup. For example, a call to @@ -457,14 +456,13 @@ static int aria_backup_data(const struct backup_target *target, cleanup_table_after_inplace_alter() deleting a #sql-alter*.frm file before we get a chance to copy it. */ - !memcmp(fi->name, tmp_file_prefix, tmp_file_prefix_length) || - !(*include_p)(fi->name, len)) + !memcmp(name, tmp_file_prefix, tmp_file_prefix_length) || + !(*include_p)(name, len)) continue; /* Consume a file name */ if ((int) sizeof path <= - snprintf(path, sizeof path, "%s/%s", - ab->d.cFileName, ab->sd.cFileName)); + snprintf(path, sizeof path, "%s/%s", ab->d.cFileName, name)); goto name_too_long; filename= path; @@ -553,24 +551,24 @@ static int aria_backup_log(const struct backup_target *target, } left= d != NULL; #else - while (ab->dir_consumed < dir->number_of_files) + do { - struct fileinfo *fi= &dir->dir_entry[ab->dir_consumed++]; - if ((fi->mystat->st_mode & S_IFMT) != S_IFREG || - (strncmp(fi->name, C_STRING_WITH_LEN("aria_log.")) && - strcmp(fi->name, "aria_log_control"))) + const char *const name= ab->d.cFileName; + if (ab->d.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + continue; + if (strncmp(name, C_STRING_WITH_LEN("aria_log.")) && + strcmp(name, "aria_log_control")) continue; else if ((int) sizeof path <= - snprintf(path, sizeof path, "%s/%s", maria_data_root, fi->name)) + snprintf(path, sizeof path, "%s/%s", maria_data_root, name)) { path[(sizeof path) - 1]= '\0'; my_error(ER_TOO_LONG_IDENT, MYF(0), path); goto err_exit; } filename= path; - break; } - left= ab->dir->number_of_files > ab->dir_consumed; + while ((left= FindNextFile(ab->dir, &ab->d)) && !filename); #endif } From 9268c3da4b6b0f62fc9034753d8dfaaabc00cc0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Aug 2026 12:14:45 +0300 Subject: [PATCH 15/17] fixup! 466fb3c8eb87d4f6f04b4ed3af21c7bb66e8ad87 --- storage/maria/ma_backup_server.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c index a6ebb5d203e60..ca8ceb277c299 100644 --- a/storage/maria/ma_backup_server.c +++ b/storage/maria/ma_backup_server.c @@ -640,13 +640,13 @@ void *aria_backup_start(THD *thd, const struct backup_target *target, #ifndef NDEBUG ab= sink->ha_data; #endif +#ifndef _WIN32 assert(ab->dir); assert(!ab->subdir); -#ifndef _WIN32 assert(!ab->d); #else - assert(!ab->dir_consumed); - assert(!ab->subdir_consumed); + assert(ab->dir != INVALID_HANDLE_VALUE); + assert(ab->subdir == INVALID_HANDLE_VALUE); #endif break; case BACKUP_PHASE_FINISH: From d56de2ab9a81ea95998f1b0dc1e62320f530c581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Aug 2026 13:10:12 +0300 Subject: [PATCH 16/17] fixup! 9268c3da4b6b0f62fc9034753d8dfaaabc00cc0d --- mysql-test/main/backup_server.test | 2 ++ storage/maria/ma_backup_server.c | 53 +++++++++++++++++++----------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/mysql-test/main/backup_server.test b/mysql-test/main/backup_server.test index 50cf326d39838..f9985c96b075f 100644 --- a/mysql-test/main/backup_server.test +++ b/mysql-test/main/backup_server.test @@ -1,6 +1,8 @@ --let $datadir=`select @@datadir` --error ER_WRONG_ARGUMENTS evalp BACKUP SERVER TO '$datadir/some_directory'; +--error 0,1 +--rmdir $MYSQLTEST_VARDIR/some_directory evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR --error 21 diff --git a/storage/maria/ma_backup_server.c b/storage/maria/ma_backup_server.c index ca8ceb277c299..36df0b2b5dbff 100644 --- a/storage/maria/ma_backup_server.c +++ b/storage/maria/ma_backup_server.c @@ -158,26 +158,35 @@ static int aria_backup_mkdir(const struct backup_target *target, #ifdef _WIN32 if (!target->path) return 0; - char path[FN_REFLEN]; - if ((int) sizeof path <= - snprintf(path, sizeof path, "%s/%s", target->path, name)) + int ret= snprintf(NULL, 0, "%s/%s", target->path, name) + 1; + char *path= malloc(ret); + if (!path) { my_error(ER_TOO_LONG_IDENT, MYF(0), name); - return -1; + return 1; } + snprintf(path, ret, "%s/%s", target->path, name); if (CreateDirectory(path, NULL)) - return 0; - DWORD err= GetLastError(); - if (err == ERROR_ALREADY_EXISTS) - return 0; - my_osmaperr(err); + ret= 0; + else + { + DWORD err= GetLastError(); + if (err != ERROR_ALREADY_EXISTS) + { + my_osmaperr(err); + my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); + ret= 1; + } + } + free(path); + return ret; #else if (target->fd == -1 || likely(!mkdirat(target->fd, name, 0777)) || errno == EEXIST) return 0; -#endif my_error(ER_CANT_CREATE_FILE, MYF(0), name, errno); return 1; +#endif } #ifndef _WIN32 @@ -242,19 +251,23 @@ static int aria_backup_file(const struct backup_target *target, int ret= -1; if (sink->stream == INVALID_HANDLE_VALUE) { - char dstpath[FN_REFLEN * 3 / 2]; - if ((int) sizeof dstpath <= - snprintf(dstpath, sizeof dstpath, "%s/%s", - target->path, path + dir_prefix)) + int len= snprintf(NULL, 0, "%s/%s", target->path, path + dir_prefix) + 1; + char *dstpath= malloc(len); + if (!dstpath) my_error(ER_TOO_LONG_IDENT, MYF(0), path + dir_prefix); - else if (!CopyFileEx(path, dstpath, NULL, NULL, NULL, - COPY_FILE_NO_BUFFERING)) + else { - my_osmaperr(GetLastError()); - my_error(ER_CANT_CREATE_FILE, MYF(0), dstpath, errno); + snprintf(dstpath, len, "%s/%s", target->path, path + dir_prefix); + if (CopyFileEx(path, dstpath, NULL, NULL, NULL, + COPY_FILE_NO_BUFFERING)) + ret= 0; + else + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_CREATE_FILE, MYF(0), dstpath, errno); + } + free(dstpath); } - else - return 0; } else { From f2d234576626c566966015d71a97acb788240b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Aug 2026 22:30:44 +0300 Subject: [PATCH 17/17] fixup! c909702dfcd2261a73cd0ca6a6ce5703ddda411a --- sql/sql_backup.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc index b97ed1bdcbcee..f7b8f6dcf03ce 100644 --- a/sql/sql_backup.cc +++ b/sql/sql_backup.cc @@ -299,7 +299,7 @@ int append(handle src, backup_fd stream, uint64_t start, uint64_t end) noexcept const int pipe_size{fcntl(stream, F_GETPIPE_SZ)}; # elif defined __FreeBSD__ || defined __APPLE__ // https://unix.stackexchange.com/questions/11946/how-big-is-the-pipe-buffer - constexpr int pipe_size{65535}; + constexpr int pipe_size{65536}; # else constexpr int pipe_size{0}; # endif