diff --git a/lang/it.php b/lang/it.php index a2c6626..1847af9 100644 --- a/lang/it.php +++ b/lang/it.php @@ -349,6 +349,7 @@ 'Backup type' => "Tipo di backup", 'Database backup successfully created' => "Backup del database creato con successo", 'Failed to create database backup' => "Impossibile creare il backup del database", + 'Backup was interrupted before it could complete' => "Il backup è stato interrotto prima di essere completato", 'Generate Bulk insert statements' => "Generare istruzioni BULK INSERT", 'Maximum size of SQL statement' => "Dimensione massima di istruzione SQL", 'Show record count with table names' => "Mostra numero di record con i nomi delle tabelle", @@ -459,5 +460,18 @@ 'Yes' => "Sì", 'You have the latest version' => "Hai l'ultima versione", + // backup progress dialog + 'Backup Database' => "Database Backup", + 'Select objects to include in backup' => "Selezionare gli oggetti da includere nel backup", + 'Starting backup' => "Avvio del backup", + 'Backup in progress' => "Backup in corso", + 'Exporting' => "Esportazione", + 'rows' => "righe", + 'Backup complete' => "Backup completato", + 'Backup failed' => "Backup non riuscito", + 'Lost contact with the server while the backup was running' => "Contatto con il server perso durante il backup", + 'Database backup successfully created' => "Backup del database creato con successo", + 'Failed to create database backup' => "Impossibile creare il backup del database", + ); ?> diff --git a/lib/backupstate.php b/lib/backupstate.php new file mode 100644 index 0000000..b09b17f --- /dev/null +++ b/lib/backupstate.php @@ -0,0 +1,163 @@ + $max_age ) + @unlink( $file ); + } + } + + public function __construct( $token ) { + $this->token = $token; + $this->path = self::statePath( $token ); + $this->data = array( + 'state' => 'running', + 'db' => '', + 'file' => '', + 'total' => 0, + 'done' => 0, + 'type' => '', + 'object' => '', + 'rows' => 0, + 'totalrows' => 0, + 'bytes' => 0, + 'started' => microtime(true), + 'updated' => microtime(true), + 'message' => '' + ); + } + + public function begin( $total, $db_name, $file_name ) { + $this->data['total'] = (int) $total; + $this->data['db'] = $db_name; + $this->data['file'] = $file_name; + $this->write( true ); + } + + // moves on to the next object of the backup + public function step( $type, $label ) { + $this->data['done']++; + $this->data['type'] = $type; + $this->data['object'] = $label; + $this->rows_base = $this->data['totalrows']; + $this->data['rows'] = 0; + $this->write( true ); + } + + // number of rows exported so far for the current object + public function rows( $count ) { + $this->data['rows'] = (int) $count; + $this->data['totalrows'] = $this->rows_base + (int) $count; + $this->write(); + } + + // called back by the Output class as the dump is written to disk + public function setBytes( $bytes ) { + $this->data['bytes'] = (int) $bytes; + $this->write(); + } + + public function isFinished() { + return $this->data['state'] != 'running'; + } + + public function finish( $state, $message, $extra = array() ) { + $this->data['state'] = $state; // 'done' or 'error' + $this->data['message'] = $message; + foreach( $extra as $key => $value ) + $this->data[$key] = $value; + $this->write( true ); + } + + public function getData() { + return $this->data; + } + + private function write( $force = false ) { + $now = microtime(true); + if ( !$force && ($now - $this->last_write) < $this->write_interval ) + return; + + $this->last_write = $now; + $this->data['updated'] = $now; + + $json = json_encode( $this->data ); + // write to a scratch file first, so a poll never sees a half written state + $tmp = $this->path . '.' . getmypid() . '.tmp'; + if ( @file_put_contents( $tmp, $json ) !== false && @rename( $tmp, $this->path ) ) + return; + + @unlink( $tmp ); + @file_put_contents( $this->path, $json, LOCK_EX ); + } + } +} +?> diff --git a/lib/export/export.php b/lib/export/export.php index b5c6f3e..a2f841d 100644 --- a/lib/export/export.php +++ b/lib/export/export.php @@ -55,6 +55,10 @@ function exportTable($sql, $options) { $id = 0; $field_info = NULL; + // optional callback, invoked after every batch with the row count exported so far + $progress = isset($options['progress']) ? $options['progress'] : NULL; + $rows_done = 0; + while(1) { $tempSql = $sql; if ($applyLimit) @@ -72,8 +76,12 @@ function exportTable($sql, $options) { while($row = $this->db->fetchRow("_temp", 'num')) { print $this->driver->createLine($row, $field_info); + $rows_done++; } + if ($progress) + call_user_func($progress, $rows_done); + if ($numRows == 0 || !$applyLimit) break; diff --git a/lib/output.php b/lib/output.php index cf0158f..7d9e763 100644 --- a/lib/output.php +++ b/lib/output.php @@ -16,9 +16,15 @@ define("CLASS_OUTPUT_INCLUDED", "1"); class Output { + // size of the buffer that is written out to the file in one go + const CHUNK_SIZE = 65536; + public $file; public $compression; public $file_handle; + public $bytes = 0; // bytes handed over to the file so far + public $progress = null; // optional object with a setBytes() method + private $buffering = false; // controls output buffering public static function buffer() { @@ -78,7 +84,15 @@ public function __construct( $file, $compression = false ) { } else { $this->file_handle = fopen( $file, 'wb' ); } - ob_start( array( $this, 'output_callback' ) ); + + // nothing to redirect the output to, leave the buffering alone so that the + // caller can still report the failure to the browser + if ( !$this->file_handle ) + return; + + // flush every CHUNK_SIZE bytes instead of holding the whole dump in memory + ob_start( array( $this, 'output_callback' ), self::CHUNK_SIZE ); + $this->buffering = true; } public function __destruct() { @@ -91,12 +105,17 @@ public function is_valid() { // only works if output is being redirected with compression public function end() { - @ob_end_flush(); + // only close the buffer we started ourselves, end() is also called by the destructor + if ( $this->buffering ) { + @ob_end_flush(); + $this->buffering = false; + } + if ( $this->file_handle ) { if ( $this->compression == 'gz' ) { gzclose( $this->file_handle ); } - if ( $this->compression == 'bz' ) { + else if ( $this->compression == 'bz' ) { bzclose( $this->file_handle ); } else { fclose( $this->file_handle ); @@ -114,6 +133,12 @@ public function output_callback( $buffer ) { } else { fwrite( $this->file_handle, $buffer ); } + + $this->bytes += strlen( $buffer ); + if ( $this->progress ) + $this->progress->setBytes( $this->bytes ); + + return ''; // nothing of this goes to the browser } } } diff --git a/modules/backup.php b/modules/backup.php index 7e3fa18..3af7c4d 100644 --- a/modules/backup.php +++ b/modules/backup.php @@ -27,4 +27,55 @@ function processRequest(&$db) { echo view( array($folder.'/backup', 'backup'), $replace, $object_list); } + /** + * Progress of a running backup, polled by the dialog through status.php while the + * backup request itself is still busy writing the dump. + * + * NOTE: status.php is a minimal bootstrap, neither v() nor __() exist here. + */ + function getModuleStatus( $id ) { + include_once(BASE_PATH . "/lib/backupstate.php"); + + $status = array('c' => 0, 'r' => 0, 's' => 0, 'state' => 'unknown'); + + $token = BackupState::sanitizeToken( $id ); + $owned = ( $token !== false && $token === Session::get('backup', 'token') ); + + // this is polled once per second, do not sit on the session lock while doing so + Session::close(); + + // only report on the backup started by this very session + if ( !$owned ) + return $status; + + $data = BackupState::read( $token ); + if ( !is_array($data) ) { + // the backup request has not written its first update yet + $status['s'] = 1; + $status['state'] = 'starting'; + return $status; + } + + $percent = 0; + if ( $data['total'] > 0 ) + $percent = (int) floor( $data['done'] / $data['total'] * 100 ); + if ( $data['state'] == 'running' && $percent > 99 ) + $percent = 99; // the last object is still being written + if ( $data['state'] == 'done' ) + $percent = 100; + + $status['c'] = $percent; + $status['s'] = 1; + $status['r'] = $data['state'] == 'running' ? 0 : 1; + $status['state'] = $data['state']; + $status['elapsed'] = (int) ( microtime(true) - $data['started'] ); + + foreach( array('done', 'total', 'type', 'object', 'rows', 'totalrows', 'bytes', 'file', 'message') as $key ) + $status[$key] = isset($data[$key]) ? $data[$key] : ''; + + $status['size'] = isset($data['size']) ? $data['size'] : 0; + + return $status; + } + ?> \ No newline at end of file diff --git a/modules/download.php b/modules/download.php index 1f46ded..611f4bb 100644 --- a/modules/download.php +++ b/modules/download.php @@ -14,6 +14,19 @@ function processRequest(&$db) { set_time_limit(0); } + include_once(BASE_PATH . "/lib/backupstate.php"); + + // the browser polls status.php while this request is still running. The token has to + // be registered in the session before we release it, otherwise the poll cannot + // tell whether the backup belongs to the user asking about it + $backup_token = BackupState::sanitizeToken( v($_REQUEST['token']) ); + if ( $backup_token !== false ) { + Session::set('backup', 'token', $backup_token); + // the dump is written to a file, so a disconnected browser (dialog closed, + // proxy timeout) is no reason to throw away a backup that is halfway done + ignore_user_abort(true); + } + Session::close(); switch( $_REQUEST['id'] ) { @@ -22,20 +35,55 @@ function processRequest(&$db) { $compression = v($_REQUEST['compression']); $filename = v($_REQUEST['filename']); $file = get_backup_filename( $compression, $filename ); + + $state = false; + if ( $backup_token !== false ) { + BackupState::cleanup(); + $state = new BackupState( $backup_token ); + $state->begin( countBackupObjects($db), Session::get('db', 'name'), $file ? basename($file) : '' ); + // a fatal error (memory limit, killed worker) must not leave the dialog polling forever + register_shutdown_function( 'backupShutdown', $state ); + } + if ( $file ) { include_once(BASE_PATH . "/lib/output.php"); $output = new Output( $file, $compression ); $message = '
'; if ( $output->is_valid() ) { - downloadDatabase($db, false); + $output->progress = $state ? $state : null; + $written = downloadDatabase($db, false, $state); $output->end(); + + if ( $written === false ) { + @unlink( $file ); // nothing was selected, do not leave an empty backup behind + $message = ''; + if ( $state ) + $state->finish( 'error', __('Select objects to include in backup') ); + } + else if ( $state ) { + clearstatcache(); + $size = @filesize( $file ); + $state->finish( 'done', __('Database backup successfully created'), + array( 'size' => $size === false ? 0 : $size ) ); + } } else { $message = ''; + if ( $state ) + $state->finish( 'error', __('Backup folder does not exist or is not writable') ); } } else { $message = ''; + if ( $state ) + $state->finish( 'error', __('Invalid filename format') ); + } + + if ( $state ) { + // the dialog stays where it is and shows the result, no page reload + header('Content-Type: application/json; charset=utf-8'); + echo json_encode( $state->getData() ); + } else { + echo view( 'backup', array( 'MESSAGE' => $message, 'FILENAME' => htmlspecialchars($filename) ), $db->getObjectList() ); } - echo view( 'backup', array( 'MESSAGE' => $message, 'FILENAME' => htmlspecialchars($filename) ), $db->getObjectList() ); } break; case 'exportres': { downloadResults($db); @@ -109,7 +157,41 @@ function downloadTable(&$db, $table) { } - function downloadDatabase(&$db, $headers = true) { + // number of objects the backup is going to write, used to show a meaningful progress bar + function countBackupObjects(&$db) { + $total = 0; + if ( is_array(v($_POST["tables"])) ) + $total += count($_POST["tables"]); + + $export_type = v($_REQUEST["exptype"]); + if ($export_type == "all" || $export_type == "struct") { + $object_types = $db->getObjectTypes(); + unset($object_types[0]); // tables are already counted above + foreach($object_types as $type) { + if ( is_array(v($_POST[$type])) ) + $total += count($_POST[$type]); + } + } + + return $total; + } + + // last resort reporting: if the request dies before the backup is marked as finished, + // record why, so the dialog shows an error instead of polling forever + function backupShutdown( $state ) { + if ( $state->isFinished() ) + return; + + $message = __('Backup was interrupted before it could complete'); + $error = error_get_last(); + $fatal = array(E_ERROR, E_PARSE, E_CORE_ERROR, E_COMPILE_ERROR, E_USER_ERROR); + if ( is_array($error) && in_array($error['type'], $fatal) ) + $message .= ': ' . $error['message']; + + $state->finish( 'error', $message ); + } + + function downloadDatabase(&$db, $headers = true, $state = false) { // don't make POST as REQUEST here. it won't work :P if ( !( is_array(v($_POST["tables"])) || is_array(v($_POST["views"])) || is_array(v($_POST["procs"])) ||is_array(v($_POST["funcs"])) || is_array(v($_POST["triggers"])) ||is_array(v($_POST["events"])) ) ) @@ -133,12 +215,18 @@ function downloadDatabase(&$db, $headers = true) { 'bulkinsert' => v($_REQUEST['bulkinsert']), 'bulksize' => v($_REQUEST['bulklimit']) == 'on' ? v($_REQUEST['bulksize'])*1024 : 0 ); + if ( $state ) + $options['progress'] = array($state, 'rows'); + foreach($tables as $table_name) { // is this table required in export? $key = array_search($table_name, $_POST["tables"]); if ($key === FALSE) continue; + if ( $state ) + $state->step('table', $table_name); + // -- -truncate command -- if (v($_REQUEST["emptycmd"]) == "on") { echo "\n" . $db->getTruncateCommand( $table_name ) . ";\n"; @@ -187,7 +275,7 @@ function downloadDatabase(&$db, $headers = true) { if (is_array(v($_POST[$type])) && count($_POST[$type]) > 0) { $func = 'get' . ucfirst( $type ); $name = substr($type, 0, -1); - exportObject($db, $name, $_POST[$type], $db->$func()); + exportObject($db, $name, $_POST[$type], $db->$func(), $state); } } } @@ -197,12 +285,15 @@ function downloadDatabase(&$db, $headers = true) { // ===================================== - function exportObject(&$db, $name, $list, $tables) { + function exportObject(&$db, $name, $list, $tables, $state = false) { foreach($tables as $table_name) { $key = array_search($table_name, $list); if ($key === FALSE) continue; + if ( $state ) + $state->step($name, $table_name); + if (v($_REQUEST["dropcmd"]) == "on") print "\ndrop $name if exists " . $db->quote($table_name) . ";\n"; diff --git a/modules/views/backup.php b/modules/views/backup.php index 58b6991..a54f906 100644 --- a/modules/views/backup.php +++ b/modules/views/backup.php @@ -5,18 +5,29 @@ div.objhead { background-color:#ececec; padding: 5px; margin: 0 0 3px 0 } span.toggler { display:inline-block; float:right; cursor: pointer; font-size:16px; margin: -5px 0 0 0 } div.obj { padding:5px; margin:0 0 0 20px } + + div#backup_progress { display:none; margin:3px 3px 6px 3px; padding:6px 10px } + div#backup_bar { height:14px } + div#backup_stage { margin:6px 0 0 0; font-weight:bold; white-space:nowrap; overflow:hidden; text-overflow:ellipsis } + div#backup_detail { margin:2px 0 0 0; color:#666 }|
. . -
+
|
@@ -98,9 +109,165 @@ } ?> +/* + * The backup is written on the server and can run for a long time. Instead of submitting + * the form into this iframe and staring at an unchanged page until php is done, the request + * is sent in the background and the progress is polled from status.php. + */ +var backupToken = ''; +var backupTimer = null; +var backupRunning = false; +var backupPollFails = 0; + +function backupToken_new() { + var chars = 'abcdefghijklmnopqrstuvwxyz0123456789', s = ''; + for (var i = 0; i < 20; i++) + s += chars.charAt(Math.floor(Math.random() * chars.length)); + return s; +} + +function backupEscape(text) { + return $('').text(text === null || text === undefined ? '' : text).html(); +} + +function backupSize(bytes) { + bytes = bytes ? bytes : 0; + var units = ['B', 'KB', 'MB', 'GB', 'TB'], i = 0; + while (bytes >= 1024 && i < units.length - 1) { bytes = bytes / 1024; i++; } + return (i === 0 ? bytes : bytes.toFixed(1)) + ' ' + units[i]; +} + +function backupTime(seconds) { + seconds = Math.max(0, Math.round(seconds ? seconds : 0)); + var m = Math.floor(seconds / 60); + return (m > 0 ? m + 'm ' : '') + (seconds % 60) + 's'; +} + +function backupMessage(text, cls) { + $('#backup_message').html(''); +} + +function backupStop() { + backupRunning = false; + if (backupTimer) { window.clearTimeout(backupTimer); backupTimer = null; } + $('#btn_export').button('enable'); +} + +function backupStart() { + if (backupRunning) + return; + + if ($('#db_objects input[type=checkbox]:checked').not('.selectall').length == 0) { + jAlert(__('Select objects to include in backup'), __('Backup Database')); + return; + } + + backupToken = backupToken_new(); + backupRunning = true; + backupPollFails = 0; + $('#backup_token').val(backupToken); + + var frm = document.frmquery; + frm.type.value = 'dl'; + frm.id.value = 'backup'; + frm.name.value = ''; + frm.query.value = ''; + + $('#btn_export').button('disable'); + $('#backup_progress').show(); + $('#backup_bar').progressbar({ value: 0 }); + $('#backup_stage').html(__('Starting backup') + '...'); + $('#backup_detail').html(' '); + backupMessage(__('Backup in progress'), 'ui-state-highlight'); + + $.ajax({ type: 'POST', + url: '?', + data: 'q=wrkfrm&' + $('#frmquery').serialize(), + dataType: 'json', + success: function(res) { if (res) backupDone(res); }, + // a proxy may drop this connection while php keeps working, so the poll, + // not this request, decides when the backup is over + error: function() { backupPoll(); } + }); + + backupTimer = window.setTimeout(backupPoll, 700); +} + +function backupPoll() { + if (!backupRunning) + return; + + $.ajax({ type: 'GET', + url: 'status.php?type=backup&id=' + backupToken + '&_=' + (new Date()).getTime(), + dataType: 'json', + success: function(res) { + if (!backupRunning) + return; + backupPollFails = 0; + if (res && res.s == 1) { + if (res.state == 'done' || res.state == 'error') { + backupDone(res); + return; + } + backupProgress(res); + } + backupTimer = window.setTimeout(backupPoll, 1000); + }, + error: function() { + if (!backupRunning) + return; + if (++backupPollFails > 10) { + backupStop(); + $('#backup_stage').html(__('Backup failed')); + backupMessage(__('Lost contact with the server while the backup was running'), 'ui-state-error'); + return; + } + backupTimer = window.setTimeout(backupPoll, 2000); + } + }); +} + +function backupProgress(res) { + if (res.state == 'starting') { + $('#backup_stage').html(__('Starting backup') + '...'); + return; + } + + $('#backup_bar').progressbar('value', res.c ? res.c : 0); + + var label = __('Exporting') + ' ' + res.done + '/' + res.total; + if (res.object) + label += ' – ' + backupEscape(res.type + ' ' + res.object); + $('#backup_stage').html(label); + + $('#backup_detail').html( backupSize(res.bytes) + ' · ' + (res.totalrows ? res.totalrows : 0) + ' ' + __('rows') + + ' · ' + backupTime(res.elapsed) ); +} + +function backupDone(res) { + if (!backupRunning) // the poll and the request itself both report the result + return; + backupStop(); + + var elapsed = (res.elapsed !== undefined) ? res.elapsed : (res.updated - res.started); + + if (res.state == 'error') { + $('#backup_stage').html(__('Backup failed')); + $('#backup_detail').html(' '); + backupMessage(res.message ? res.message : __('Failed to create database backup'), 'ui-state-error'); + return; + } + + $('#backup_bar').progressbar('value', 100); + $('#backup_stage').html(__('Backup complete')); + $('#backup_detail').html( backupEscape(res.file) + ' · ' + backupSize(res.size ? res.size : res.bytes) + + ' · ' + (res.totalrows ? res.totalrows : 0) + ' ' + __('rows') + ' · ' + backupTime(elapsed) ); + backupMessage(res.message ? res.message : __('Database backup successfully created'), 'ui-state-highlight'); +} + $(function() { - $('#popup_overlay').remove(); // we do not want to show the popup overlay when form is submitted - $('#btn_export').button().click(function() { exportBackup() }); + $('#popup_overlay').remove(); // progress is reported in the dialog itself, no blocking overlay + $('#btn_export').button().click(function() { backupStart() }); 0 ) { |