Show progress while a server side backup runs (+ fix output buffering bugs)

lang/it.php

@@ -349,6 +349,7 @@
 		'Backup type' => "Tipo di backup",
 		'Database backup successfully created' => "Backup del database creato con successo",
 		'Failed to create database backup' => "Impossibile creare il backup del database",
+		'Backup was interrupted before it could complete' => "Il backup è stato interrotto prima di essere completato",
 		'Generate Bulk insert statements' => "Generare istruzioni BULK INSERT",
 		'Maximum size of SQL statement' => "Dimensione massima di istruzione SQL",
 		'Show record count with table names' => "Mostra numero di record con i nomi delle tabelle",
@@ -459,5 +460,18 @@
 		'Yes' => "Sì",
 		'You have the latest version' => "Hai l'ultima versione",
 
+		// backup progress dialog
+		'Backup Database' => "Database Backup",
+		'Select objects to include in backup' => "Selezionare gli oggetti da includere nel backup",
+		'Starting backup' => "Avvio del backup",
+		'Backup in progress' => "Backup in corso",
+		'Exporting' => "Esportazione",
+		'rows' => "righe",
+		'Backup complete' => "Backup completato",
+		'Backup failed' => "Backup non riuscito",
+		'Lost contact with the server while the backup was running' => "Contatto con il server perso durante il backup",
+		'Database backup successfully created' => "Backup del database creato con successo",
+		'Failed to create database backup' => "Impossibile creare il backup del database",
+
 	);
 ?>

lib/backupstate.php

@@ -0,0 +1,163 @@
+<?php
+/**
+ * This file is a part of MyWebSQL package
+ * tracks the progress of a server side backup, so that the browser can poll it
+ * while the backup request is still running (see status.php / modules/backup.php)
+ *
+ * @file:      lib/backupstate.php
+ * @author     Samnan ur Rehman
+ * @copyright  (c) 2008-2014 Samnan ur Rehman
+ * @web        https://github.com/Samnan/MyWebSQL
+ * @license    https://github.com/Samnan/MyWebSQL/license
+ */
+
+if (!defined("CLASS_BACKUPSTATE_INCLUDED"))
+{
+	define("CLASS_BACKUPSTATE_INCLUDED", "1");
+
+	// NOTE: this class is also loaded from status.php, where neither v() nor __()
+	//       are available. Keep it free of those helpers.
+
+	class BackupState {
+		private $token;
+		private $path;
+		private $data;
+		private $last_write = 0;
+		private $write_interval = 0.4;	// seconds between two throttled updates
+		private $rows_base = 0;			// rows counted before the current object
+
+		// only simple tokens are accepted, they end up in a file name
+		public static function sanitizeToken( $token ) {
+			if ( !is_string($token) )
+				return false;
+			return preg_match('/^[A-Za-z0-9]{8,40}$/', $token) ? $token : false;
+		}
+
+		public static function folder() {
+			$dir = BASE_PATH . '/tmp/';
+			if ( is_dir($dir) && is_writable($dir) )
+				return $dir;
+			return rtrim(sys_get_temp_dir(), '/' . DIRECTORY_SEPARATOR) . '/';
+		}
+
+		public static function statePath( $token ) {
+			return self::folder() . 'backup-' . $token . '.json';
+		}
+
+		// returns the state array, or false when there is nothing (yet) to report
+		public static function read( $token ) {
+			$token = self::sanitizeToken( $token );
+			if ( $token === false )
+				return false;
+
+			$path = self::statePath( $token );
+			if ( !file_exists($path) )
+				return false;
+
+			$raw = @file_get_contents( $path );
+			if ( $raw === false || $raw === '' )
+				return false;
+
+			$data = json_decode( $raw, true );
+			// a half written file is not an error, the caller simply keeps the previous values
+			return is_array($data) ? $data : false;
+		}
+
+		// removes state files left behind by aborted requests
+		public static function cleanup( $max_age = 86400 ) {
+			$files = @glob( self::folder() . 'backup-*.json' );
+			if ( !is_array($files) )
+				return;
+			$now = time();
+			foreach( $files as $file ) {
+				if ( $now - @filemtime($file) > $max_age )
+					@unlink( $file );
+			}
+		}
+
+		public function __construct( $token ) {
+			$this->token = $token;
+			$this->path = self::statePath( $token );
+			$this->data = array(
+				'state'     => 'running',
+				'db'        => '',
+				'file'      => '',
+				'total'     => 0,
+				'done'      => 0,
+				'type'      => '',
+				'object'    => '',
+				'rows'      => 0,
+				'totalrows' => 0,
+				'bytes'     => 0,
+				'started'   => microtime(true),
+				'updated'   => microtime(true),
+				'message'   => ''
+			);
+		}
+
+		public function begin( $total, $db_name, $file_name ) {
+			$this->data['total'] = (int) $total;
+			$this->data['db'] = $db_name;
+			$this->data['file'] = $file_name;
+			$this->write( true );
+		}
+
+		// moves on to the next object of the backup
+		public function step( $type, $label ) {
+			$this->data['done']++;
+			$this->data['type'] = $type;
+			$this->data['object'] = $label;
+			$this->rows_base = $this->data['totalrows'];
+			$this->data['rows'] = 0;
+			$this->write( true );
+		}
+
+		// number of rows exported so far for the current object
+		public function rows( $count ) {
+			$this->data['rows'] = (int) $count;
+			$this->data['totalrows'] = $this->rows_base + (int) $count;
+			$this->write();
+		}
+
+		// called back by the Output class as the dump is written to disk
+		public function setBytes( $bytes ) {
+			$this->data['bytes'] = (int) $bytes;
+			$this->write();
+		}
+
+		public function isFinished() {
+			return $this->data['state'] != 'running';
+		}
+
+		public function finish( $state, $message, $extra = array() ) {
+			$this->data['state'] = $state;			// 'done' or 'error'
+			$this->data['message'] = $message;
+			foreach( $extra as $key => $value )
+				$this->data[$key] = $value;
+			$this->write( true );
+		}
+
+		public function getData() {
+			return $this->data;
+		}
+
+		private function write( $force = false ) {
+			$now = microtime(true);
+			if ( !$force && ($now - $this->last_write) < $this->write_interval )
+				return;
+
+			$this->last_write = $now;
+			$this->data['updated'] = $now;
+
+			$json = json_encode( $this->data );
+			// write to a scratch file first, so a poll never sees a half written state
+			$tmp = $this->path . '.' . getmypid() . '.tmp';
+			if ( @file_put_contents( $tmp, $json ) !== false && @rename( $tmp, $this->path ) )
+				return;
+
+			@unlink( $tmp );
+			@file_put_contents( $this->path, $json, LOCK_EX );
+		}
+	}
+}
+?>

lib/export/export.php

@@ -55,6 +55,10 @@ function exportTable($sql, $options) {
 		$id = 0;
 		$field_info = NULL;
 
+		// optional callback, invoked after every batch with the row count exported so far
+		$progress = isset($options['progress']) ? $options['progress'] : NULL;
+		$rows_done = 0;
+
 		while(1) {
 			$tempSql = $sql;
 			if ($applyLimit)
@@ -72,8 +76,12 @@ function exportTable($sql, $options) {
 
 			while($row = $this->db->fetchRow("_temp", 'num')) {
 				print $this->driver->createLine($row, $field_info);
+				$rows_done++;
 			}
 
+			if ($progress)
+				call_user_func($progress, $rows_done);
+
 			if ($numRows == 0 || !$applyLimit)
 				break;
 

lib/output.php

@@ -16,9 +16,15 @@
 	define("CLASS_OUTPUT_INCLUDED", "1");
 
 	class Output {
+		// size of the buffer that is written out to the file in one go
+		const CHUNK_SIZE = 65536;
+
 		public $file;
 		public $compression;
 		public $file_handle;
+		public $bytes = 0;			// bytes handed over to the file so far
+		public $progress = null;	// optional object with a setBytes() method
+		private $buffering = false;
 
 		// controls output buffering
 		public static function buffer() {
@@ -78,7 +84,15 @@ public function __construct( $file, $compression = false ) {
 			} else {
 				$this->file_handle = fopen( $file, 'wb' );
 			}
-			ob_start( array( $this, 'output_callback' ) );
+
+			// nothing to redirect the output to, leave the buffering alone so that the
+			// caller can still report the failure to the browser
+			if ( !$this->file_handle )
+				return;
+
+			// flush every CHUNK_SIZE bytes instead of holding the whole dump in memory
+			ob_start( array( $this, 'output_callback' ), self::CHUNK_SIZE );
+			$this->buffering = true;
 		}
 
 		public function __destruct() {
@@ -91,12 +105,17 @@ public function is_valid() {
 
 		// only works if output is being redirected with compression
 		public function end() {
-			@ob_end_flush();
+			// only close the buffer we started ourselves, end() is also called by the destructor
+			if ( $this->buffering ) {
+				@ob_end_flush();
+				$this->buffering = false;
+			}
+
 			if ( $this->file_handle ) {
 				if ( $this->compression == 'gz' ) {
 					gzclose( $this->file_handle );
 				}
-				if ( $this->compression == 'bz' ) {
+				else if ( $this->compression == 'bz' ) {
 					bzclose( $this->file_handle );
 				} else {
 					fclose( $this->file_handle );
@@ -114,6 +133,12 @@ public function output_callback( $buffer ) {
 			} else {
 				fwrite( $this->file_handle, $buffer );
 			}
+
+			$this->bytes += strlen( $buffer );
+			if ( $this->progress )
+				$this->progress->setBytes( $this->bytes );
+
+			return '';	// nothing of this goes to the browser
 		}
 	}
 }

modules/backup.php

@@ -27,4 +27,55 @@ function processRequest(&$db) {
 		echo view( array($folder.'/backup', 'backup'), $replace, $object_list);
 	}
 
+	/**
+	 * Progress of a running backup, polled by the dialog through status.php while the
+	 * backup request itself is still busy writing the dump.
+	 *
+	 * NOTE: status.php is a minimal bootstrap, neither v() nor __() exist here.
+	 */
+	function getModuleStatus( $id ) {
+		include_once(BASE_PATH . "/lib/backupstate.php");
+
+		$status = array('c' => 0, 'r' => 0, 's' => 0, 'state' => 'unknown');
+
+		$token = BackupState::sanitizeToken( $id );
+		$owned = ( $token !== false && $token === Session::get('backup', 'token') );
+
+		// this is polled once per second, do not sit on the session lock while doing so
+		Session::close();
+
+		// only report on the backup started by this very session
+		if ( !$owned )
+			return $status;
+
+		$data = BackupState::read( $token );
+		if ( !is_array($data) ) {
+			// the backup request has not written its first update yet
+			$status['s'] = 1;
+			$status['state'] = 'starting';
+			return $status;
+		}
+
+		$percent = 0;
+		if ( $data['total'] > 0 )
+			$percent = (int) floor( $data['done'] / $data['total'] * 100 );
+		if ( $data['state'] == 'running' && $percent > 99 )
+			$percent = 99;		// the last object is still being written
+		if ( $data['state'] == 'done' )
+			$percent = 100;
+
+		$status['c'] = $percent;
+		$status['s'] = 1;
+		$status['r'] = $data['state'] == 'running' ? 0 : 1;
+		$status['state'] = $data['state'];
+		$status['elapsed'] = (int) ( microtime(true) - $data['started'] );
+
+		foreach( array('done', 'total', 'type', 'object', 'rows', 'totalrows', 'bytes', 'file', 'message') as $key )
+			$status[$key] = isset($data[$key]) ? $data[$key] : '';
+
+		$status['size'] = isset($data['size']) ? $data['size'] : 0;
+
+		return $status;
+	}
+
 ?>