diff --git a/debian/control b/debian/control index 66d46aa70..54627f0c9 100644 --- a/debian/control +++ b/debian/control @@ -14,7 +14,7 @@ Build-Depends: libexif-dev (>= 0.6.20), libgail-3-dev, libgirepository1.0-dev (>= 0.9.12), - libglib2.0-dev (>= 2.45.7), + libglib2.0-dev (>= 2.52.0), libglib2.0-doc, libgsf-1-dev, libgstreamer1.0-dev (>= 1.22), @@ -52,7 +52,7 @@ Architecture: any Section: libdevel Depends: gir1.2-nemo-3.0 (= ${binary:Version}), - libglib2.0-dev (>= 2.45.7), + libglib2.0-dev (>= 2.52.0), libgtk-3-dev (>= 3.10), libnemo-extension1 (= ${binary:Version}), ${misc:Depends}, diff --git a/libnemo-private/meson.build b/libnemo-private/meson.build index 7e168cb4c..293ae89f3 100644 --- a/libnemo-private/meson.build +++ b/libnemo-private/meson.build @@ -87,6 +87,7 @@ nemo_private_sources = [ 'nemo-vfs-file.c', 'nemo-widget-action.c', 'nemo-widget-menu-item.c', + 'nemo-writeback-pacer.c', ] nemo_private_deps = [ diff --git a/libnemo-private/nemo-file-operations.c b/libnemo-private/nemo-file-operations.c index 3e8a8efbd..cf9d37998 100644 --- a/libnemo-private/nemo-file-operations.c +++ b/libnemo-private/nemo-file-operations.c @@ -71,6 +71,7 @@ #include "nemo-file-undo-operations.h" #include "nemo-file-undo-manager.h" #include "nemo-job-queue.h" +#include "nemo-writeback-pacer.h" /* TODO: TESTING!!! */ @@ -118,6 +119,8 @@ typedef struct { gchar *target_name; NemoCopyCallback done_callback; gpointer done_callback_data; + NemoWritebackPacer *writeback_pacer; + GError *writeback_error; } CopyMoveJob; typedef struct { @@ -4199,6 +4202,13 @@ copy_file_progress_callback (goffset current_num_bytes, report_copy_progress (pdata->job, pdata->source_info, pdata->transfer_info); + + if (pdata->job->writeback_pacer != NULL && + !nemo_writeback_pacer_add (pdata->job->writeback_pacer, + new_size, + &pdata->job->writeback_error)) { + g_cancellable_cancel (pdata->job->common.cancellable); + } } } @@ -4925,6 +4935,41 @@ copy_files (CopyMoveJob *job, g_free (dest_fs_type); } +static void +finish_writeback_pacer (CopyMoveJob *job, + GFile *dest) +{ + if (job->writeback_pacer == NULL) { + return; + } + + nemo_writeback_pacer_finish (job->writeback_pacer, &job->writeback_error); + job->writeback_pacer = NULL; + + if (job->writeback_error != NULL) { + gchar *primary, *secondary; + + if (job->is_move) { + primary = f (_("Error while moving.")); + secondary = f (_("There was an error moving the file into %F."), dest); + } else { + primary = f (_("Error while copying.")); + secondary = f (_("There was an error copying the file into %F."), dest); + } + + run_error (&job->common, + primary, + secondary, + job->writeback_error->message, + FALSE, + GTK_STOCK_OK, + NULL); + g_cancellable_cancel (job->common.cancellable); + } + + g_clear_error (&job->writeback_error); +} + static gboolean copy_job_done (gpointer user_data) { @@ -4973,6 +5018,7 @@ copy_job (GIOSchedulerJob *io_job, common->io_job = io_job; dest_fs_id = NULL; + dest = NULL; nemo_progress_info_start (common->progress); @@ -4997,11 +5043,12 @@ copy_job (GIOSchedulerJob *io_job, dest, &dest_fs_id, source_info.num_bytes); - g_object_unref (dest); if (job_aborted (common)) { goto aborted; } + job->writeback_pacer = nemo_writeback_pacer_new (dest, common->cancellable); + nemo_progress_info_start (common->progress); memset (&transfer_info, 0, sizeof (transfer_info)); @@ -5010,6 +5057,8 @@ copy_job (GIOSchedulerJob *io_job, &source_info, &transfer_info); aborted: + finish_writeback_pacer (job, dest); + g_clear_object (&dest); g_free (dest_fs_id); @@ -5635,6 +5684,8 @@ move_job (GIOSchedulerJob *io_job, goto aborted; } + job->writeback_pacer = nemo_writeback_pacer_new (job->destination, common->cancellable); + memset (&transfer_info, 0, sizeof (transfer_info)); move_files (job, fallbacks, @@ -5642,6 +5693,7 @@ move_job (GIOSchedulerJob *io_job, &source_info, &transfer_info); aborted: + finish_writeback_pacer (job, job->destination); g_list_free_full (fallbacks, move_file_copy_fallback_free); g_free (dest_fs_id); diff --git a/libnemo-private/nemo-writeback-pacer.c b/libnemo-private/nemo-writeback-pacer.c new file mode 100644 index 000000000..325d45ac9 --- /dev/null +++ b/libnemo-private/nemo-writeback-pacer.c @@ -0,0 +1,311 @@ +/* -*- Mode: C; indent-tabs-mode: nil; c-basic-offset: 4; tab-width: 4 -*- */ + +/* + nemo-writeback-pacer.c - keeps copies to slow destinations in step + with the device + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public + License along with this program; if not, write to the + Free Software Foundation, Inc., 51 Franklin Street - Suite 500, + Boston, MA 02110-1335, USA. +*/ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include + +#include +#include +#include +#include +#include + +#include "nemo-writeback-pacer.h" + +/* Each full window of written data is handed to a helper thread, which syncs + * the filesystem while the copy fills the next one. The copy only waits if + * the previous window is still syncing, so at most two windows are unsynced + * and progress follows the device rather than the page cache. + * + * A window holds about WINDOW_DURATION of data at the rate of the last sync, + * clamped to MIN/MAX_WINDOW_SIZE and growing at most 2x per sync. + * + * A sync can't be interrupted, so a cancelled job stops waiting for it and + * leaves the helper to finish on its own. */ +#define WINDOW_DURATION (G_USEC_PER_SEC / 2) +#define MIN_WINDOW_SIZE (1024 * 1024) +#define MAX_WINDOW_SIZE (64 * 1024 * 1024) +#define CANCEL_CHECK_INTERVAL (G_USEC_PER_SEC / 10) + +struct NemoWritebackPacer { + gint ref_count; + int fd; + GThread *thread; + GCancellable *cancellable; + + GMutex mutex; + GCond cond; + guint64 requested; + guint64 completed; + gboolean stopping; + int sync_errno; + gboolean error_reported; + goffset handed_over; + goffset synced; + goffset last_sync_bytes; + gint64 last_sync_usec; + + goffset window_size; + goffset window_bytes; + gboolean written; +}; + +/* The network types and the FUSE rule are KIO's, from KMountPoint. */ +static gboolean +is_on_network_filesystem (const char *path) +{ + const char * const network_types[] = { + "cifs", "smb3", "smbfs", "nfs", "nfs3", "nfs4", "afs", "ncpfs", + "glusterfs", "ceph", "fuse.curlftpfs", "fuse.sshfs", "fuse.ceph", "9p", + NULL + }; + GUnixMountEntry *mount; + const char *type, *device; + gboolean network; + + mount = g_unix_mount_for (path, NULL); + + if (mount == NULL) { + return FALSE; + } + + type = g_unix_mount_get_fs_type (mount); + device = g_unix_mount_get_device_path (mount); + network = g_strv_contains (network_types, type) || + (g_str_has_prefix (type, "fuse.") && !g_str_has_prefix (device, "/dev/")); + + g_unix_mount_free (mount); + + return network; +} + +static gboolean +is_on_removable_drive (const char *path) +{ + GFile *directory; + GMount *mount; + GDrive *drive; + gboolean removable; + + directory = g_file_new_for_path (path); + mount = g_file_find_enclosing_mount (directory, NULL, NULL); + g_object_unref (directory); + if (mount == NULL) { + return FALSE; + } + + drive = g_mount_get_drive (mount); + removable = drive != NULL && g_drive_is_removable (drive); + + g_clear_object (&drive); + g_object_unref (mount); + + return removable; +} + +static void +pacer_unref (NemoWritebackPacer *pacer) +{ + if (!g_atomic_int_dec_and_test (&pacer->ref_count)) { + return; + } + + close (pacer->fd); + g_object_unref (pacer->cancellable); + g_mutex_clear (&pacer->mutex); + g_cond_clear (&pacer->cond); + g_free (pacer); +} + +static gpointer +sync_thread_func (gpointer data) +{ + NemoWritebackPacer *pacer = data; + + g_mutex_lock (&pacer->mutex); + + while (TRUE) { + guint64 generation; + goffset bytes; + gint64 start; + int result; + + while (pacer->completed == pacer->requested && !pacer->stopping) { + g_cond_wait (&pacer->cond, &pacer->mutex); + } + + if (pacer->completed == pacer->requested || + g_cancellable_is_cancelled (pacer->cancellable)) { + break; + } + + generation = pacer->requested; + bytes = pacer->handed_over - pacer->synced; + pacer->synced = pacer->handed_over; + g_mutex_unlock (&pacer->mutex); + + start = g_get_monotonic_time (); + result = syncfs (pacer->fd) == 0 ? 0 : errno; + + g_mutex_lock (&pacer->mutex); + if (result != 0 && pacer->sync_errno == 0) { + pacer->sync_errno = result; + } + pacer->last_sync_bytes = bytes; + pacer->last_sync_usec = MAX (g_get_monotonic_time () - start, 1); + pacer->completed = generation; + g_cond_broadcast (&pacer->cond); + } + + g_mutex_unlock (&pacer->mutex); + pacer_unref (pacer); + + return NULL; +} + +/* Called with the mutex held. Waits until the generation is synced or the job + * is cancelled, then returns FALSE if a failed sync hasn't been reported yet. */ +static gboolean +wait_for_sync (NemoWritebackPacer *pacer, + guint64 generation, + GError **error) +{ + while (pacer->completed < generation && + !g_cancellable_is_cancelled (pacer->cancellable)) { + g_cond_wait_until (&pacer->cond, &pacer->mutex, + g_get_monotonic_time () + CANCEL_CHECK_INTERVAL); + } + + if (pacer->sync_errno == 0 || pacer->error_reported) { + return TRUE; + } + + pacer->error_reported = TRUE; + g_set_error_literal (error, + G_IO_ERROR, + g_io_error_from_errno (pacer->sync_errno), + g_strerror (pacer->sync_errno)); + return FALSE; +} + +NemoWritebackPacer * +nemo_writeback_pacer_new (GFile *directory, + GCancellable *cancellable) +{ + NemoWritebackPacer *pacer; + char *path, *real_path; + int fd; + + if (!g_file_is_native (directory) || + g_getenv ("NEMO_DISABLE_WRITEBACK_PACING") != NULL) { + return NULL; + } + + path = g_file_get_path (directory); + real_path = realpath (path, NULL); + g_free (path); + + if (real_path == NULL) { + return NULL; + } + + fd = -1; + if (g_getenv ("NEMO_FORCE_WRITEBACK_PACING") != NULL || + is_on_removable_drive (real_path) || + is_on_network_filesystem (real_path)) { + fd = open (real_path, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + } + free (real_path); + + if (fd < 0) { + return NULL; + } + + pacer = g_new0 (NemoWritebackPacer, 1); + pacer->ref_count = 2; + pacer->fd = fd; + pacer->cancellable = g_object_ref (cancellable); + pacer->window_size = MIN_WINDOW_SIZE; + g_mutex_init (&pacer->mutex); + g_cond_init (&pacer->cond); + pacer->thread = g_thread_new ("nemo-writeback", sync_thread_func, pacer); + + return pacer; +} + +gboolean +nemo_writeback_pacer_add (NemoWritebackPacer *pacer, + goffset bytes_written, + GError **error) +{ + goffset target; + gboolean ret; + + pacer->written = TRUE; + pacer->window_bytes += bytes_written; + if (pacer->window_bytes < pacer->window_size) { + return TRUE; + } + + g_mutex_lock (&pacer->mutex); + pacer->handed_over += pacer->window_bytes; + pacer->requested++; + g_cond_broadcast (&pacer->cond); + ret = wait_for_sync (pacer, pacer->requested - 1, error); + + if (pacer->last_sync_usec > 0) { + target = (goffset) ((gdouble) pacer->last_sync_bytes * WINDOW_DURATION / pacer->last_sync_usec); + pacer->window_size = CLAMP (target, + MIN_WINDOW_SIZE, + MIN (pacer->window_size * 2, MAX_WINDOW_SIZE)); + pacer->last_sync_usec = 0; + } + g_mutex_unlock (&pacer->mutex); + + pacer->window_bytes = 0; + + return ret; +} + +gboolean +nemo_writeback_pacer_finish (NemoWritebackPacer *pacer, + GError **error) +{ + gboolean ret; + + g_mutex_lock (&pacer->mutex); + if (pacer->written) { + pacer->requested++; + } + pacer->stopping = TRUE; + g_cond_broadcast (&pacer->cond); + ret = wait_for_sync (pacer, pacer->requested, error); + g_mutex_unlock (&pacer->mutex); + + g_thread_unref (pacer->thread); + pacer_unref (pacer); + + return ret; +} diff --git a/libnemo-private/nemo-writeback-pacer.h b/libnemo-private/nemo-writeback-pacer.h new file mode 100644 index 000000000..e9cd6d6d5 --- /dev/null +++ b/libnemo-private/nemo-writeback-pacer.h @@ -0,0 +1,47 @@ +/* -*- Mode: C; indent-tabs-mode: nil; c-basic-offset: 4; tab-width: 4 -*- */ + +/* + nemo-writeback-pacer.h - keeps copies to slow destinations in step + with the device + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public + License along with this program; if not, write to the + Free Software Foundation, Inc., 51 Franklin Street - Suite 500, + Boston, MA 02110-1335, USA. +*/ + +#ifndef __NEMO_WRITEBACK_PACER_H__ +#define __NEMO_WRITEBACK_PACER_H__ + +#include + +typedef struct NemoWritebackPacer NemoWritebackPacer; + +/* Returns NULL unless the directory is on a removable drive or a network + * filesystem, or NEMO_FORCE_WRITEBACK_PACING is set. Waits stop when the + * cancellable is cancelled. */ +NemoWritebackPacer *nemo_writeback_pacer_new (GFile *directory, + GCancellable *cancellable); + +/* When a window fills, waits for the previous one to sync. A failed sync is + * reported once per pacer, by this or by finish. */ +gboolean nemo_writeback_pacer_add (NemoWritebackPacer *pacer, + goffset bytes_written, + GError **error); + +/* Waits for everything written to reach the device, unless cancelled, then + * frees the pacer. Sync errors after a cancel aren't seen. */ +gboolean nemo_writeback_pacer_finish (NemoWritebackPacer *pacer, + GError **error); + +#endif /* __NEMO_WRITEBACK_PACER_H__ */ diff --git a/meson.build b/meson.build index 9162dab36..f51c22289 100644 --- a/meson.build +++ b/meson.build @@ -72,7 +72,7 @@ endif ################################################################################ # Find dependencies -glib_version = '>=2.45.7' +glib_version = '>=2.52.0' math = cc.find_library('m', required: true)