diff --git a/docs/cn/streaming_log.md b/docs/cn/streaming_log.md index a58ab0cdf9..198ca141c3 100644 --- a/docs/cn/streaming_log.md +++ b/docs/cn/streaming_log.md @@ -272,6 +272,10 @@ CHECK(x > y); // Check failed: x > y. 如果重命名失败(比如日志所在目录不可写),brpc会打印一条错误,然后等文件又长了一个`log_rotate_size_mb`再重试,而不是每写一条日志都去关闭、重开、重命名一次文件。期间日志继续追加到当前文件,不会丢。如果某个历史文件因为被其它进程占用等原因移不走,切分会整体放弃这一次操作,而不会用当前文件把它覆盖掉,因此不会悄悄丢掉一代日志。如果发现文件比失败时更小,说明它已经被外部替换或截断,切分立刻恢复正常。 +切分过程中不会提前关闭日志句柄:POSIX允许rename一个正在打开的文件,所以即使rename失败,或者路径因为权限收紧(比如运行中文件变成0400、目录变成0500)或进程降权而无法再次打开,已有的句柄仍然可以继续写,日志不会丢。新建的文件会沿用被替换文件的权限,一个0600的私有日志不会因为切分变成0644而对其他用户可读。 + +历史文件的清理按目录里实际存在的文件进行,而不是假设序号连续:某次移位中途失败留下的空洞不会让更老的日志永久残留,`log_rotate_max_backups`设得再大也不会让每次切分去尝试那么多次rename。 + > 不要在同一个日志文件上同时使用logrotate等外部切分工具。外部工具把文件改名之后,brpc下一次写入会重新创建原路径的文件,两边的切分规则会互相干扰,保留下来的历史文件数也不再受`log_rotate_max_backups`控制。二选一。 两个flag都可以在运行时通过`/flags`修改。 diff --git a/docs/en/streaming_log.md b/docs/en/streaming_log.md index 5914a2a0bc..0afb755184 100644 --- a/docs/en/streaming_log.md +++ b/docs/en/streaming_log.md @@ -274,6 +274,10 @@ Rotation is disabled by default, so upgrading changes neither the file names nor If the rename fails, for example because the directory holding the log file is not writable, brpc prints an error and retries only once the file has grown by another `log_rotate_size_mb`, instead of closing, reopening and renaming the file for every single log. Logs keep being appended to the current file in the meantime, none of them are lost. A backup that cannot be renamed out of the way, for example because another process holds it open, stops the rotation altogether rather than being overwritten with the current file, so no generation is silently dropped. If the file turns out to be smaller than it was when rotation failed, it has been replaced or truncated from the outside and rotation resumes immediately. +The handle is never closed ahead of the rename: POSIX allows renaming a file that is still open, so even when the rename fails, or when the path can no longer be opened because permissions were tightened (the file turned 0400 or the directory 0500 under a running process) or privileges were dropped, the handle that is already open keeps accepting logs and none of them are lost. The fresh file is created with the permissions of the file it replaces, so a private 0600 log file does not turn into a world readable 0644 one on the first rotation. + +Old backups are cleaned up by listing what is really in the directory instead of assuming that the indexes are contiguous: a hole left behind by a partly failed shift cannot keep the older generations around forever, and a large `log_rotate_max_backups` does not make every rotation attempt that many renames. + > Do not run an external rotator such as logrotate on the same log file. Once an external tool renames the file, the next write recreates it under the original name, the two sets of rotation rules interfere with each other and the number of files kept around is no longer bounded by `log_rotate_max_backups`. Use one or the other. Both flags are modifiable at run time via `/flags`. diff --git a/src/butil/logging.cc b/src/butil/logging.cc index e39e5efeaf..29f63240d9 100644 --- a/src/butil/logging.cc +++ b/src/butil/logging.cc @@ -83,9 +83,8 @@ typedef pthread_mutex_t* MutexHandle; #if defined(OS_POSIX) #include "butil/errno.h" #include "butil/fd_guard.h" -#endif -#if defined(OS_LINUX) -#include +#include +#include // open() with the mode of the file rotation replaces #endif #if defined(OS_ANDROID) @@ -205,6 +204,12 @@ FileHandle log_file = nullptr; // LoggingLock just like `log_file' itself. int64_t log_file_size = 0; +// Permissions of the log file, or -1 when they are unknown. A file recreated +// by rotation is created with them instead of falling back to 0666 & ~umask, +// which would turn a private 0600 log file into a world readable one the +// first time it is rotated. Guarded by LoggingLock. +int64_t log_file_mode = -1; + // Size of the log file when rotation last failed, -1 when the last attempt // succeeded. The size that triggered a failed rotation does not change by // itself, so an unconditional retry would close, reopen and rename the file @@ -397,6 +402,40 @@ MutexHandle LoggingLock::log_mutex = nullptr; pthread_mutex_t LoggingLock::log_mutex = PTHREAD_MUTEX_INITIALIZER; #endif +#if defined(OS_POSIX) +// Open the log file for appending, creating it with `mode' when it does not +// exist yet. Rotation recreates the file right after renaming the old one +// away, and creating it with the default 0666 & ~umask would silently make a +// private log file readable by everybody else. A file that already exists +// keeps the permissions it has, they are never changed behind the user's +// back. +FileHandle OpenLogFile(const PathString& path, int64_t mode) { + if (mode < 0) { + return fopen(path.c_str(), "a"); + } + // O_EXCL tells us whether the file was created here, which is also what + // keeps the fchmod() below from touching a file somebody else created. + const int fd = ::open(path.c_str(), O_CREAT | O_EXCL | O_APPEND | O_WRONLY, + static_cast(mode)); + if (fd < 0) { + if (errno == EEXIST) { + return fopen(path.c_str(), "a"); + } + return nullptr; + } + // The umask can only take bits away, so the file is never more permissive + // than `mode' in the window before the exact bits are restored. + if (::fchmod(fd, static_cast(mode)) != 0) { + // Not fatal: the file exists and is writable through `fd'. + } + FILE* const file = ::fdopen(fd, "a"); + if (file == nullptr) { + ::close(fd); + } + return file; +} +#endif // OS_POSIX + // Called by logging functions to ensure that debug_file is initialized // and can be used for writing. Returns false if the file could not be // initialized. debug_file will be nullptr in this case. @@ -430,14 +469,20 @@ bool InitializeLogFileHandle() { log_file_size = GetFileSizeEx(log_file, &file_size) ? static_cast(file_size.QuadPart) : 0; #elif defined(OS_POSIX) - log_file = fopen(log_file_name->c_str(), "a"); + log_file = OpenLogFile(*log_file_name, log_file_mode); if (log_file == nullptr) { fprintf(stderr, "Fail to fopen %s: %s", log_file_name->c_str(), berror()); return false; } struct stat st; - log_file_size = (0 == fstat(fileno(log_file), &st)) ? - static_cast(st.st_size) : 0; + if (0 == fstat(fileno(log_file), &st)) { + log_file_size = static_cast(st.st_size); + // Take the permissions from the file that is actually in use, so + // that a rotation, which recreates the file, keeps them. + log_file_mode = static_cast(st.st_mode & 07777); + } else { + log_file_size = 0; + } #endif } @@ -467,9 +512,13 @@ void CloseLogFileUnlocked() { // ".1" and a fresh file is opened under the original name. // Existing backups are shifted down (".1" -> ".2" -> ...) beforehand and the // ones beyond --log_rotate_max_backups are removed, so that ".1" is always -// the most recent backup. Every rotation also removes the backups left over -// from a previously larger --log_rotate_max_backups, so that lowering the -// flag at run time really lowers the number of files kept. +// the most recent backup. The backups are found by listing the directory +// rather than by probing 1, 2, 3..., so a hole left behind by an earlier +// failure neither strands a backup forever nor makes a huge +// --log_rotate_max_backups cost one rename() per index. +// +// The fresh file is created with the permissions of the file it replaces: a +// private 0600 log file must not become world readable on the first rotation. // // A log file that disappeared from under the process is not a rotation // failure: the handle is closed so that the next write starts a fresh file, @@ -489,6 +538,12 @@ void CloseLogFileUnlocked() { // file that turns out to be smaller than it was at that point has been // replaced or truncated from the outside, and rotation resumes immediately. // +// On POSIX nothing is closed before the rename succeeded, so the open handle +// outlives every failure of a rotation and keeps receiving logs even when the +// path can not be opened again (permissions tightened under a running +// process, privileges dropped, ...). Closing it first used to drop every log +// written after such a failure, which never happened without rotation. +// // Sharing one log file between multiple processes is NOT supported together // with rotation: on POSIX LoggingLock only serializes the threads of a single // process, and after the rename the other processes would keep writing to the @@ -535,35 +590,270 @@ std::string PathAsString(const PathString& path) { #endif } -// Back off after a rotation that could not rename `from' to `to': the size -// that triggered it does not change by itself, so retrying on the next log -// would close, reopen and rename the file for every single log. The failure -// is printed once rather than per log. The caller must hold the logging -// lock and `err' must have been captured before anything else could -// overwrite it. +// Back off after a rotation that failed in the way described by +// `what_failed': the size that triggered it does not change by itself, so +// retrying on the next log would close, reopen and rename the file for every +// single log. The failure is printed once rather than per log. The caller +// must hold the logging lock and `err' must have been captured before +// anything else could overwrite it. void BackOffAfterFailedRotation(int64_t size_before_rotation, - const PathString& from, - const PathString& to, + const std::string& what_failed, SystemErrorCode err) { failed_rotation_size = size_before_rotation; if (rotation_failure_reported) { return; } rotation_failure_reported = true; - fprintf(stderr, "Fail to rotate %s: renaming %s to %s failed: %s, " - "retrying after another --log_rotate_size_mb of logs\n", - PathAsString(*log_file_name).c_str(), PathAsString(from).c_str(), - PathAsString(to).c_str(), + fprintf(stderr, "Fail to rotate %s: %s: %s, retrying after another " + "--log_rotate_size_mb of logs\n", + PathAsString(*log_file_name).c_str(), what_failed.c_str(), SystemErrorCodeToString(err).c_str()); } -// Rename the current log file to ".1" and shift the older -// backups down by one. The caller must hold the logging lock. +// Whether `name' is "." with a plain positive index. Everything +// else in the directory (a rotated name of an external rotator such as +// "foo.log.2024-01-01", a leftover of some other tool, ...) is not ours and +// is left alone. +bool ParseBackupIndex(const PathString& base, const PathString& name, + int* index) { + const PathString prefix = base + '.'; + if (name.size() <= prefix.size() || + 0 != name.compare(0, prefix.size(), prefix)) { + return false; + } + int64_t value = 0; + for (size_t i = prefix.size(); i < name.size(); ++i) { + const PathString::value_type c = name[i]; + if (c < '0' || c > '9') { + return false; + } + value = value * 10 + (c - '0'); + if (value > std::numeric_limits::max()) { + return false; + } + } + if (value <= 0) { + return false; + } + *index = static_cast(value); + return true; +} + +// Collect the indexes of the backups that exist right now, in no particular +// order. Returns false when the directory can not be listed: the caller must +// not shift anything then, because assuming that there is no backup would +// rename the current file over ".1" and destroy it. +bool ListBackupIndices(const PathString& path, std::vector* indexes) { +#if defined(OS_WIN) + const PathString::size_type sep = path.find_last_of(L"\\/"); + const PathString dir = + (sep == PathString::npos) ? PathString(L".") : path.substr(0, sep); + const PathString base = + (sep == PathString::npos) ? path : path.substr(sep + 1); + const PathString pattern = dir + L"\\" + base + L".*"; + WIN32_FIND_DATAW entry; + HANDLE const handle = ::FindFirstFileW(pattern.c_str(), &entry); + if (handle == INVALID_HANDLE_VALUE) { + // No backup yet, which is not an error. Anything else means that the + // directory could not be listed. + return ::GetLastError() == ERROR_FILE_NOT_FOUND; + } + do { + int index = 0; + if (ParseBackupIndex(base, PathString(entry.cFileName), &index)) { + indexes->push_back(index); + } + } while (0 != ::FindNextFileW(handle, &entry)); + ::FindClose(handle); + return true; +#elif defined(OS_NACL) + return false; +#else + const PathString::size_type sep = path.rfind('/'); + const PathString dir = + (sep == PathString::npos) ? PathString(".") : path.substr(0, sep); + const PathString base = + (sep == PathString::npos) ? path : path.substr(sep + 1); + DIR* const stream = ::opendir(dir.c_str()); + if (stream == nullptr) { + return false; + } + bool ok = true; + errno = 0; + while (struct dirent* const entry = ::readdir(stream)) { + int index = 0; + if (ParseBackupIndex(base, PathString(entry->d_name), &index)) { + indexes->push_back(index); + } + errno = 0; + } + if (errno != 0) { + ok = false; + } + ::closedir(stream); + return ok; +#endif +} + +// Fall back to probing ".1", ".2", ... when the directory can +// not be listed, which happens for a log directory that is writable without +// being readable. Unlike treating a failed listing as "no backup at all", +// every index that is about to be renamed over is checked here, so a backup +// can not be destroyed. The bound keeps this rare path from turning a huge +// --log_rotate_max_backups into the per-index cost described above. +void ProbeBackupIndices(int max_backups, std::vector* indexes) { + const int limit = std::min(max_backups, 1000); + for (int i = 1; i < limit; ++i) { + if (!PathExists(MakeBackupLogPath(*log_file_name, i))) { + break; + } + indexes->push_back(i); + } +} + +// Shift the backups down by one, dropping the ones that no longer fit into +// --log_rotate_max_backups. Returns false when a rename failed, and the +// caller must then leave the current log file alone. +bool ShiftBackupsUnlocked(int max_backups, int64_t size_before_rotation) { + std::vector indexes; + if (!ListBackupIndices(*log_file_name, &indexes)) { + ProbeBackupIndices(max_backups, &indexes); + } + std::sort(indexes.begin(), indexes.end()); + indexes.erase(std::unique(indexes.begin(), indexes.end()), indexes.end()); + // Walk from the highest index down, so that the destination of a rename is + // always either a slot that was just vacated or one that is about to fall + // off the end. Working from the indexes that really exist, instead of from + // 1 to --log_rotate_max_backups, is what makes a hole harmless: the + // backups above it are still shifted and, once they reach the limit, + // removed, so a partly failed rotation can not keep an orphan around. + for (size_t i = indexes.size(); i-- > 0;) { + const int index = indexes[i]; + const PathString from = MakeBackupLogPath(*log_file_name, index); + if (static_cast(index) + 1 > max_backups) { + // Would fall behind the newest backup after the shift. A file that + // can not be removed stays where it is and the next rotation tries + // again, without spinning here. + DeleteFilePath(from); + continue; + } + const PathString to = MakeBackupLogPath(*log_file_name, index + 1); + if (!RenameFilePath(from, to)) { + // Captured in a statement of its own: the error code has to be + // read before building the message, since formatting it is free to + // touch errno, and the order of function arguments is unspecified. + const SystemErrorCode err = GetLastSystemErrorCode(); + BackOffAfterFailedRotation( + size_before_rotation, + butil::string_printf("renaming %s to %s failed", + PathAsString(from).c_str(), + PathAsString(to).c_str()), + err); + return false; + } + } + return true; +} + +// Rename the current log file to ".1" and reopen a fresh file +// under the original name. The backups have already been shifted away by the +// caller. Returns true when the fresh file is open and ready for writing. +bool RotateCurrentFileUnlocked(int64_t size_before_rotation) { + const PathString first_backup = MakeBackupLogPath(*log_file_name, 1); +#if defined(OS_POSIX) + // POSIX lets us rename a file that is still open, so the handle survives + // every failure below and keeps pointing at the same inode: if the path + // can not be opened again there is still a working handle to write + // through, and no log is lost. That is also why nothing is closed before + // the rename below. + if (log_file) { + // Bytes sitting in the stdio buffer belong to the file renamed below. + fflush(log_file); + // Take the permissions from the file that is replaced, which also + // picks up a chmod() done after the file was opened. + struct stat st; + if (0 == fstat(fileno(log_file), &st)) { + log_file_mode = static_cast(st.st_mode & 07777); + } + } + if (!RenameFilePath(*log_file_name, first_backup)) { + const SystemErrorCode err = GetLastSystemErrorCode(); + BackOffAfterFailedRotation( + size_before_rotation, + butil::string_printf("renaming %s to %s failed", + PathAsString(*log_file_name).c_str(), + PathAsString(first_backup).c_str()), + err); + return false; + } + FILE* const fresh = OpenLogFile(*log_file_name, log_file_mode); + if (fresh == nullptr) { + // Taken before the rollback below, whose own failure would otherwise + // be the one reported. + const SystemErrorCode err = GetLastSystemErrorCode(); + // The path can not be opened again. Put the backup back under its + // original name so the file keeps the name logs are expected under. + // Whichever way that rename goes, the handle below still owns the + // inode and keeps writing to it, so no log is lost. + RenameFilePath(first_backup, *log_file_name); + BackOffAfterFailedRotation( + size_before_rotation, + butil::string_printf("reopening %s failed", + PathAsString(*log_file_name).c_str()), + err); + return false; + } + if (log_file) { + CloseFile(log_file); + } + log_file = fresh; + // The fresh file is empty, unless something recreated the path before it + // was opened, so read the size instead of assuming zero. + struct stat st; + log_file_size = (0 == fstat(fileno(log_file), &st)) ? + static_cast(st.st_size) : 0; + return true; +#else + // An opened file can not be renamed on Windows, so the handle has to be + // closed first. Reopening right away keeps the window, and the loss of the + // log that triggered the rotation, as small as possible. + CloseLogFileUnlocked(); + if (!RenameFilePath(*log_file_name, first_backup)) { + // Captured before reopening the handle below, which is free to + // overwrite the last error (its fallback to ".\\debug.log" usually + // succeeds and leaves nothing behind to report). + const SystemErrorCode err = GetLastSystemErrorCode(); + // Put the handle back on the file that is still there, so the next log + // keeps appending instead of being dropped. + InitializeLogFileHandle(); + BackOffAfterFailedRotation( + size_before_rotation, + butil::string_printf("renaming %s to %s failed", + PathAsString(*log_file_name).c_str(), + PathAsString(first_backup).c_str()), + err); + return false; + } + if (!InitializeLogFileHandle()) { + const SystemErrorCode err = GetLastSystemErrorCode(); + BackOffAfterFailedRotation( + size_before_rotation, + butil::string_printf("reopening %s failed", + PathAsString(*log_file_name).c_str()), + err); + return false; + } + return true; +#endif +} + +// Rotate the log file: shift the backups down, then move the current file to +// ".1". The caller must hold the logging lock. void RotateLogFileUnlocked() { if (!log_file_name) { return; } - // Remember it before closing, CloseLogFileUnlocked() resets the size. + // Remember it before anything can reset it. const int64_t size_before_rotation = log_file_size; if (!PathExists(*log_file_name)) { // The file was renamed or removed from the outside. There is nothing @@ -580,57 +870,13 @@ void RotateLogFileUnlocked() { failed_rotation_size = size_before_rotation; return; } - // Close before renaming: on Windows an opened file cannot be renamed, and - // on POSIX the handle would keep pointing at the renamed inode. - CloseLogFileUnlocked(); - const int max_backups = std::max(1, FLAGS_log_rotate_max_backups); - // Remove the oldest backup, and any left over from a previously larger - // --log_rotate_max_backups, so that lowering the flag at run time really - // lowers the number of files kept. - for (int i = max_backups; ; ++i) { - const PathString path = MakeBackupLogPath(*log_file_name, i); - if (!PathExists(path)) { - break; - } - DeleteFilePath(path); - if (PathExists(path)) { - break; // could not remove it, do not spin - } - } - // Rotation numbers the backups contiguously, so the first missing index - // bounds the existing ones from above. Shifting from the highest - // existing index rather than from --log_rotate_max_backups keeps a huge - // value (only validated to be a positive int) from turning into as many - // rename() calls on every rotation, all under the logging lock. - int highest_backup = 0; - for (int i = 1; i < max_backups; ++i) { - if (!PathExists(MakeBackupLogPath(*log_file_name, i))) { - break; - } - highest_backup = i; - } - // Shift the backups down by one. A rename that fails aborts the - // rotation: renaming the current file over ".1" anyway - // would destroy the newest backup, the very one that the failed shift - // could not move out of the way. - for (int i = highest_backup; i >= 1; --i) { - const PathString from = MakeBackupLogPath(*log_file_name, i); - const PathString to = MakeBackupLogPath(*log_file_name, i + 1); - if (!RenameFilePath(from, to)) { - BackOffAfterFailedRotation(size_before_rotation, from, to, - GetLastSystemErrorCode()); - return; - } + if (!ShiftBackupsUnlocked(max_backups, size_before_rotation) || + !RotateCurrentFileUnlocked(size_before_rotation)) { + return; // backed off, the current file was left in place } - const PathString first_backup = MakeBackupLogPath(*log_file_name, 1); - if (RenameFilePath(*log_file_name, first_backup)) { - failed_rotation_size = -1; - rotation_failure_reported = false; - return; - } - BackOffAfterFailedRotation(size_before_rotation, *log_file_name, - first_backup, GetLastSystemErrorCode()); + failed_rotation_size = -1; + rotation_failure_reported = false; } // Rotate the log file if appending `incoming_size' bytes would grow it beyond @@ -664,6 +910,28 @@ void MaybeRotateLogFileUnlocked(size_t incoming_size) { RotateLogFileUnlocked(); } +// Re-read the size of the log file. `log_file_size' is what decides when to +// rotate, and a short write or a failed flush leaves it behind the real size +// for the rest of the process, which postpones rotation of every following +// file as well. Re-reading it is cheaper than being wrong until restart. +void ResyncLogFileSize() { + if (!log_file) { + log_file_size = 0; + return; + } +#if defined(OS_WIN) + LARGE_INTEGER file_size; + if (GetFileSizeEx(log_file, &file_size)) { + log_file_size = static_cast(file_size.QuadPart); + } +#else + struct stat st; + if (0 == fstat(fileno(log_file), &st)) { + log_file_size = static_cast(st.st_size); + } +#endif +} + void Log2File(const std::string& log) { // We can have multiple threads and/or processes, so try to prevent them // from clobbering each other's writes. @@ -686,14 +954,23 @@ void Log2File(const std::string& log) { #if defined(OS_WIN) SetFilePointer(log_file, 0, 0, SEEK_END); DWORD num_written = 0; - WriteFile(log_file, static_cast(log.data()), - static_cast(log.size()), &num_written, nullptr); - log_file_size += num_written; + if (WriteFile(log_file, static_cast(log.data()), + static_cast(log.size()), &num_written, nullptr) && + num_written == static_cast(log.size())) { + log_file_size += num_written; + } else { + ResyncLogFileSize(); + } #else - if (fwrite(log.data(), log.size(), 1, log_file) == 1) { - log_file_size += (int64_t)log.size(); + const size_t written = fwrite(log.data(), 1, log.size(), log_file); + log_file_size += static_cast(written); + const bool flushed = fflush(log_file) == 0; + if (written != log.size() || !flushed) { + // clearerr() drops the sticky error flag, so that the next log + // gets a chance to be written instead of failing forever. + clearerr(log_file); + ResyncLogFileSize(); } - fflush(log_file); #endif } } @@ -1063,6 +1340,9 @@ bool BaseInitLoggingImpl(const LoggingSettings& settings) { // Give rotation another chance, the destination may have changed. failed_rotation_size = -1; rotation_failure_reported = false; + // The permissions of the previous destination say nothing about the new + // one. It is read again from the file that is opened below. + log_file_mode = -1; return InitializeLogFileHandle(); } diff --git a/test/logging_unittest.cc b/test/logging_unittest.cc index 8374140f90..dd13d84776 100644 --- a/test/logging_unittest.cc +++ b/test/logging_unittest.cc @@ -666,6 +666,59 @@ class LogRotationTest : public LoggingTest { } } + // A log carries the path of this source file in its header, so how many + // bytes it occupies depends on where the tree is checked out and a test + // that counts logs to cross the rotation threshold breaks as soon as that + // path changes length. The helpers below therefore watch the file instead + // of counting. + + // Write logs until a rotation happens, which shows up as the current file + // restarting: it is renamed away and recreated, so its size drops from + // about --log_rotate_size_mb down to a single log. Returns the number of + // logs written, or -1 when no rotation was seen within `max_logs'. + int WriteLogsUntilRotation(int max_logs, size_t size) { + const std::string content(size, 'x'); + off_t last = 0; + FileSize(_log_path, &last); + for (int i = 0; i < max_logs; ++i) { + LOG(INFO) << content; + off_t now = 0; + FileSize(_log_path, &now); // 0 while the file is gone + if (now < last) { + return i + 1; + } + last = now; + } + return -1; + } + + // Write logs until the current file is larger than `limit'. Rotation must + // be off, otherwise the file restarts before it can grow that far. + bool WriteLogsUntilLargerThan(off_t limit, int max_logs, size_t size) { + const std::string content(size, 'x'); + off_t now = 0; + for (int i = 0; i < max_logs; ++i) { + if (FileSize(_log_path, &now) && now > limit) { + return true; + } + LOG(INFO) << content; + } + return FileSize(_log_path, &now) && now > limit; + } + + // Write logs until `path' shows up, or `max_logs' of them were written. + bool WriteLogsUntilFileExists(const std::string& path, int max_logs, + size_t size) { + const std::string content(size, 'x'); + for (int i = 0; i < max_logs; ++i) { + if (FileSize(path, nullptr)) { + return true; + } + LOG(INFO) << content; + } + return FileSize(path, nullptr); + } + std::string _log_path; private: @@ -857,11 +910,15 @@ TEST_F(LogRotationTest, external_rename_does_not_consume_a_backup) { } } - // Take the file away the way an external log rotator would, then log - // enough that a rotation would otherwise have been due. + // Take the file away the way an external log rotator would. Logging goes + // on through the handle that is still open, which now points at the + // renamed inode, until a rotation is due: that attempt has to notice the + // missing file, leave the backups alone and start a fresh file under the + // original name. Wait for exactly that instead of guessing how many logs + // it takes, since the length of the header decides that. const std::string moved = _log_path + ".moved"; ASSERT_EQ(0, rename(_log_path.c_str(), moved.c_str())); - WriteLogs(800, 1000); + ASSERT_TRUE(WriteLogsUntilFileExists(_log_path, 100000, 1000)); CloseLogFile(); off_t size = 0; @@ -949,14 +1006,14 @@ TEST_F(LogRotationTest, recovers_after_the_log_file_is_replaced) { // rather than let the final rename overwrite it with the current log file, // which would silently drop the newest generation on every rotation. TEST_F(LogRotationTest, shift_failure_keeps_the_newest_backup) { + const off_t rotate_size = 1L * 1024 * 1024; FLAGS_log_rotate_size_mb = 1; FLAGS_log_rotate_max_backups = 2; InitLoggingToTempFile(); - // Rotate once so that ".1" holds a complete generation. The size is - // not enough to tell the generations apart, the rotation threshold - // makes them equally large, the content is compared instead. - WriteLogs(1200, 1000); + // Rotate once so that ".1" holds a complete generation. The number of + // logs needed is not fixed, see WriteLogsUntilRotation(). + ASSERT_GT(WriteLogsUntilRotation(100000, 1000), 0); std::string first_backup; ASSERT_TRUE(butil::ReadFileToString( butil::FilePath(BackupPath(1)), &first_backup)); @@ -967,7 +1024,8 @@ TEST_F(LogRotationTest, shift_failure_keeps_the_newest_backup) { ASSERT_EQ(0, mkdir(BackupPath(2).c_str(), 0755)); ASSERT_EQ(0, mkdir((BackupPath(2) + "/x").c_str(), 0755)); - WriteLogs(1200, 1000); // crosses the threshold, the shift fails + // Grow past the threshold: the rotation is attempted and fails. + ASSERT_TRUE(WriteLogsUntilLargerThan(rotate_size, 100000, 1000)); CloseLogFile(); // ".1" still holds the very same generation, rotation did not trade it @@ -979,22 +1037,111 @@ TEST_F(LogRotationTest, shift_failure_keeps_the_newest_backup) { // The current file kept growing instead of losing a generation. off_t size = 0; ASSERT_TRUE(FileSize(_log_path, &size)); - ASSERT_GT(size, 1L * 1024 * 1024); + ASSERT_GT(size, rotate_size); // Once the obstacle is gone, the backed off retry shifts the preserved // backup and rotation works again. ASSERT_EQ(0, rmdir((BackupPath(2) + "/x").c_str())); ASSERT_EQ(0, rmdir(BackupPath(2).c_str())); - WriteLogs(1200, 1000); + ASSERT_TRUE(WriteLogsUntilFileExists(BackupPath(2), 100000, 1000)); CloseLogFile(); ASSERT_TRUE(butil::ReadFileToString( butil::FilePath(BackupPath(2)), &backup)); ASSERT_EQ(first_backup, backup); // the preserved generation, shifted - ASSERT_TRUE(FileSize(BackupPath(1), &size)); - ASSERT_GT(size, 1L * 1024 * 1024); } +// Backups are numbered by renaming them one by one, so a partly failed shift +// leaves a hole behind: ".1" and ".3", ".4" without a ".2". Cleaning up from +// the first missing index would never reach the files above the hole, which +// is how the oldest generations used to survive forever after the limit was +// lowered. +TEST_F(LogRotationTest, rotation_removes_backups_left_in_a_hole) { + const off_t rotate_size = 1L * 1024 * 1024; + // Fill the file past the threshold while rotation is still off. + FLAGS_log_rotate_size_mb = 0; + FLAGS_log_rotate_max_backups = 1; + InitLoggingToTempFile(); + ASSERT_TRUE(WriteLogsUntilLargerThan(rotate_size, 100000, 1000)); + CloseLogFile(); + + // The state a partly failed shift leaves behind, with a hole at ".2". + for (int i = 1; i <= 4; ++i) { + if (i == 2) { + continue; + } + FILE* const file = fopen(BackupPath(i).c_str(), "w"); + ASSERT_TRUE(file != nullptr) << BackupPath(i); + ASSERT_EQ((size_t)1, fwrite("x", 1, 1, file)); + fclose(file); + } + + FLAGS_log_rotate_size_mb = 1; + WriteLogs(1, 10); // the file is over the threshold, rotate + CloseLogFile(); + + // Only the limit itself survived: one backup, and no orphan above a hole. + ASSERT_TRUE(FileSize(BackupPath(1), nullptr)); + ASSERT_FALSE(FileSize(BackupPath(2), nullptr)); + ASSERT_FALSE(FileSize(BackupPath(3), nullptr)); + ASSERT_FALSE(FileSize(BackupPath(4), nullptr)); +} + +// Rotation recreates the log file, and it has to be recreated with the +// permissions of the file it replaces: falling back to 0666 & ~umask turns a +// private 0600 log file into a world readable one on the first rotation. +#if !defined(OS_WIN) +TEST_F(LogRotationTest, rotation_keeps_the_file_mode) { + FLAGS_log_rotate_size_mb = 1; + FLAGS_log_rotate_max_backups = 2; + InitLoggingToTempFile(); + // A chmod() done while the process runs must be honoured as well, not only + // the mode the file was opened with. + ASSERT_EQ(0, chmod(_log_path.c_str(), 0600)); + + ASSERT_GT(WriteLogsUntilRotation(100000, 1000), 0); + CloseLogFile(); + + struct stat st; + ASSERT_EQ(0, stat(_log_path.c_str(), &st)); + ASSERT_EQ(0600, (int)(st.st_mode & 07777)); + // The rotated generation is the same inode and keeps its own permissions. + ASSERT_EQ(0, stat(BackupPath(1).c_str(), &st)); + ASSERT_EQ(0600, (int)(st.st_mode & 07777)); +} + +// Closing the handle before the rename drops every log as soon as the path can +// not be opened again while the handle that is still open could write just +// fine: permissions tightened under a running process, privileges dropped at +// start up, ... Without rotation the handle is never reopened, so closing it +// early is a regression that the feature introduced. +TEST_F(LogRotationTest, keeps_writing_when_the_file_can_not_be_reopened) { + const off_t rotate_size = 1L * 1024 * 1024; + FLAGS_log_rotate_size_mb = 1; + FLAGS_log_rotate_max_backups = 2; + InitLoggingToTempFile(); + ASSERT_GT(WriteLogsUntilRotation(100000, 1000), 0); // ".1" exists + + // Make the shift fail, and make the current file impossible to reopen + // while the handle that is already open keeps working. + ASSERT_EQ(0, mkdir(BackupPath(2).c_str(), 0755)); + ASSERT_EQ(0, mkdir((BackupPath(2) + "/x").c_str(), 0755)); + ASSERT_EQ(0, chmod(_log_path.c_str(), 0400)); + + off_t before = 0; + ASSERT_TRUE(FileSize(_log_path, &before)); + // Rotation keeps failing, so the file keeps growing. A log written after + // the failed attempt would be lost if the handle had been closed to try + // the rename, which is what this checks. + ASSERT_TRUE(WriteLogsUntilLargerThan(before + rotate_size, 100000, 1000)); + CloseLogFile(); + + ASSERT_EQ(0, chmod(_log_path.c_str(), 0644)); + ASSERT_EQ(0, rmdir((BackupPath(2) + "/x").c_str())); + ASSERT_EQ(0, rmdir(BackupPath(2).c_str())); +} +#endif // !defined(OS_WIN) + // --log_rotate_max_backups is only validated to be a positive int, so a // huge value must not turn into one rename() per index on every rotation, // all of them under the logging lock. @@ -1005,16 +1152,20 @@ TEST_F(LogRotationTest, huge_max_backups_shifts_only_existing_backups) { butil::Timer timer; timer.start(); - WriteLogs(2500, 1000); // ~2.5MB, a couple of rotations + // Two rotations say everything there is to say: the cost has to follow the + // backups that exist (two of them) and not --log_rotate_max_backups, whose + // 1e8 renames would stall the logging lock for minutes. Watching for the + // backups instead of counting logs keeps the test independent of how many + // bytes a log occupies, which depends on the length of the source path in + // its header. + ASSERT_TRUE(WriteLogsUntilFileExists(BackupPath(2), 100000, 1000)); CloseLogFile(); timer.stop(); - // The rotations above renamed a handful of files rather than 1e8 of - // them, which would stall the logging threads for minutes. ASSERT_LT(timer.m_elapsed(), 10 * 1000); ASSERT_TRUE(FileSize(BackupPath(1), nullptr)); ASSERT_TRUE(FileSize(BackupPath(2), nullptr)); - ASSERT_FALSE(FileSize(BackupPath(3), nullptr)); + ASSERT_FALSE(FileSize(BackupPath(3), nullptr)); // no third rotation yet } struct SyncLogArgs {