Fix resumable-transport DDoS: terminate degenerate pullBundleChunk requests

csharp/src/HgResume.Api/ApiConfig.cs

@@ -43,6 +43,37 @@ public sealed class ApiConfig
     /// </summary>
     public required bool RequireManageSecret { get; init; }
 
+    /// <summary>Default upper bound on a client-supplied chunkSize (20 MB, matching the Chorus client).</summary>
+    public const int DefaultChunkSizeMax = 20 * 1024 * 1024;
+
+    /// <summary>
+    /// Upper bound the server clamps a client-supplied <c>chunkSize</c> to. <see cref="GetChunkAsync"/>
+    /// already caps each read at the remaining bundle length, so this is cheap defense against a client
+    /// asking for an absurd allocation.
+    /// </summary>
+    public int ChunkSizeMax { get; init; } = DefaultChunkSizeMax;
+
+    /// <summary>
+    /// How old a <c>.async_run</c> lock that is NOT tracked by this process's in-memory registry must be
+    /// before it is treated as a dead generation (API restarted mid-bundle, lock file left behind) and
+    /// cleaned up so a new generation can be spawned. A live generation in this process is recognised via
+    /// the registry regardless of age, so this only governs cross-restart recovery. Default 2 minutes.
+    /// </summary>
+    public TimeSpan StaleBundleLockThreshold { get; init; } = TimeSpan.FromSeconds(120);
+
+    /// <summary>
+    /// Maximum number of times the server will (re)spawn bundle generation for a single pull transaction
+    /// before giving up with FAIL, so a bundle that genuinely cannot be produced fails fast instead of
+    /// respawning hg forever. Default 3.
+    /// </summary>
+    public int MaxBundleGenAttempts { get; init; } = 3;
+
+    /// <summary>Age after which the cache GC reaps a <c>.bundle/.metadata/.async_run/.incoming</c> file. Default 24h.</summary>
+    public TimeSpan CacheTtl { get; init; } = TimeSpan.FromHours(24);
+
+    /// <summary>How often the cache GC background service runs. Default 1h.</summary>
+    public TimeSpan CacheGcInterval { get; init; } = TimeSpan.FromMinutes(60);
+
     public static ApiConfig FromEnvironment(bool isDevelopment)
     {
         string cache = Env("HGRESUME_CACHE_PATH", "/var/cache/hgresume");
@@ -59,6 +90,12 @@ public static ApiConfig FromEnvironment(bool isDevelopment)
             ResetCleanupAgeDays = EnvInt("HGRESUME_RESET_CLEANUP_AGE_DAYS", 31),
             ManageSecret = Environment.GetEnvironmentVariable("HGRESUME_MANAGE_SECRET") is { Length: > 0 } s ? s : null,
             RequireManageSecret = EnvBool("HGRESUME_REQUIRE_MANAGE_SECRET", !isDevelopment),
+            ChunkSizeMax = EnvInt("HGRESUME_CHUNK_SIZE_MAX", DefaultChunkSizeMax),
+            StaleBundleLockThreshold =
+                TimeSpan.FromSeconds(EnvInt("HGRESUME_STALE_BUNDLE_LOCK_SECONDS", 120)),
+            MaxBundleGenAttempts = EnvInt("HGRESUME_MAX_BUNDLE_GEN_ATTEMPTS", 3),
+            CacheTtl = TimeSpan.FromHours(EnvInt("HGRESUME_CACHE_TTL_HOURS", 24)),
+            CacheGcInterval = TimeSpan.FromMinutes(EnvInt("HGRESUME_CACHE_GC_INTERVAL_MINUTES", 60)),
         };
     }
 

csharp/src/HgResume.Api/AsyncRunner.cs

@@ -98,6 +98,35 @@ await File.WriteAllTextAsync(lockFile,
 
     public bool IsRunning() => File.Exists(_lockFile);
 
+    /// <summary>
+    /// True when this lock is backed by a background task still running in THIS process. Survives as a
+    /// real liveness signal (unlike a bare lock file, which also persists across a process restart that
+    /// killed the generation). Used to tell an in-flight bundle from a dead lock left by a crash.
+    /// </summary>
+    public bool IsTrackedRunning() => Running.ContainsKey(_lockFile);
+
+    /// <summary>
+    /// True when a lock file exists for a generation this process is NOT running (not in <see
+    /// cref="Running"/>) and it was last written longer than <paramref name="threshold"/> ago — i.e. a
+    /// generation abandoned by a crash/restart. Such a lock must be cleaned up and respawned rather than
+    /// polled forever. A lock this process IS running is never stale regardless of age.
+    /// </summary>
+    public bool IsStaleLock(TimeSpan threshold)
+    {
+        if (!File.Exists(_lockFile) || Running.ContainsKey(_lockFile))
+        {
+            return false;
+        }
+        try
+        {
+            return DateTime.UtcNow - File.GetLastWriteTimeUtc(_lockFile) > threshold;
+        }
+        catch
+        {
+            return false; // if we can't stat it, don't treat it as reapable
+        }
+    }
+
     public async Task<bool> IsCompleteAsync(CancellationToken ct = default)
     {
         if (!File.Exists(_lockFile))

csharp/src/HgResume.Api/CacheGarbageCollector.cs

@@ -0,0 +1,91 @@
+namespace HgResume.Api;
+
+/// <summary>
+/// Periodically reaps abandoned transaction artifacts from <see cref="ApiConfig.CachePath"/>.
+///
+/// Only <c>finishPull/PushBundle</c> cleaned these up, so a client that looped (or simply went away)
+/// left its <c>.bundle/.metadata/.async_run/.incoming</c> files behind forever. The July 2026 incident
+/// left <b>1,258</b> leaked <c>.async_run</c> locks. The PHP deployment would have needed a cron job;
+/// the .NET host is long-running, so a <see cref="BackgroundService"/> fits without any image
+/// process-model change.
+/// </summary>
+public sealed class CacheGarbageCollector : BackgroundService
+{
+    // Transaction artifacts BundleHelper/AsyncRunner write into the cache dir. Repo-reset backups live
+    // under their own TempRepoFolder and are reaped by RepoManageService, so we don't touch those.
+    private static readonly string[] Extensions = { ".bundle", ".metadata", ".async_run", ".incoming" };
+
+    private readonly ApiConfig _config;
+    private readonly ILogger<CacheGarbageCollector> _logger;
+
+    public CacheGarbageCollector(ApiConfig config, ILogger<CacheGarbageCollector> logger)
+    {
+        _config = config;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        // Run once at startup (reap anything a prior process left behind), then on the configured cadence.
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                Collect();
+            }
+            catch (Exception e)
+            {
+                _logger.LogWarning(e, "Cache GC pass failed");
+            }
+
+            try
+            {
+                await Task.Delay(_config.CacheGcInterval, stoppingToken);
+            }
+            catch (OperationCanceledException)
+            {
+                break;
+            }
+        }
+    }
+
+    /// <summary>Deletes cache artifacts whose last-write time is older than <see cref="ApiConfig.CacheTtl"/>.</summary>
+    public int Collect()
+    {
+        if (!Directory.Exists(_config.CachePath))
+        {
+            return 0;
+        }
+
+        DateTime cutoff = DateTime.UtcNow - _config.CacheTtl;
+        int removed = 0;
+        foreach (string path in Directory.EnumerateFiles(_config.CachePath))
+        {
+            if (!Extensions.Contains(Path.GetExtension(path)))
+            {
+                continue;
+            }
+            try
+            {
+                if (File.GetLastWriteTimeUtc(path) < cutoff)
+                {
+                    File.Delete(path);
+                    removed++;
+                }
+            }
+            catch (Exception e)
+            {
+                // A file deleted or rewritten by a concurrent request between stat and delete is fine;
+                // just skip it this pass.
+                _logger.LogDebug(e, "Cache GC could not reap {Path}", path);
+            }
+        }
+
+        if (removed > 0)
+        {
+            _logger.LogInformation("Cache GC reaped {Count} stale cache file(s) from {Path}",
+                removed, _config.CachePath);
+        }
+        return removed;
+    }
+}

csharp/src/HgResume.Api/HgResumeApi.cs

@@ -207,6 +207,13 @@ public async Task<HgResumeResponse> PullBundleChunkInternalAsync(string repoId,
             {
                 return Fail("invalid offset");
             }
+            // Clamp a client-supplied chunkSize to a configured cap. GetChunkAsync already bounds each
+            // read to the remaining bundle length, so this is cheap defense against an absurd value (and
+            // keeps offset + chunkSize from overflowing in the size checks below).
+            if (chunkSize > _config.ChunkSizeMax)
+            {
+                chunkSize = _config.ChunkSizeMax;
+            }
 
             var hg = new HgRunner(repoPath);
             if (!await hg.IsValidBaseAsync(baseHashes, ct))
@@ -248,13 +255,49 @@ public async Task<HgResumeResponse> PullBundleChunkInternalAsync(string repoId,
                         ["Error"] = "Cannot request data for bundle that doesnt exist yet",
                     });
                 }
-                // first pull request (offset == 0): make a new bundle
-                asyncRunner = waitForBundleToFinish
-                    ? await hg.MakeBundleAndWaitUntilFinishedAsync(sortedBase, bundleFilename, ct)
-                    : hg.MakeBundle(sortedBase, bundleFilename);
-                bundle.SetProp("tip", await hg.GetTipAsync(ct));
-                bundle.SetProp("repoId", repoId);
-                bundle.State = BundleHelper.State_Bundle;
+
+                // Concurrent-generation guard + crash recovery. A duplicate first-chunk retry can arrive
+                // before the .bundle file appears; without this it spawns a second `hg bundle` for the same
+                // transId. A generation is "in flight" if a background task in this process is still running
+                // it, or a lock file exists that is not yet old enough to be considered abandoned.
+                bool generationInFlight = asyncRunner.IsTrackedRunning() ||
+                    (asyncRunner.IsRunning() && !asyncRunner.IsStaleLock(_config.StaleBundleLockThreshold));
+
+                if (generationInFlight)
+                {
+                    // Someone else is already building this bundle. Don't respawn; poll it via State_Bundle.
+                    if (bundle.State != BundleHelper.State_Bundle &&
+                        bundle.State != BundleHelper.State_Downloading)
+                    {
+                        bundle.State = BundleHelper.State_Bundle;
+                    }
+                }
+                else
+                {
+                    // Reap a stale lock left by a generation this process no longer runs (API restarted
+                    // mid-bundle) so it can't deadlock the transaction, then respawn.
+                    if (asyncRunner.IsRunning())
+                    {
+                        asyncRunner.CleanUp();
+                    }
+
+                    // Bound respawns so a bundle that genuinely can't be produced fails fast with FAIL
+                    // instead of respawning hg forever (the OOM loop).
+                    int attempts = (int.TryParse(bundle.GetProp("genAttempts"), out var prev) ? prev : 0) + 1;
+                    if (attempts > _config.MaxBundleGenAttempts)
+                    {
+                        return Fail($"bundle generation failed after {attempts - 1} attempts");
+                    }
+                    bundle.SetProp("genAttempts", attempts.ToString());
+
+                    // first pull request (offset == 0): make a new bundle
+                    asyncRunner = waitForBundleToFinish
+                        ? await hg.MakeBundleAndWaitUntilFinishedAsync(sortedBase, bundleFilename, ct)
+                        : hg.MakeBundle(sortedBase, bundleFilename);
+                    bundle.SetProp("tip", await hg.GetTipAsync(ct));
+                    bundle.SetProp("repoId", repoId);
+                    bundle.State = BundleHelper.State_Bundle;
+                }
             }
 
             var response = new HgResumeResponse(HgResumeResponse.SUCCESS);