Use constant timestamps for shadowed sources JAR entries

src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/OrderedParallelWriter.kt

@@ -0,0 +1,44 @@
+package com.github.jengelman.gradle.plugins.shadow.internal
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Deferred
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.runBlocking
+
+/**
+ * Computes entry bytes in parallel on [Dispatchers.Default] and writes them in submission order,
+ * which keeps the output deterministic.
+ *
+ * Entries written directly bypass the queue, so [flush] must be called before writing them.
+ */
+internal class OrderedParallelWriter(private val scope: CoroutineScope) {
+  private val pendingEntries = ArrayDeque<PendingEntry>()
+
+  fun submit(compute: () -> ByteArray, write: (ByteArray) -> Unit) {
+    pendingEntries.addLast(PendingEntry(scope.async(Dispatchers.Default) { compute() }, write))
+    if (pendingEntries.size >= MAX_PENDING_ENTRIES) {
+      pendingEntries.removeFirst().writeBytes()
+    }
+  }
+
+  fun flush() {
+    while (pendingEntries.isNotEmpty()) {
+      pendingEntries.removeFirst().writeBytes()
+    }
+  }
+
+  private fun PendingEntry.writeBytes() {
+    write(runBlocking { deferredBytes.await() })
+  }
+
+  private class PendingEntry(val deferredBytes: Deferred<ByteArray>, val write: (ByteArray) -> Unit)
+
+  private companion object {
+    /**
+     * Maximum number of in-flight parallel tasks in the sliding window. Bounds memory consumption
+     * on large archives while keeping worker threads saturated.
+     */
+    const val MAX_PENDING_ENTRIES = 128
+  }
+}

src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ShadowedSourcesJar.kt

@@ -4,6 +4,7 @@ import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator
 import com.github.jengelman.gradle.plugins.shadow.relocation.relocatePath
 import java.io.File
 import java.io.InputStream
+import kotlinx.coroutines.runBlocking
 import org.apache.tools.zip.ZipOutputStream
 import org.gradle.api.file.FileCollection
 import org.vafer.jdeb.shaded.objectweb.asm.ClassReader
@@ -19,7 +20,6 @@ internal fun generateSourcesJar(
   dependencies: Iterable<File>,
   relocators: Iterable<Relocator>,
   unusedClasses: Set<String>,
-  preserveFileTimestamps: Boolean,
 ) =
   try {
     zipOutStream.use { zos ->
@@ -35,11 +35,7 @@ internal fun generateSourcesJar(
 
       val manifestEntry = "META-INF/MANIFEST.MF"
       visitedFiles.add(manifestEntry)
-      zos.writeEntry(
-        name = manifestEntry,
-        preserveLastModified = preserveFileTimestamps,
-        unixMode = UnixMode.file(),
-      ) {
+      zos.writeEntry(name = manifestEntry, unixMode = UnixMode.file()) {
         write("Manifest-Version: 1.0\n\n".toByteArray())
       }
 
@@ -50,70 +46,68 @@ internal fun generateSourcesJar(
         }
       }
 
-      // Relocates and writes a project or dependency source file, or a resource.
-      fun addEntry(name: String, lastModified: Long, readBytes: () -> ByteArray) {
-        if (name.isModuleInfo) return
-        val isSource = name.isSourceFile()
-        if (isSource && isUnused(name, unusedClasses, sourceToClasses)) return
-        val relocatedPath =
-          if (isSource) relocators.relocateSourcePath(name) else relocators.relocatePath(name)
-        if (!visitedFiles.add(relocatedPath)) return
-        val bytes =
-          if (isSource) {
-            relocators
-              .remapSourceFile(readBytes().decodeToString(), name, relocatedPath)
-              .toByteArray()
-          } else {
-            readBytes()
-          }
-        zos.writeEntry(
-          name = relocatedPath,
-          preserveLastModified = preserveFileTimestamps,
-          lastModified = lastModified,
-          unixMode = UnixMode.file(),
-        ) {
-          write(bytes)
-        }
-      }
+      runBlocking {
+        val parallelWriter = OrderedParallelWriter(this)
 
-      filesWithRelPaths
-        .sortedBy { it.second }
-        .forEach { (file, relPath) ->
-          addEntry(relPath, file.lastModified()) { file.readBytes() }
+        fun writeToZip(entryName: String, bytes: ByteArray) {
+          zos.writeEntry(name = entryName, unixMode = UnixMode.file()) { write(bytes) }
         }
 
-      sourcesJars.forEach { jarFile ->
-        jarFile.useZip {
-          entries()
-            .toList()
-            .filterNot { entry ->
-              val name = entry.name
-              entry.isDirectory ||
-                name == "META-INF/MANIFEST.MF" ||
-                name.endsWith(".class") ||
-                name.startsWith("META-INF/INDEX.LIST") ||
-                (name.startsWith("META-INF/") &&
-                  (name.endsWith(".SF") || name.endsWith(".DSA") || name.endsWith(".RSA")))
-            }
-            .sortedBy { it.name }
-            .forEach { entry ->
-              addEntry(entry.name, entry.time) { getInputStream(entry).use { it.readBytes() } }
-            }
+        // Relocates and writes a project or dependency source file, or a resource.
+        fun addEntry(name: String, readBytes: () -> ByteArray) {
+          if (name.isModuleInfo) return
+          val isSource = name.isSourceFile()
+          if (isSource && isUnused(name, unusedClasses, sourceToClasses)) return
+          val relocatedPath =
+            if (isSource) relocators.relocateSourcePath(name) else relocators.relocatePath(name)
+          if (!visitedFiles.add(relocatedPath)) return
+          if (!isSource) {
+            // Resources gain nothing from parallelism, write them directly to avoid buffering.
+            parallelWriter.flush()
+            writeToZip(relocatedPath, readBytes())
+            return
+          }
+          val rawBytes = readBytes()
+          parallelWriter.submit(
+            compute = {
+              relocators
+                .remapSourceFile(rawBytes.decodeToString(), name, relocatedPath)
+                .toByteArray()
+            },
+            write = { writeToZip(relocatedPath, it) },
+          )
         }
-      }
 
-      val entries = zos.entries.map { it.name }
-      val added = entries.toMutableSet()
-      entries.forEach { name ->
-        name.parentDirectoryEntries().forEach { entryName ->
-          if (!added.add(entryName)) return@forEach
-          zos.writeEntry(
-            name = entryName,
-            preserveLastModified = preserveFileTimestamps,
-            unixMode = UnixMode.directory(),
-          )
+        filesWithRelPaths
+          .sortedBy { it.second }
+          .forEach { (file, relPath) ->
+            addEntry(relPath) { file.readBytes() }
+          }
+
+        sourcesJars.forEach { jarFile ->
+          jarFile.useZip {
+            entries()
+              .toList()
+              .filterNot { entry ->
+                val name = entry.name
+                entry.isDirectory ||
+                  name == "META-INF/MANIFEST.MF" ||
+                  name.endsWith(".class") ||
+                  name.startsWith("META-INF/INDEX.LIST") ||
+                  (name.startsWith("META-INF/") &&
+                    (name.endsWith(".SF") || name.endsWith(".DSA") || name.endsWith(".RSA")))
+              }
+              .sortedBy { it.name }
+              .forEach { entry ->
+                addEntry(entry.name) { getInputStream(entry).use { it.readBytes() } }
+              }
+          }
         }
+
+        parallelWriter.flush()
       }
+
+      zos.writeParentDirectoryEntries()
     }
   } catch (e: Exception) {
     sourcesJarFile.delete()

src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/Zip.kt

@@ -8,6 +8,7 @@ import org.apache.tools.zip.UnixStat
 import org.apache.tools.zip.Zip64Mode
 import org.apache.tools.zip.ZipEntry
 import org.apache.tools.zip.ZipOutputStream
+import org.gradle.api.file.FileCopyDetails
 import org.gradle.api.tasks.bundling.ZipEntryCompression
 
 @JvmInline
@@ -103,6 +104,38 @@ internal inline fun ZipOutputStream.writeEntry(
   }
 }
 
+/**
+ * Writes the missing parent directory entries of all written entries, taking the timestamp and
+ * permissions from [visitedDirs] when available.
+ */
+internal fun ZipOutputStream.writeParentDirectoryEntries(
+  preserveLastModified: Boolean = false,
+  visitedDirs: Map<String, FileCopyDetails> = emptyMap(),
+) {
+  val entries = entries.map { it.name }
+  val added = entries.toMutableSet()
+  val currentTimeMillis = System.currentTimeMillis()
+
+  entries.forEach { name ->
+    name.parentDirectoryEntries().asReversed().forEach { entryName ->
+      if (!added.add(entryName)) return@forEach
+      val details = visitedDirs[entryName.removeSuffix("/")]
+      val (lastModified, unixMode) =
+        if (details == null) {
+          currentTimeMillis to UnixMode.directory()
+        } else {
+          details.lastModified to UnixMode.directory(details.permissions.toUnixNumeric())
+        }
+      writeEntry(
+        name = entryName,
+        preserveLastModified = preserveLastModified,
+        lastModified = lastModified,
+        unixMode = unixMode,
+      )
+    }
+  }
+}
+
 internal fun String.parentDirectoryEntries(): List<String> {
   val parents = mutableListOf<String>()
   var parent = substringBeforeLast('/', "")

src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/relocation/Relocator.kt

@@ -9,7 +9,7 @@ import org.gradle.api.tasks.Input
  * [org.apache.maven.plugins.shade.relocation.Relocator.java](https://github.com/apache/maven-shade-plugin/blob/master/src/main/java/org/apache/maven/plugins/shade/relocation/Relocator.java).
  *
  * Implementations of [Relocator] must be thread-safe, as their methods may be invoked concurrently
- * across multiple worker threads during parallel bytecode remapping.
+ * across multiple worker threads during parallel bytecode and source remapping.
  *
  * @author Jason van Zyl
  * @author John Engelman