Remove runtime dependency on log4j-core

CHANGELOG.md

@@ -18,6 +18,7 @@
 
 - Bump min Gradle requirement to 9.4.0. ([#2114](https://github.com/GradleUp/shadow/pull/2114))
 - Remove runtime dependencies on Commons Codec and Commons IO by using JDK APIs. ([#2136](https://github.com/GradleUp/shadow/pull/2136))
+- Remove runtime dependency on `log4j-core` by reading and writing `Log4j2Plugins.dat` files directly in `Log4j2PluginsCacheFileTransformer`. ([#2343](https://github.com/GradleUp/shadow/pull/2343))
 - **POTENTIALLY BREAKING:** Remove `Serializable` from `DependencyFilter`. ([#2144](https://github.com/GradleUp/shadow/pull/2144))
 - Bump default R8 from `9.1.31` to `9.5.28`. ([#2340](https://github.com/GradleUp/shadow/pull/2340))
 - Allow repackaging Service file classes with R8. ([#2174](https://github.com/GradleUp/shadow/pull/2174))

build.gradle.kts

@@ -132,7 +132,6 @@ dependencies {
   compileOnly(libs.kotlin.gradlePlugin)
   compileOnly(libs.kotlin.reflect)
   api(libs.apache.ant) // Types from Ant are exposed in the public API.
-  implementation(libs.apache.log4j)
   implementation(libs.jdependency)
   implementation(libs.jdom2)
   implementation(libs.kotlin.metadata)
@@ -155,6 +154,7 @@ dependencies {
 testing.suites {
   named<JvmTestSuite>("test") {
     dependencies {
+      implementation(libs.apache.log4j)
       implementation(libs.xmlunit)
     }
   }

src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/TransformersTest.kt

@@ -14,11 +14,11 @@ import com.github.jengelman.gradle.plugins.shadow.testkit.getContent
 import com.github.jengelman.gradle.plugins.shadow.testkit.getStream
 import com.github.jengelman.gradle.plugins.shadow.testkit.requireResourceAsPath
 import com.github.jengelman.gradle.plugins.shadow.testkit.requireResourceAsStream
+import com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer.Companion.PLUGIN_CACHE_FILE
 import java.util.jar.Attributes as JarAttribute
 import kotlin.io.path.appendText
 import kotlin.io.path.readText
 import kotlin.io.path.writeText
-import org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor.PLUGIN_CACHE_FILE
 import org.junit.jupiter.api.Test
 
 class TransformersTest : BaseTransformerTest() {

src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformer.kt

@@ -4,21 +4,25 @@ import com.github.jengelman.gradle.plugins.shadow.internal.checkDupStrategy
 import com.github.jengelman.gradle.plugins.shadow.internal.writeEntry
 import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator
 import com.github.jengelman.gradle.plugins.shadow.relocation.relocateClass
-import java.io.FilterOutputStream
-import java.io.OutputStream
-import java.net.URL
-import java.nio.file.Path
-import java.util.Collections
-import java.util.Enumeration
-import kotlin.io.path.createTempFile
-import kotlin.io.path.deleteIfExists
-import kotlin.io.path.outputStream
-import org.apache.logging.log4j.core.config.plugins.processor.PluginCache
-import org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor.PLUGIN_CACHE_FILE
+import java.io.DataInputStream
+import java.io.DataOutputStream
+import java.util.Locale
+import java.util.SortedMap
 import org.apache.tools.zip.ZipOutputStream
 import org.gradle.api.file.FileTreeElement
 
 /**
+ * Merges Log4j 2 plugin cache files (`Log4j2Plugins.dat`) and relocates the plugin classes in them.
+ *
+ * The binary format is read and written directly instead of depending on `log4j-core`, mirroring
+ * [org.apache.logging.log4j.core.config.plugins.processor.PluginCache](https://github.com/apache/logging-log4j2/blob/2.x/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCache.java).
+ * The format is frozen in Log4j 2. Log4j 3 no longer generates it, but still reads it for plugins
+ * compiled against Log4j 2, see
+ * [org.apache.logging.log4j.plugins.model.PluginRegistry](https://github.com/apache/logging-log4j2/blob/main/log4j-plugins/src/main/java/org/apache/logging/log4j/plugins/model/PluginRegistry.java).
+ * Log4j 3 plugins are registered as
+ * `META-INF/services/org.apache.logging.log4j.plugins.model.PluginService` instead, which can be
+ * merged by [ServiceFileTransformer].
+ *
  * Modified from
  * [org.apache.logging.log4j.maven.plugins.shade.transformer.Log4j2PluginCacheFileTransformer.java](https://github.com/apache/logging-log4j-transform/blob/main/log4j-transform-maven-shade-plugin-extensions/src/main/java/org/apache/logging/log4j/maven/plugins/shade/transformer/Log4j2PluginCacheFileTransformer.java).
  *
@@ -27,66 +31,77 @@ import org.gradle.api.file.FileTreeElement
  */
 @CacheableTransformer
 public open class Log4j2PluginsCacheFileTransformer : ResourceTransformer {
-  /** Log4j config files to share across the transformation stages. */
-  private val tempFiles = mutableListOf<Path>()
+  /** Plugin entries keyed by category and then by plugin key, sorted like Log4j's `PluginCache`. */
+  private val categories = sortedMapOf<String, SortedMap<String, PluginEntry>>()
 
   /** [Relocator] instances to share across the transformation stages. */
   private val tempRelocators = mutableListOf<Relocator>()
 
+  /** Whether any dat file is collected, even if it contains no plugins. */
+  private var hasDatFile = false
+
   override fun canTransformResource(element: FileTreeElement): Boolean {
     return (PLUGIN_CACHE_FILE == element.path).also { flag -> checkDupStrategy(flag, element) }
   }
 
   override fun transform(context: TransformerContext) {
-    val temporaryFile = createTempFile("Log4j2Plugins", ".dat")
-    tempFiles.add(temporaryFile)
-    context.inputStream.copyTo(temporaryFile.outputStream())
+    // Don't close the input stream, it is managed by the caller.
+    val input = DataInputStream(context.inputStream.buffered())
+    repeat(input.readInt()) {
+      val category = categories.getOrPut(input.readUTF().lowercase(Locale.ROOT)) { sortedMapOf() }
+      repeat(input.readInt()) {
+        // Always read all parts of the entry, even if not adding, to advance the stream.
+        val entry =
+          PluginEntry(
+            key = input.readUTF(),
+            className = input.readUTF(),
+            name = input.readUTF(),
+            printable = input.readBoolean(),
+            defer = input.readBoolean(),
+          )
+        // The first found entry wins, same as Log4j's `PluginCache`.
+        category.putIfAbsent(entry.key, entry)
+      }
+    }
     tempRelocators.addAll(context.relocators)
+    hasDatFile = true
   }
 
   /** @return `true` if any dat file collected. */
-  override fun hasTransformedResource(): Boolean = tempFiles.isNotEmpty()
+  override fun hasTransformedResource(): Boolean = hasDatFile
 
   override fun modifyOutputStream(os: ZipOutputStream, preserveFileTimestamps: Boolean) {
-    try {
-      val aggregator = PluginCache()
-      aggregator.loadCacheFiles(urlEnumeration)
-      relocatePlugins(aggregator)
-      os.writeEntry(PLUGIN_CACHE_FILE, preserveFileTimestamps) {
-        // Prevent the aggregator to close the jar output.
-        aggregator.writeCache(CloseShieldOutputStream(this))
-      }
-    } finally {
-      deleteTempFiles()
-    }
-  }
-
-  internal fun relocatePlugins(pluginCache: PluginCache) {
-    pluginCache.allCategories.values.forEach { currentMap ->
-      currentMap.values.forEach { currentPluginEntry ->
-        currentPluginEntry.className = tempRelocators.relocateClass(currentPluginEntry.className)
+    os.writeEntry(PLUGIN_CACHE_FILE, preserveFileTimestamps) {
+      // Don't close the output stream, it would close the jar output.
+      val output = DataOutputStream(buffered())
+      output.writeInt(categories.size)
+      categories.forEach { (category, entries) ->
+        output.writeUTF(category)
+        output.writeInt(entries.size)
+        entries.values.forEach { entry ->
+          output.writeUTF(entry.key)
+          output.writeUTF(tempRelocators.relocateClass(entry.className))
+          output.writeUTF(entry.name)
+          output.writeBoolean(entry.printable)
+          output.writeBoolean(entry.defer)
+        }
       }
+      output.flush()
     }
+    categories.clear()
+    hasDatFile = false
   }
 
-  private fun deleteTempFiles() {
-    val pathIterator = tempFiles.listIterator()
-    while (pathIterator.hasNext()) {
-      val path = pathIterator.next()
-      path.deleteIfExists()
-      pathIterator.remove()
-    }
-  }
+  private class PluginEntry(
+    val key: String,
+    val className: String,
+    val name: String,
+    val printable: Boolean,
+    val defer: Boolean,
+  )
 
-  private val urlEnumeration: Enumeration<URL>
-    get() {
-      val urls = tempFiles.map { it.toUri().toURL() }
-      return Collections.enumeration(urls)
-    }
-
-  private class CloseShieldOutputStream(output: OutputStream) : FilterOutputStream(output) {
-    override fun close() {
-      flush()
-    }
+  internal companion object {
+    const val PLUGIN_CACHE_FILE: String =
+      "META-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat"
   }
 }

src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformerTest.kt

@@ -12,16 +12,17 @@ import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator
 import com.github.jengelman.gradle.plugins.shadow.relocation.SimpleRelocator
 import com.github.jengelman.gradle.plugins.shadow.testkit.JarPath
 import com.github.jengelman.gradle.plugins.shadow.testkit.getBytes
-import com.github.jengelman.gradle.plugins.shadow.testkit.requireResourceAsPath
+import com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer.Companion.PLUGIN_CACHE_FILE
 import com.github.jengelman.gradle.plugins.shadow.util.zipOutputStream
 import java.io.ByteArrayOutputStream
+import java.io.DataOutputStream
 import java.net.URL
 import java.util.Collections
 import java.util.jar.JarInputStream
 import kotlin.io.path.createTempFile
 import kotlin.io.path.writeBytes
 import org.apache.logging.log4j.core.config.plugins.processor.PluginCache
-import org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor.PLUGIN_CACHE_FILE
+import org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor
 import org.apache.tools.zip.ZipOutputStream
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.params.ParameterizedTest
@@ -45,6 +46,11 @@ class Log4j2PluginsCacheFileTransformerTest :
       assertThat(canTransformResource(PLUGIN_CACHE_FILE)).isTrue()
     }
 
+  @Test
+  fun pluginCacheFilePathMatchesLog4j() {
+    assertThat(PLUGIN_CACHE_FILE).isEqualTo(PluginProcessor.PLUGIN_CACHE_FILE)
+  }
+
   @Test
   fun relocateClassesInsideDatFile() =
     with(transformer) {
@@ -56,13 +62,7 @@ class Log4j2PluginsCacheFileTransformerTest :
         modifyOutputStream(zos, true)
       }
 
-      // Extract the .dat file bytes to a temp file to avoid JarURLConnection locking tempJar on
-      // Windows
-      val tempDat = createTempFile(directory = tempDir, suffix = ".dat")
-      tempDat.writeBytes(JarPath(tempJar).use { it.getBytes(PLUGIN_CACHE_FILE) })
-
-      val cache = PluginCache()
-      cache.loadCacheFiles(Collections.enumeration(listOf(tempDat.toUri().toURL())))
+      val cache = loadPluginCache(JarPath(tempJar).use { it.getBytes(PLUGIN_CACHE_FILE) })
 
       assertThat(cache.getCategory("lookup")["date"]?.className)
         .isEqualTo("new.location.org.apache.logging.log4j.core.lookup.DateLookup")
@@ -97,24 +97,93 @@ class Log4j2PluginsCacheFileTransformerTest :
       }
     }
 
+  @Test
+  fun mergeSameAsLog4jPluginCache() =
+    with(transformer) {
+      val one =
+        datBytes(
+          "Lookup" to listOf(entry("date", "foo.DateLookup"), entry("env", "foo.EnvLookup")),
+          "Converter" to listOf(entry("message", "foo.MessageConverter")),
+        )
+      val two =
+        datBytes(
+          "lookup" to
+            listOf(entry("date", "bar.DateLookup"), entry("java", "bar.JavaLookup", true)),
+          "Core" to listOf(entry("console", "bar.ConsoleAppender", defer = true)),
+        )
+      transform(TransformerContext(PLUGIN_CACHE_FILE, one.inputStream()))
+      transform(TransformerContext(PLUGIN_CACHE_FILE, two.inputStream()))
+
+      val actual = transformToJar().use { it.getBytes(PLUGIN_CACHE_FILE) }
+
+      val expected =
+        ByteArrayOutputStream().also { output ->
+          PluginCache().apply {
+            loadCacheFiles(Collections.enumeration(listOf(one, two).map(::datUrl)))
+            writeCache(output)
+          }
+        }
+      assertThat(actual.contentEquals(expected.toByteArray())).isTrue()
+      assertThat(loadPluginCache(actual).getCategory("lookup")["date"]?.className)
+        .isEqualTo("foo.DateLookup")
+    }
+
   @ParameterizedTest
   @MethodSource("relocationProvider")
   fun relocations(pattern: String, shadedPattern: String, expected: String) =
     with(transformer) {
-      val aggregator =
-        PluginCache().apply { loadCacheFiles(Collections.enumeration(listOf(pluginCacheUrl))) }
       transform(context(SimpleRelocator(pattern, shadedPattern)))
-      relocatePlugins(aggregator)
 
-      for (pluginEntryMap in aggregator.allCategories.values) {
+      val cache = loadPluginCache(transformToJar().use { it.getBytes(PLUGIN_CACHE_FILE) })
+
+      for (pluginEntryMap in cache.allCategories.values) {
         for (entry in pluginEntryMap.values) {
           assertThat(entry.className).startsWith(expected)
         }
       }
     }
 
+  private fun loadPluginCache(bytes: ByteArray): PluginCache {
+    return PluginCache().apply { loadCacheFiles(Collections.enumeration(listOf(datUrl(bytes)))) }
+  }
+
+  /** Writes the bytes to a temp file to avoid JarURLConnection locking jars on Windows. */
+  private fun datUrl(bytes: ByteArray): URL {
+    val tempDat = createTempFile(directory = tempDir, suffix = ".dat")
+    tempDat.writeBytes(bytes)
+    return tempDat.toUri().toURL()
+  }
+
+  private class Entry(
+    val key: String,
+    val className: String,
+    val printable: Boolean,
+    val defer: Boolean,
+  )
+
   private companion object {
-    val pluginCacheUrl: URL = requireResourceAsPath(PLUGIN_CACHE_FILE).toUri().toURL()
+    fun entry(key: String, className: String, printable: Boolean = false, defer: Boolean = false) =
+      Entry(key, className, printable, defer)
+
+    /** Writes the raw `Log4j2Plugins.dat` format, keeping the category names as they are. */
+    fun datBytes(vararg categories: Pair<String, List<Entry>>): ByteArray {
+      val bytes = ByteArrayOutputStream()
+      DataOutputStream(bytes).use { output ->
+        output.writeInt(categories.size)
+        categories.forEach { (category, entries) ->
+          output.writeUTF(category)
+          output.writeInt(entries.size)
+          entries.forEach {
+            output.writeUTF(it.key)
+            output.writeUTF(it.className)
+            output.writeUTF(it.key)
+            output.writeBoolean(it.printable)
+            output.writeBoolean(it.defer)
+          }
+        }
+      }
+      return bytes.toByteArray()
+    }
 
     fun context(vararg relocators: Relocator): TransformerContext {
       return resourceContext(PLUGIN_CACHE_FILE, relocators = relocators)