From 9480ce49c0192b320a69e0196e63e2e64310deb1 Mon Sep 17 00:00:00 2001 From: Goooler Date: Sun, 25 Oct 2026 15:57:48 +0800 Subject: [PATCH 1/2] Remove runtime dependency on log4j-core Read and write `Log4j2Plugins.dat` files directly in `Log4j2PluginsCacheFileTransformer` instead of using Log4j's `PluginCache`, so Shadow no longer pulls `log4j-core` and its CVEs into users' build classpaths. The output stays byte-for-byte identical, which is verified against `PluginCache` in tests. Closes #2060. --- CHANGELOG.md | 1 + build.gradle.kts | 3 +- .../shadow/transformers/TransformersTest.kt | 2 +- .../Log4j2PluginsCacheFileTransformer.kt | 119 ++++++++++-------- .../Log4j2PluginsCacheFileTransformerTest.kt | 97 +++++++++++--- 5 files changed, 154 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21f10487f..b5baab853 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - Bump min Gradle requirement to 9.4.0. ([#2114](https://github.com/GradleUp/shadow/pull/2114)) - Remove runtime dependencies on Commons Codec and Commons IO by using JDK APIs. ([#2136](https://github.com/GradleUp/shadow/pull/2136)) +- Remove runtime dependency on `log4j-core` by reading and writing `Log4j2Plugins.dat` files directly in `Log4j2PluginsCacheFileTransformer`. ([#2343](https://github.com/GradleUp/shadow/pull/2343)) - **POTENTIALLY BREAKING:** Remove `Serializable` from `DependencyFilter`. ([#2144](https://github.com/GradleUp/shadow/pull/2144)) - Bump default R8 from `9.1.31` to `9.5.28`. ([#2340](https://github.com/GradleUp/shadow/pull/2340)) - Allow repackaging Service file classes with R8. ([#2174](https://github.com/GradleUp/shadow/pull/2174)) diff --git a/build.gradle.kts b/build.gradle.kts index e6f99982b..4c1dc6083 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -132,7 +132,6 @@ dependencies { compileOnly(libs.kotlin.gradlePlugin) compileOnly(libs.kotlin.reflect) api(libs.apache.ant) // Types from Ant are exposed in the public API. - implementation(libs.apache.log4j) implementation(libs.jdependency) implementation(libs.jdom2) implementation(libs.kotlin.metadata) @@ -155,6 +154,8 @@ dependencies { testing.suites { named("test") { dependencies { + // Used to verify the compatibility of the Log4j plugin cache files written by Shadow. + implementation(libs.apache.log4j) implementation(libs.xmlunit) } } diff --git a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/TransformersTest.kt b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/TransformersTest.kt index a56eaef39..88a9c94cf 100644 --- a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/TransformersTest.kt +++ b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/TransformersTest.kt @@ -14,11 +14,11 @@ import com.github.jengelman.gradle.plugins.shadow.testkit.getContent import com.github.jengelman.gradle.plugins.shadow.testkit.getStream import com.github.jengelman.gradle.plugins.shadow.testkit.requireResourceAsPath import com.github.jengelman.gradle.plugins.shadow.testkit.requireResourceAsStream +import com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer.Companion.PLUGIN_CACHE_FILE import java.util.jar.Attributes as JarAttribute import kotlin.io.path.appendText import kotlin.io.path.readText import kotlin.io.path.writeText -import org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor.PLUGIN_CACHE_FILE import org.junit.jupiter.api.Test class TransformersTest : BaseTransformerTest() { diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformer.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformer.kt index 54792f0b4..12482e919 100644 --- a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformer.kt +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformer.kt @@ -4,21 +4,25 @@ import com.github.jengelman.gradle.plugins.shadow.internal.checkDupStrategy import com.github.jengelman.gradle.plugins.shadow.internal.writeEntry import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator import com.github.jengelman.gradle.plugins.shadow.relocation.relocateClass -import java.io.FilterOutputStream -import java.io.OutputStream -import java.net.URL -import java.nio.file.Path -import java.util.Collections -import java.util.Enumeration -import kotlin.io.path.createTempFile -import kotlin.io.path.deleteIfExists -import kotlin.io.path.outputStream -import org.apache.logging.log4j.core.config.plugins.processor.PluginCache -import org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor.PLUGIN_CACHE_FILE +import java.io.DataInputStream +import java.io.DataOutputStream +import java.util.Locale +import java.util.SortedMap import org.apache.tools.zip.ZipOutputStream import org.gradle.api.file.FileTreeElement /** + * Merges Log4j 2 plugin cache files (`Log4j2Plugins.dat`) and relocates the plugin classes in them. + * + * The binary format is read and written directly instead of depending on `log4j-core`, mirroring + * [org.apache.logging.log4j.core.config.plugins.processor.PluginCache](https://github.com/apache/logging-log4j2/blob/2.x/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/processor/PluginCache.java). + * The format is frozen in Log4j 2. Log4j 3 no longer generates it, but still reads it for plugins + * compiled against Log4j 2, see + * [org.apache.logging.log4j.plugins.model.PluginRegistry](https://github.com/apache/logging-log4j2/blob/main/log4j-plugins/src/main/java/org/apache/logging/log4j/plugins/model/PluginRegistry.java). + * Log4j 3 plugins are registered as + * `META-INF/services/org.apache.logging.log4j.plugins.model.PluginService` instead, which can be + * merged by [ServiceFileTransformer]. + * * Modified from * [org.apache.logging.log4j.maven.plugins.shade.transformer.Log4j2PluginCacheFileTransformer.java](https://github.com/apache/logging-log4j-transform/blob/main/log4j-transform-maven-shade-plugin-extensions/src/main/java/org/apache/logging/log4j/maven/plugins/shade/transformer/Log4j2PluginCacheFileTransformer.java). * @@ -27,66 +31,77 @@ import org.gradle.api.file.FileTreeElement */ @CacheableTransformer public open class Log4j2PluginsCacheFileTransformer : ResourceTransformer { - /** Log4j config files to share across the transformation stages. */ - private val tempFiles = mutableListOf() + /** Plugin entries keyed by category and then by plugin key, sorted like Log4j's `PluginCache`. */ + private val categories = sortedMapOf>() /** [Relocator] instances to share across the transformation stages. */ private val tempRelocators = mutableListOf() + /** Whether any dat file is collected, even if it contains no plugins. */ + private var hasDatFile = false + override fun canTransformResource(element: FileTreeElement): Boolean { return (PLUGIN_CACHE_FILE == element.path).also { flag -> checkDupStrategy(flag, element) } } override fun transform(context: TransformerContext) { - val temporaryFile = createTempFile("Log4j2Plugins", ".dat") - tempFiles.add(temporaryFile) - context.inputStream.copyTo(temporaryFile.outputStream()) + // Don't close the input stream, it is managed by the caller. + val input = DataInputStream(context.inputStream.buffered()) + repeat(input.readInt()) { + val category = categories.getOrPut(input.readUTF().lowercase(Locale.ROOT)) { sortedMapOf() } + repeat(input.readInt()) { + // Always read all parts of the entry, even if not adding, to advance the stream. + val entry = + PluginEntry( + key = input.readUTF(), + className = input.readUTF(), + name = input.readUTF(), + printable = input.readBoolean(), + defer = input.readBoolean(), + ) + // The first found entry wins, same as Log4j's `PluginCache`. + category.putIfAbsent(entry.key, entry) + } + } tempRelocators.addAll(context.relocators) + hasDatFile = true } /** @return `true` if any dat file collected. */ - override fun hasTransformedResource(): Boolean = tempFiles.isNotEmpty() + override fun hasTransformedResource(): Boolean = hasDatFile override fun modifyOutputStream(os: ZipOutputStream, preserveFileTimestamps: Boolean) { - try { - val aggregator = PluginCache() - aggregator.loadCacheFiles(urlEnumeration) - relocatePlugins(aggregator) - os.writeEntry(PLUGIN_CACHE_FILE, preserveFileTimestamps) { - // Prevent the aggregator to close the jar output. - aggregator.writeCache(CloseShieldOutputStream(this)) - } - } finally { - deleteTempFiles() - } - } - - internal fun relocatePlugins(pluginCache: PluginCache) { - pluginCache.allCategories.values.forEach { currentMap -> - currentMap.values.forEach { currentPluginEntry -> - currentPluginEntry.className = tempRelocators.relocateClass(currentPluginEntry.className) + os.writeEntry(PLUGIN_CACHE_FILE, preserveFileTimestamps) { + // Don't close the output stream, it would close the jar output. + val output = DataOutputStream(buffered()) + output.writeInt(categories.size) + categories.forEach { (category, entries) -> + output.writeUTF(category) + output.writeInt(entries.size) + entries.values.forEach { entry -> + output.writeUTF(entry.key) + output.writeUTF(tempRelocators.relocateClass(entry.className)) + output.writeUTF(entry.name) + output.writeBoolean(entry.printable) + output.writeBoolean(entry.defer) + } } + output.flush() } + categories.clear() + hasDatFile = false } - private fun deleteTempFiles() { - val pathIterator = tempFiles.listIterator() - while (pathIterator.hasNext()) { - val path = pathIterator.next() - path.deleteIfExists() - pathIterator.remove() - } - } + private class PluginEntry( + val key: String, + val className: String, + val name: String, + val printable: Boolean, + val defer: Boolean, + ) - private val urlEnumeration: Enumeration - get() { - val urls = tempFiles.map { it.toUri().toURL() } - return Collections.enumeration(urls) - } - - private class CloseShieldOutputStream(output: OutputStream) : FilterOutputStream(output) { - override fun close() { - flush() - } + internal companion object { + const val PLUGIN_CACHE_FILE: String = + "META-INF/org/apache/logging/log4j/core/config/plugins/Log4j2Plugins.dat" } } diff --git a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformerTest.kt b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformerTest.kt index dcecb4709..c1bf5d417 100644 --- a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformerTest.kt +++ b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/Log4j2PluginsCacheFileTransformerTest.kt @@ -12,16 +12,17 @@ import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator import com.github.jengelman.gradle.plugins.shadow.relocation.SimpleRelocator import com.github.jengelman.gradle.plugins.shadow.testkit.JarPath import com.github.jengelman.gradle.plugins.shadow.testkit.getBytes -import com.github.jengelman.gradle.plugins.shadow.testkit.requireResourceAsPath +import com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer.Companion.PLUGIN_CACHE_FILE import com.github.jengelman.gradle.plugins.shadow.util.zipOutputStream import java.io.ByteArrayOutputStream +import java.io.DataOutputStream import java.net.URL import java.util.Collections import java.util.jar.JarInputStream import kotlin.io.path.createTempFile import kotlin.io.path.writeBytes import org.apache.logging.log4j.core.config.plugins.processor.PluginCache -import org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor.PLUGIN_CACHE_FILE +import org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor import org.apache.tools.zip.ZipOutputStream import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest @@ -45,6 +46,11 @@ class Log4j2PluginsCacheFileTransformerTest : assertThat(canTransformResource(PLUGIN_CACHE_FILE)).isTrue() } + @Test + fun pluginCacheFilePathMatchesLog4j() { + assertThat(PLUGIN_CACHE_FILE).isEqualTo(PluginProcessor.PLUGIN_CACHE_FILE) + } + @Test fun relocateClassesInsideDatFile() = with(transformer) { @@ -56,13 +62,7 @@ class Log4j2PluginsCacheFileTransformerTest : modifyOutputStream(zos, true) } - // Extract the .dat file bytes to a temp file to avoid JarURLConnection locking tempJar on - // Windows - val tempDat = createTempFile(directory = tempDir, suffix = ".dat") - tempDat.writeBytes(JarPath(tempJar).use { it.getBytes(PLUGIN_CACHE_FILE) }) - - val cache = PluginCache() - cache.loadCacheFiles(Collections.enumeration(listOf(tempDat.toUri().toURL()))) + val cache = loadPluginCache(JarPath(tempJar).use { it.getBytes(PLUGIN_CACHE_FILE) }) assertThat(cache.getCategory("lookup")["date"]?.className) .isEqualTo("new.location.org.apache.logging.log4j.core.lookup.DateLookup") @@ -97,24 +97,93 @@ class Log4j2PluginsCacheFileTransformerTest : } } + @Test + fun mergeSameAsLog4jPluginCache() = + with(transformer) { + val one = + datBytes( + "Lookup" to listOf(entry("date", "foo.DateLookup"), entry("env", "foo.EnvLookup")), + "Converter" to listOf(entry("message", "foo.MessageConverter")), + ) + val two = + datBytes( + "lookup" to + listOf(entry("date", "bar.DateLookup"), entry("java", "bar.JavaLookup", true)), + "Core" to listOf(entry("console", "bar.ConsoleAppender", defer = true)), + ) + transform(TransformerContext(PLUGIN_CACHE_FILE, one.inputStream())) + transform(TransformerContext(PLUGIN_CACHE_FILE, two.inputStream())) + + val actual = transformToJar().use { it.getBytes(PLUGIN_CACHE_FILE) } + + val expected = + ByteArrayOutputStream().also { output -> + PluginCache().apply { + loadCacheFiles(Collections.enumeration(listOf(one, two).map(::datUrl))) + writeCache(output) + } + } + assertThat(actual.contentEquals(expected.toByteArray())).isTrue() + assertThat(loadPluginCache(actual).getCategory("lookup")["date"]?.className) + .isEqualTo("foo.DateLookup") + } + @ParameterizedTest @MethodSource("relocationProvider") fun relocations(pattern: String, shadedPattern: String, expected: String) = with(transformer) { - val aggregator = - PluginCache().apply { loadCacheFiles(Collections.enumeration(listOf(pluginCacheUrl))) } transform(context(SimpleRelocator(pattern, shadedPattern))) - relocatePlugins(aggregator) - for (pluginEntryMap in aggregator.allCategories.values) { + val cache = loadPluginCache(transformToJar().use { it.getBytes(PLUGIN_CACHE_FILE) }) + + for (pluginEntryMap in cache.allCategories.values) { for (entry in pluginEntryMap.values) { assertThat(entry.className).startsWith(expected) } } } + private fun loadPluginCache(bytes: ByteArray): PluginCache { + return PluginCache().apply { loadCacheFiles(Collections.enumeration(listOf(datUrl(bytes)))) } + } + + /** Writes the bytes to a temp file to avoid JarURLConnection locking jars on Windows. */ + private fun datUrl(bytes: ByteArray): URL { + val tempDat = createTempFile(directory = tempDir, suffix = ".dat") + tempDat.writeBytes(bytes) + return tempDat.toUri().toURL() + } + + private class Entry( + val key: String, + val className: String, + val printable: Boolean, + val defer: Boolean, + ) + private companion object { - val pluginCacheUrl: URL = requireResourceAsPath(PLUGIN_CACHE_FILE).toUri().toURL() + fun entry(key: String, className: String, printable: Boolean = false, defer: Boolean = false) = + Entry(key, className, printable, defer) + + /** Writes the raw `Log4j2Plugins.dat` format, keeping the category names as they are. */ + fun datBytes(vararg categories: Pair>): ByteArray { + val bytes = ByteArrayOutputStream() + DataOutputStream(bytes).use { output -> + output.writeInt(categories.size) + categories.forEach { (category, entries) -> + output.writeUTF(category) + output.writeInt(entries.size) + entries.forEach { + output.writeUTF(it.key) + output.writeUTF(it.className) + output.writeUTF(it.key) + output.writeBoolean(it.printable) + output.writeBoolean(it.defer) + } + } + } + return bytes.toByteArray() + } fun context(vararg relocators: Relocator): TransformerContext { return resourceContext(PLUGIN_CACHE_FILE, relocators = relocators) From 79bba0777de7aa2375cd8f56891404ecc3672852 Mon Sep 17 00:00:00 2001 From: Zongle Wang Date: Sun, 25 Oct 2026 16:03:43 +0800 Subject: [PATCH 2/2] Apply batched suggestions from code review Co-authored-by: Zongle Wang --- build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4c1dc6083..0b42398a2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -154,7 +154,6 @@ dependencies { testing.suites { named("test") { dependencies { - // Used to verify the compatibility of the Log4j plugin cache files written by Shadow. implementation(libs.apache.log4j) implementation(libs.xmlunit) }