Avoid reopening jars per resource request and deleteOnExit per upload

http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectivesSpec.scala

@@ -267,6 +267,35 @@ class FileAndResourceDirectivesSpec extends RoutingSpec with Inspectors with Ins
           1.second.dilated).data.asByteBuffer.getInt shouldEqual 0xCAFEBABE
       }
     }
+    "return the resource content from an archive with metadata taken from the archive entry" in {
+      val route = getFromResource("com/typesafe/config/Config.class")
+
+      def runCheck() =
+        Get() ~> route ~> check {
+          val entity = responseEntity.toStrict(1.second.dilated).awaitResult(1.second.dilated)
+          entity.contentLength shouldEqual entity.data.length
+          header[`Last-Modified`] shouldBe defined
+        }
+
+      runCheck()
+      runCheck() // the archive is shared between requests, so make sure it is still usable afterwards
+    }
+    "return the resource content from an archive when the jar file cache is disabled" in {
+      val route =
+        withSettings(RoutingSettings(system).withUseJarFileCache(false)) {
+          getFromResource("com/typesafe/config/Config.class")
+        }
+
+      def runCheck() =
+        Get() ~> route ~> check {
+          val entity = responseEntity.toStrict(1.second.dilated).awaitResult(1.second.dilated)
+          entity.contentLength shouldEqual entity.data.length
+          entity.data.asByteBuffer.getInt shouldEqual 0xCAFEBABE
+        }
+
+      runCheck()
+      runCheck() // every request opens and closes the jar file of its own, so make sure that is repeatable
+    }
     "return the file content with MediaType 'application/octet-stream' on unknown file extensions" in {
       Get() ~> getFromResource("sample.xyz") ~> check {
         mediaType shouldEqual `application/octet-stream`

http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileUploadDirectivesSpec.scala

@@ -458,6 +458,21 @@ class FileUploadDirectivesSpec extends RoutingSpec with Eventually {
 
     }
 
+    "collect its temporary files in a single directory" in {
+      // the directory is removed by one shutdown hook, instead of registering every single uploaded file with
+      // `File.deleteOnExit`, which the JVM would remember for the lifetime of the process
+      val first = UploadTempFiles.create()
+      val second = UploadTempFiles.create()
+      try {
+        first.getParentFile.getName should startWith("pekko-http-uploads")
+        second.getParentFile shouldEqual first.getParentFile
+        (first should not).equal(second)
+      } finally {
+        first.delete()
+        second.delete()
+      }
+    }
+
   }
 
   private def read(file: File): String = {

http/src/main/mima-filters/2.0.x.backwards.excludes/routing-use-jar-file-cache.excludes

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# new use-jar-file-cache routing setting
+ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.pekko.http.javadsl.settings.RoutingSettings.getUseJarFileCache")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.pekko.http.scaladsl.settings.RoutingSettings.useJarFileCache")

http/src/main/resources/reference.conf

@@ -19,6 +19,15 @@ pekko.http {
     # Enables/disables ETag and `If-Modified-Since` support for FileAndResourceDirectives
     file-get-conditional = on
 
+    # Enables/disables the use of the JDK's jar file cache when FileAndResourceDirectives serve a resource that
+    # lives in a jar file. This is the same cache that the class loader uses, so with the cache enabled a resource
+    # is served without opening and parsing the jar file for every single request.
+    #
+    # Turn this off if the jar files that resources are served from have to be replaceable while the server is
+    # running (an open jar file cannot be replaced on Windows). Note that this makes every request open and parse
+    # the jar file again.
+    use-jar-file-cache = on
+
     # Enables/disables the rendering of the "rendered by" footer in directory listings
     render-vanity-footer = yes
 

http/src/main/scala/org/apache/pekko/http/impl/settings/RoutingSettingsImpl.scala

@@ -28,7 +28,8 @@ private[http] final case class RoutingSettingsImpl(
     rangeCountLimit: Int,
     rangeCoalescingThreshold: Long,
     decodeMaxBytesPerChunk: Int,
-    decodeMaxSize: Long) extends pekko.http.scaladsl.settings.RoutingSettings {
+    decodeMaxSize: Long,
+    useJarFileCache: Boolean) extends pekko.http.scaladsl.settings.RoutingSettings {
 
   override def productPrefix = "RoutingSettings"
 }
@@ -41,5 +42,6 @@ object RoutingSettingsImpl extends SettingsCompanionImpl[RoutingSettingsImpl]("p
     c.getInt("range-count-limit"),
     c.getBytes("range-coalescing-threshold"),
     c.getIntBytes("decode-max-bytes-per-chunk"),
-    c.getPossiblyInfiniteBytes("decode-max-size"))
+    c.getPossiblyInfiniteBytes("decode-max-size"),
+    c.getBoolean("use-jar-file-cache"))
 }

http/src/main/scala/org/apache/pekko/http/javadsl/settings/RoutingSettings.scala

@@ -32,6 +32,11 @@ abstract class RoutingSettings private[pekko] () { self: RoutingSettingsImpl =>
   def getRangeCoalescingThreshold: Long
   def getDecodeMaxBytesPerChunk: Int
 
+  /**
+   * @since 2.0.0
+   */
+  def getUseJarFileCache: Boolean
+
   def withVerboseErrorMessages(verboseErrorMessages: Boolean): RoutingSettings =
     self.copy(verboseErrorMessages = verboseErrorMessages)
   def withFileGetConditional(fileGetConditional: Boolean): RoutingSettings =
@@ -44,6 +49,11 @@ abstract class RoutingSettings private[pekko] () { self: RoutingSettingsImpl =>
   def withDecodeMaxBytesPerChunk(decodeMaxBytesPerChunk: Int): RoutingSettings =
     self.copy(decodeMaxBytesPerChunk = decodeMaxBytesPerChunk)
   def withDecodeMaxSize(decodeMaxSize: Long): RoutingSettings = self.copy(decodeMaxSize = decodeMaxSize)
+
+  /**
+   * @since 2.0.0
+   */
+  def withUseJarFileCache(useJarFileCache: Boolean): RoutingSettings = self.copy(useJarFileCache = useJarFileCache)
 }
 
 object RoutingSettings extends SettingsCompanion[RoutingSettings] {

http/src/main/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectives.scala

@@ -14,8 +14,8 @@
 package org.apache.pekko.http.scaladsl.server
 package directives
 
-import java.io.File
-import java.net.{ URI, URL }
+import java.io.{ File, FileNotFoundException, InputStream }
+import java.net.{ JarURLConnection, URL, URLConnection }
 
 import scala.annotation.tailrec
 import scala.jdk.CollectionConverters._
@@ -109,17 +109,20 @@ trait FileAndResourceDirectives {
       resourceName: String, contentType: ContentType, classLoader: ClassLoader = _defaultClassLoader): Route =
     if (!resourceName.endsWith('/'))
       get {
-        Option(classLoader.getResource(resourceName)).flatMap(ResourceFile.apply) match {
-          case Some(ResourceFile(url, length, lastModified)) =>
-            conditionalFor(length, lastModified) {
-              if (length > 0) {
-                withRangeSupportAndPrecompressedMediaTypeSupport {
-                  complete(HttpEntity.Default(contentType, length,
-                    StreamConverters.fromInputStream(() => url.openStream())))
-                }
-              } else complete(HttpEntity.Empty)
-            }
-          case _ => reject // not found or directory
+        extractSettings { settings =>
+          val useJarFileCache = settings.useJarFileCache
+          Option(classLoader.getResource(resourceName)).flatMap(ResourceFile(_, useJarFileCache)) match {
+            case Some(ResourceFile(url, length, lastModified)) =>
+              conditionalFor(length, lastModified) {
+                if (length > 0) {
+                  withRangeSupportAndPrecompressedMediaTypeSupport {
+                    complete(HttpEntity.Default(contentType, length,
+                      StreamConverters.fromInputStream(() => openStream(url, useJarFileCache))))
+                  }
+                } else complete(HttpEntity.Empty)
+              }
+            case _ => reject // not found or directory
+          }
         }
       }
     else reject // don't serve the content of resource "directories"
@@ -278,35 +281,58 @@ object FileAndResourceDirectives extends FileAndResourceDirectives {
   }
 
   object ResourceFile {
-    def apply(url: URL): Option[ResourceFile] = url.getProtocol match {
+    def apply(url: URL): Option[ResourceFile] = apply(url, useJarFileCache = true)
+
+    /**
+     * @param useJarFileCache whether the JDK's jar file cache may be used for resources inside a jar file, see the
+     *                        `pekko.http.routing.use-jar-file-cache` setting
+     * @since 2.0.0
+     */
+    def apply(url: URL, useJarFileCache: Boolean): Option[ResourceFile] = url.getProtocol match {
       case "file" =>
         val file = new File(url.toURI)
         if (file.isDirectory) None
         else Some(ResourceFile(url, file.length(), file.lastModified()))
       case "jar" =>
-        val path = new URI(url.getPath).getPath // remove "file:" prefix and normalize whitespace
-        val bangIndex = path.indexOf('!')
-        val filePath = path.substring(0, bangIndex)
-        val resourcePath = path.substring(bangIndex + 2)
-        val jar = new java.util.zip.ZipFile(filePath)
-        try {
-          val entry = jar.getEntry(resourcePath)
-          if (entry.isDirectory) None
-          else Option(jar.getInputStream(entry)).map { is =>
-            is.close()
-            ResourceFile(url, entry.getSize, entry.getTime)
-          }
-        } finally jar.close()
-      case _ =>
-        val conn = url.openConnection()
-        try {
-          conn.setUseCaches(false) // otherwise the JDK will keep the connection open when we close!
-          val len = conn.getContentLength
-          val lm = conn.getLastModified
-          Some(ResourceFile(url, len, lm))
-        } finally conn.getInputStream.close()
+        url.openConnection() match {
+          case jarConnection: JarURLConnection =>
+            // Ask the connection for the entry instead of opening the jar file here: opening it means reading and
+            // parsing the whole central directory again for every single request. With the cache enabled the JDK
+            // reuses the same open jar file as the class loader does, so nothing is opened here at all in the common
+            // case. Without it this connection owns the jar file and has to close it again.
+            jarConnection.setUseCaches(useJarFileCache)
+            try {
+              val entry = Option(jarConnection.getJarEntry).filterNot(_.isDirectory)
+              if (!useJarFileCache) jarConnection.getJarFile.close()
+              entry.map(e => ResourceFile(url, e.getSize, e.getTime))
+            } catch {
+              case _: FileNotFoundException => None // the entry disappeared from the jar in the meantime
+            }
+          case connection => fromUrlConnection(url, connection)
+        }
+      case _ => fromUrlConnection(url, url.openConnection())
     }
+
+    private def fromUrlConnection(url: URL, connection: URLConnection): Option[ResourceFile] =
+      try {
+        connection.setUseCaches(false) // otherwise the JDK will keep the connection open when we close!
+        val len = connection.getContentLength
+        val lm = connection.getLastModified
+        Some(ResourceFile(url, len, lm))
+      } finally connection.getInputStream.close()
   }
+
+  /**
+   * Opens the resource content. `URL.openStream` would always use the JDK's caches, so when they are disabled the
+   * connection has to be set up by hand. Closing the returned stream then also closes the jar file it came from.
+   */
+  private def openStream(url: URL, useJarFileCache: Boolean): InputStream =
+    if (useJarFileCache || url.getProtocol != "jar") url.openStream()
+    else {
+      val connection = url.openConnection()
+      connection.setUseCaches(false)
+      connection.getInputStream
+    }
   case class ResourceFile(url: URL, length: Long, lastModified: Long)
 
   trait DirectoryRenderer extends pekko.http.javadsl.server.directives.DirectoryRenderer {

http/src/main/scala/org/apache/pekko/http/scaladsl/server/directives/FileUploadDirectives.scala

@@ -14,15 +14,15 @@
 package org.apache.pekko.http.scaladsl.server.directives
 
 import java.io.File
-import java.nio.file.Files
+import java.nio.file.{ Files, Path }
 
 import scala.collection.immutable
 import scala.concurrent.{ Future, Promise }
 import scala.util.{ Failure, Success }
 
 import org.apache.pekko
 import pekko.Done
-import pekko.annotation.ApiMayChange
+import pekko.annotation.{ ApiMayChange, InternalApi }
 import pekko.http.impl.util.StreamUtils
 import pekko.http.javadsl
 import pekko.http.scaladsl.model.{ ContentType, Multipart }
@@ -174,11 +174,7 @@ trait FileUploadDirectives {
     extractRequestContext.flatMap { ctx =>
       implicit val ec = ctx.executionContext
 
-      def tempDest(fileInfo: FileInfo): File = {
-        val dest = Files.createTempFile("pekko-http-upload", ".tmp").toFile
-        dest.deleteOnExit()
-        dest
-      }
+      def tempDest(fileInfo: FileInfo): File = UploadTempFiles.create()
 
       storeUploadedFiles(fieldName, tempDest).map { files =>
         files.map {
@@ -196,6 +192,36 @@ trait FileUploadDirectives {
 
 object FileUploadDirectives extends FileUploadDirectives
 
+/**
+ * INTERNAL API
+ *
+ * Temporary files for uploads that the application may never consume.
+ *
+ * The files are collected in a directory of their own that a single shutdown hook removes on exit. Registering every
+ * file with `File.deleteOnExit` instead would keep its path in a JVM-wide set for the lifetime of the process, also
+ * long after the file itself has been deleted, so that a long-running server accepting uploads would slowly grow its
+ * heap.
+ */
+@InternalApi
+private[directives] object UploadTempFiles {
+  private lazy val directory: Path = {
+    val dir = Files.createTempDirectory("pekko-http-uploads") // owner-only permissions where the file system has them
+    Runtime.getRuntime.addShutdownHook(new Thread(
+      () => deleteRecursively(dir.toFile),
+      "pekko-http-upload-cleanup"))
+    dir
+  }
+
+  def create(): File = Files.createTempFile(directory, "pekko-http-upload", ".tmp").toFile
+
+  private def deleteRecursively(file: File): Unit = {
+    val children = file.listFiles()
+    if (children ne null) children.foreach(deleteRecursively)
+    file.delete()
+    ()
+  }
+}
+
 /**
  * Additional metadata about the file being uploaded/that was uploaded using the [[FileUploadDirectives]]
  *

http/src/main/scala/org/apache/pekko/http/scaladsl/settings/RoutingSettings.scala

@@ -33,6 +33,14 @@ abstract class RoutingSettings private[pekko] () extends pekko.http.javadsl.sett
   def decodeMaxBytesPerChunk: Int
   def decodeMaxSize: Long
 
+  /**
+   * Whether resources that live in a jar file are served through the JDK's jar file cache, the same cache the class
+   * loader uses, instead of opening and parsing the jar file for every request.
+   *
+   * @since 2.0.0
+   */
+  def useJarFileCache: Boolean
+
   /* Java APIs */
   def getVerboseErrorMessages: Boolean = this.verboseErrorMessages
   def getFileGetConditional: Boolean = this.fileGetConditional
@@ -42,6 +50,11 @@ abstract class RoutingSettings private[pekko] () extends pekko.http.javadsl.sett
   def getDecodeMaxBytesPerChunk: Int = this.decodeMaxBytesPerChunk
   def getDecodeMaxSize: Long = this.decodeMaxSize
 
+  /**
+   * @since 2.0.0
+   */
+  def getUseJarFileCache: Boolean = this.useJarFileCache
+
   override def withVerboseErrorMessages(verboseErrorMessages: Boolean): RoutingSettings =
     self.copy(verboseErrorMessages = verboseErrorMessages)
   override def withFileGetConditional(fileGetConditional: Boolean): RoutingSettings =
@@ -54,6 +67,12 @@ abstract class RoutingSettings private[pekko] () extends pekko.http.javadsl.sett
   override def withDecodeMaxBytesPerChunk(decodeMaxBytesPerChunk: Int): RoutingSettings =
     self.copy(decodeMaxBytesPerChunk = decodeMaxBytesPerChunk)
   override def withDecodeMaxSize(decodeMaxSize: Long): RoutingSettings = self.copy(decodeMaxSize = decodeMaxSize)
+
+  /**
+   * @since 2.0.0
+   */
+  override def withUseJarFileCache(useJarFileCache: Boolean): RoutingSettings =
+    self.copy(useJarFileCache = useJarFileCache)
 }
 
 object RoutingSettings extends SettingsCompanion[RoutingSettings] {