diff --git a/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectivesSpec.scala b/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectivesSpec.scala index 77a262b031..de3f1e70fe 100644 --- a/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectivesSpec.scala +++ b/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectivesSpec.scala @@ -267,6 +267,35 @@ class FileAndResourceDirectivesSpec extends RoutingSpec with Inspectors with Ins 1.second.dilated).data.asByteBuffer.getInt shouldEqual 0xCAFEBABE } } + "return the resource content from an archive with metadata taken from the archive entry" in { + val route = getFromResource("com/typesafe/config/Config.class") + + def runCheck() = + Get() ~> route ~> check { + val entity = responseEntity.toStrict(1.second.dilated).awaitResult(1.second.dilated) + entity.contentLength shouldEqual entity.data.length + header[`Last-Modified`] shouldBe defined + } + + runCheck() + runCheck() // the archive is shared between requests, so make sure it is still usable afterwards + } + "return the resource content from an archive when the jar file cache is disabled" in { + val route = + withSettings(RoutingSettings(system).withUseJarFileCache(false)) { + getFromResource("com/typesafe/config/Config.class") + } + + def runCheck() = + Get() ~> route ~> check { + val entity = responseEntity.toStrict(1.second.dilated).awaitResult(1.second.dilated) + entity.contentLength shouldEqual entity.data.length + entity.data.asByteBuffer.getInt shouldEqual 0xCAFEBABE + } + + runCheck() + runCheck() // every request opens and closes the jar file of its own, so make sure that is repeatable + } "return the file content with MediaType 'application/octet-stream' on unknown file extensions" in { Get() ~> getFromResource("sample.xyz") ~> check { mediaType shouldEqual `application/octet-stream` diff --git a/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileUploadDirectivesSpec.scala b/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileUploadDirectivesSpec.scala index ce7f211cab..e0a41b0335 100644 --- a/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileUploadDirectivesSpec.scala +++ b/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileUploadDirectivesSpec.scala @@ -458,6 +458,21 @@ class FileUploadDirectivesSpec extends RoutingSpec with Eventually { } + "collect its temporary files in a single directory" in { + // the directory is removed by one shutdown hook, instead of registering every single uploaded file with + // `File.deleteOnExit`, which the JVM would remember for the lifetime of the process + val first = UploadTempFiles.create() + val second = UploadTempFiles.create() + try { + first.getParentFile.getName should startWith("pekko-http-uploads") + second.getParentFile shouldEqual first.getParentFile + (first should not).equal(second) + } finally { + first.delete() + second.delete() + } + } + } private def read(file: File): String = { diff --git a/http/src/main/mima-filters/2.0.x.backwards.excludes/routing-use-jar-file-cache.excludes b/http/src/main/mima-filters/2.0.x.backwards.excludes/routing-use-jar-file-cache.excludes new file mode 100644 index 0000000000..c64d04b47b --- /dev/null +++ b/http/src/main/mima-filters/2.0.x.backwards.excludes/routing-use-jar-file-cache.excludes @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# new use-jar-file-cache routing setting +ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.pekko.http.javadsl.settings.RoutingSettings.getUseJarFileCache") +ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.pekko.http.scaladsl.settings.RoutingSettings.useJarFileCache") diff --git a/http/src/main/resources/reference.conf b/http/src/main/resources/reference.conf index e3f50b7d9a..1c0b69e988 100644 --- a/http/src/main/resources/reference.conf +++ b/http/src/main/resources/reference.conf @@ -19,6 +19,15 @@ pekko.http { # Enables/disables ETag and `If-Modified-Since` support for FileAndResourceDirectives file-get-conditional = on + # Enables/disables the use of the JDK's jar file cache when FileAndResourceDirectives serve a resource that + # lives in a jar file. This is the same cache that the class loader uses, so with the cache enabled a resource + # is served without opening and parsing the jar file for every single request. + # + # Turn this off if the jar files that resources are served from have to be replaceable while the server is + # running (an open jar file cannot be replaced on Windows). Note that this makes every request open and parse + # the jar file again. + use-jar-file-cache = on + # Enables/disables the rendering of the "rendered by" footer in directory listings render-vanity-footer = yes diff --git a/http/src/main/scala/org/apache/pekko/http/impl/settings/RoutingSettingsImpl.scala b/http/src/main/scala/org/apache/pekko/http/impl/settings/RoutingSettingsImpl.scala index 29012c2e30..edc4f5d263 100644 --- a/http/src/main/scala/org/apache/pekko/http/impl/settings/RoutingSettingsImpl.scala +++ b/http/src/main/scala/org/apache/pekko/http/impl/settings/RoutingSettingsImpl.scala @@ -28,7 +28,8 @@ private[http] final case class RoutingSettingsImpl( rangeCountLimit: Int, rangeCoalescingThreshold: Long, decodeMaxBytesPerChunk: Int, - decodeMaxSize: Long) extends pekko.http.scaladsl.settings.RoutingSettings { + decodeMaxSize: Long, + useJarFileCache: Boolean) extends pekko.http.scaladsl.settings.RoutingSettings { override def productPrefix = "RoutingSettings" } @@ -41,5 +42,6 @@ object RoutingSettingsImpl extends SettingsCompanionImpl[RoutingSettingsImpl]("p c.getInt("range-count-limit"), c.getBytes("range-coalescing-threshold"), c.getIntBytes("decode-max-bytes-per-chunk"), - c.getPossiblyInfiniteBytes("decode-max-size")) + c.getPossiblyInfiniteBytes("decode-max-size"), + c.getBoolean("use-jar-file-cache")) } diff --git a/http/src/main/scala/org/apache/pekko/http/javadsl/settings/RoutingSettings.scala b/http/src/main/scala/org/apache/pekko/http/javadsl/settings/RoutingSettings.scala index 3bc0d1bfa3..6f4f9d7d87 100644 --- a/http/src/main/scala/org/apache/pekko/http/javadsl/settings/RoutingSettings.scala +++ b/http/src/main/scala/org/apache/pekko/http/javadsl/settings/RoutingSettings.scala @@ -32,6 +32,11 @@ abstract class RoutingSettings private[pekko] () { self: RoutingSettingsImpl => def getRangeCoalescingThreshold: Long def getDecodeMaxBytesPerChunk: Int + /** + * @since 2.0.0 + */ + def getUseJarFileCache: Boolean + def withVerboseErrorMessages(verboseErrorMessages: Boolean): RoutingSettings = self.copy(verboseErrorMessages = verboseErrorMessages) def withFileGetConditional(fileGetConditional: Boolean): RoutingSettings = @@ -44,6 +49,11 @@ abstract class RoutingSettings private[pekko] () { self: RoutingSettingsImpl => def withDecodeMaxBytesPerChunk(decodeMaxBytesPerChunk: Int): RoutingSettings = self.copy(decodeMaxBytesPerChunk = decodeMaxBytesPerChunk) def withDecodeMaxSize(decodeMaxSize: Long): RoutingSettings = self.copy(decodeMaxSize = decodeMaxSize) + + /** + * @since 2.0.0 + */ + def withUseJarFileCache(useJarFileCache: Boolean): RoutingSettings = self.copy(useJarFileCache = useJarFileCache) } object RoutingSettings extends SettingsCompanion[RoutingSettings] { diff --git a/http/src/main/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectives.scala b/http/src/main/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectives.scala index 19a6c99551..14231a9d3c 100644 --- a/http/src/main/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectives.scala +++ b/http/src/main/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectives.scala @@ -14,8 +14,8 @@ package org.apache.pekko.http.scaladsl.server package directives -import java.io.File -import java.net.{ URI, URL } +import java.io.{ File, FileNotFoundException, InputStream } +import java.net.{ JarURLConnection, URL, URLConnection } import scala.annotation.tailrec import scala.jdk.CollectionConverters._ @@ -109,17 +109,20 @@ trait FileAndResourceDirectives { resourceName: String, contentType: ContentType, classLoader: ClassLoader = _defaultClassLoader): Route = if (!resourceName.endsWith('/')) get { - Option(classLoader.getResource(resourceName)).flatMap(ResourceFile.apply) match { - case Some(ResourceFile(url, length, lastModified)) => - conditionalFor(length, lastModified) { - if (length > 0) { - withRangeSupportAndPrecompressedMediaTypeSupport { - complete(HttpEntity.Default(contentType, length, - StreamConverters.fromInputStream(() => url.openStream()))) - } - } else complete(HttpEntity.Empty) - } - case _ => reject // not found or directory + extractSettings { settings => + val useJarFileCache = settings.useJarFileCache + Option(classLoader.getResource(resourceName)).flatMap(ResourceFile(_, useJarFileCache)) match { + case Some(ResourceFile(url, length, lastModified)) => + conditionalFor(length, lastModified) { + if (length > 0) { + withRangeSupportAndPrecompressedMediaTypeSupport { + complete(HttpEntity.Default(contentType, length, + StreamConverters.fromInputStream(() => openStream(url, useJarFileCache)))) + } + } else complete(HttpEntity.Empty) + } + case _ => reject // not found or directory + } } } else reject // don't serve the content of resource "directories" @@ -278,35 +281,58 @@ object FileAndResourceDirectives extends FileAndResourceDirectives { } object ResourceFile { - def apply(url: URL): Option[ResourceFile] = url.getProtocol match { + def apply(url: URL): Option[ResourceFile] = apply(url, useJarFileCache = true) + + /** + * @param useJarFileCache whether the JDK's jar file cache may be used for resources inside a jar file, see the + * `pekko.http.routing.use-jar-file-cache` setting + * @since 2.0.0 + */ + def apply(url: URL, useJarFileCache: Boolean): Option[ResourceFile] = url.getProtocol match { case "file" => val file = new File(url.toURI) if (file.isDirectory) None else Some(ResourceFile(url, file.length(), file.lastModified())) case "jar" => - val path = new URI(url.getPath).getPath // remove "file:" prefix and normalize whitespace - val bangIndex = path.indexOf('!') - val filePath = path.substring(0, bangIndex) - val resourcePath = path.substring(bangIndex + 2) - val jar = new java.util.zip.ZipFile(filePath) - try { - val entry = jar.getEntry(resourcePath) - if (entry.isDirectory) None - else Option(jar.getInputStream(entry)).map { is => - is.close() - ResourceFile(url, entry.getSize, entry.getTime) - } - } finally jar.close() - case _ => - val conn = url.openConnection() - try { - conn.setUseCaches(false) // otherwise the JDK will keep the connection open when we close! - val len = conn.getContentLength - val lm = conn.getLastModified - Some(ResourceFile(url, len, lm)) - } finally conn.getInputStream.close() + url.openConnection() match { + case jarConnection: JarURLConnection => + // Ask the connection for the entry instead of opening the jar file here: opening it means reading and + // parsing the whole central directory again for every single request. With the cache enabled the JDK + // reuses the same open jar file as the class loader does, so nothing is opened here at all in the common + // case. Without it this connection owns the jar file and has to close it again. + jarConnection.setUseCaches(useJarFileCache) + try { + val entry = Option(jarConnection.getJarEntry).filterNot(_.isDirectory) + if (!useJarFileCache) jarConnection.getJarFile.close() + entry.map(e => ResourceFile(url, e.getSize, e.getTime)) + } catch { + case _: FileNotFoundException => None // the entry disappeared from the jar in the meantime + } + case connection => fromUrlConnection(url, connection) + } + case _ => fromUrlConnection(url, url.openConnection()) } + + private def fromUrlConnection(url: URL, connection: URLConnection): Option[ResourceFile] = + try { + connection.setUseCaches(false) // otherwise the JDK will keep the connection open when we close! + val len = connection.getContentLength + val lm = connection.getLastModified + Some(ResourceFile(url, len, lm)) + } finally connection.getInputStream.close() } + + /** + * Opens the resource content. `URL.openStream` would always use the JDK's caches, so when they are disabled the + * connection has to be set up by hand. Closing the returned stream then also closes the jar file it came from. + */ + private def openStream(url: URL, useJarFileCache: Boolean): InputStream = + if (useJarFileCache || url.getProtocol != "jar") url.openStream() + else { + val connection = url.openConnection() + connection.setUseCaches(false) + connection.getInputStream + } case class ResourceFile(url: URL, length: Long, lastModified: Long) trait DirectoryRenderer extends pekko.http.javadsl.server.directives.DirectoryRenderer { diff --git a/http/src/main/scala/org/apache/pekko/http/scaladsl/server/directives/FileUploadDirectives.scala b/http/src/main/scala/org/apache/pekko/http/scaladsl/server/directives/FileUploadDirectives.scala index f5ca470057..0269ce5781 100644 --- a/http/src/main/scala/org/apache/pekko/http/scaladsl/server/directives/FileUploadDirectives.scala +++ b/http/src/main/scala/org/apache/pekko/http/scaladsl/server/directives/FileUploadDirectives.scala @@ -14,7 +14,7 @@ package org.apache.pekko.http.scaladsl.server.directives import java.io.File -import java.nio.file.Files +import java.nio.file.{ Files, Path } import scala.collection.immutable import scala.concurrent.{ Future, Promise } @@ -22,7 +22,7 @@ import scala.util.{ Failure, Success } import org.apache.pekko import pekko.Done -import pekko.annotation.ApiMayChange +import pekko.annotation.{ ApiMayChange, InternalApi } import pekko.http.impl.util.StreamUtils import pekko.http.javadsl import pekko.http.scaladsl.model.{ ContentType, Multipart } @@ -174,11 +174,7 @@ trait FileUploadDirectives { extractRequestContext.flatMap { ctx => implicit val ec = ctx.executionContext - def tempDest(fileInfo: FileInfo): File = { - val dest = Files.createTempFile("pekko-http-upload", ".tmp").toFile - dest.deleteOnExit() - dest - } + def tempDest(fileInfo: FileInfo): File = UploadTempFiles.create() storeUploadedFiles(fieldName, tempDest).map { files => files.map { @@ -196,6 +192,36 @@ trait FileUploadDirectives { object FileUploadDirectives extends FileUploadDirectives +/** + * INTERNAL API + * + * Temporary files for uploads that the application may never consume. + * + * The files are collected in a directory of their own that a single shutdown hook removes on exit. Registering every + * file with `File.deleteOnExit` instead would keep its path in a JVM-wide set for the lifetime of the process, also + * long after the file itself has been deleted, so that a long-running server accepting uploads would slowly grow its + * heap. + */ +@InternalApi +private[directives] object UploadTempFiles { + private lazy val directory: Path = { + val dir = Files.createTempDirectory("pekko-http-uploads") // owner-only permissions where the file system has them + Runtime.getRuntime.addShutdownHook(new Thread( + () => deleteRecursively(dir.toFile), + "pekko-http-upload-cleanup")) + dir + } + + def create(): File = Files.createTempFile(directory, "pekko-http-upload", ".tmp").toFile + + private def deleteRecursively(file: File): Unit = { + val children = file.listFiles() + if (children ne null) children.foreach(deleteRecursively) + file.delete() + () + } +} + /** * Additional metadata about the file being uploaded/that was uploaded using the [[FileUploadDirectives]] * diff --git a/http/src/main/scala/org/apache/pekko/http/scaladsl/settings/RoutingSettings.scala b/http/src/main/scala/org/apache/pekko/http/scaladsl/settings/RoutingSettings.scala index 9bd12b754b..0c084ffb90 100644 --- a/http/src/main/scala/org/apache/pekko/http/scaladsl/settings/RoutingSettings.scala +++ b/http/src/main/scala/org/apache/pekko/http/scaladsl/settings/RoutingSettings.scala @@ -33,6 +33,14 @@ abstract class RoutingSettings private[pekko] () extends pekko.http.javadsl.sett def decodeMaxBytesPerChunk: Int def decodeMaxSize: Long + /** + * Whether resources that live in a jar file are served through the JDK's jar file cache, the same cache the class + * loader uses, instead of opening and parsing the jar file for every request. + * + * @since 2.0.0 + */ + def useJarFileCache: Boolean + /* Java APIs */ def getVerboseErrorMessages: Boolean = this.verboseErrorMessages def getFileGetConditional: Boolean = this.fileGetConditional @@ -42,6 +50,11 @@ abstract class RoutingSettings private[pekko] () extends pekko.http.javadsl.sett def getDecodeMaxBytesPerChunk: Int = this.decodeMaxBytesPerChunk def getDecodeMaxSize: Long = this.decodeMaxSize + /** + * @since 2.0.0 + */ + def getUseJarFileCache: Boolean = this.useJarFileCache + override def withVerboseErrorMessages(verboseErrorMessages: Boolean): RoutingSettings = self.copy(verboseErrorMessages = verboseErrorMessages) override def withFileGetConditional(fileGetConditional: Boolean): RoutingSettings = @@ -54,6 +67,12 @@ abstract class RoutingSettings private[pekko] () extends pekko.http.javadsl.sett override def withDecodeMaxBytesPerChunk(decodeMaxBytesPerChunk: Int): RoutingSettings = self.copy(decodeMaxBytesPerChunk = decodeMaxBytesPerChunk) override def withDecodeMaxSize(decodeMaxSize: Long): RoutingSettings = self.copy(decodeMaxSize = decodeMaxSize) + + /** + * @since 2.0.0 + */ + override def withUseJarFileCache(useJarFileCache: Boolean): RoutingSettings = + self.copy(useJarFileCache = useJarFileCache) } object RoutingSettings extends SettingsCompanion[RoutingSettings] {