Import: copy a raw repository's files (Nexus, another Forge, a web server) into a hosted raw repository

.github/workflows/ci.yml

@@ -726,6 +726,7 @@ jobs:
           docker pull mirror.gcr.io/library/redis:5.0
           docker pull mirror.gcr.io/sonatype/nexus3:3.70.1
           docker pull mirror.gcr.io/verdaccio/verdaccio:6.11.1
+          docker pull mirror.gcr.io/library/nginx:1.27.5-alpine
 
       - name: Run e2e tests (forge-e2e feature)
         id: tests

README.md

@@ -240,8 +240,8 @@ The guides below are also a site, with navigation and search:
   built into the server: design, operation, and writing one.
 - **[Importing repositories](./docs/import.md)** — copy a hosted repository
   of Nexus, Artifactory, Verdaccio, a registry or another Forge (NuGet, npm,
-  Docker) into Forge through the format's own protocol: dry run, conflicts, resuming, and
-  the cutover.
+  Docker, raw) into Forge through the format's own protocol: dry run,
+  conflicts, resuming, and the cutover.
 
 Run `cargo xtask help` for the complete command list.
 

crates/db/src/import_dao.rs

@@ -25,9 +25,9 @@ use uuid::Uuid;
 pub struct RepositoryImport {
     pub id: i64,
     pub repository_id: i64,
-    /// The source: a NuGet service index or a registry. For an import stored
-    /// before 2.2, the Nexus server that `source_repository` names a
-    /// repository of.
+    /// The source: a NuGet service index, a registry, or a raw repository.
+    /// For an import stored before 2.2, the Nexus server that
+    /// `source_repository` names a repository of.
     pub url: String,
     /// The Nexus repository of an import stored before 2.2; `None` since.
     pub source_repository: Option<String>,
@@ -47,8 +47,9 @@ pub struct RepositoryImport {
     pub state: String,
     /// `scan` or `copy`.
     pub phase: String,
-    /// What the scan walks: `search`, `catalog`, `all` (npm's `/-/all`) or
-    /// `names`.
+    /// What the scan walks: `search`, `catalog`, `all` (npm's `/-/all`),
+    /// `assets` (Nexus' asset listing), `directories` (another Forge's raw
+    /// listing) or `names`.
     pub scan_listing: String,
     #[serde(skip)]
     pub cursor: Option<String>,
@@ -82,7 +83,8 @@ pub struct NewImport<'a> {
     pub on_conflict: &'a str,
     pub concurrency: i32,
     pub dry_run: bool,
-    /// Where the scan starts: `search`, `catalog`, `all` or `names`.
+    /// Where the scan starts: `search`, `catalog`, `all`, `assets`,
+    /// `directories` or `names`.
     pub scan_listing: &'a str,
     pub source_server: Option<&'a str>,
     pub source_total: Option<i64>,
@@ -444,8 +446,9 @@ pub struct ImportItem {
     #[serde(skip)]
     pub import_id: i64,
     /// `component` (a version, a Docker tag), `manifest` (a Docker referrer
-    /// held by digest) or `tag` (an npm dist-tag: `name` the package,
-    /// `version` the tag).
+    /// held by digest), `tag` (an npm dist-tag: `name` the package,
+    /// `version` the tag) or `file` (a raw file: `name` its path, `version`
+    /// empty).
     pub kind: String,
     pub name: String,
     pub version: String,

crates/db/src/raw_dao.rs

@@ -270,6 +270,58 @@ pub async fn store(
     )))
 }
 
+/// When the file at `path` last changed (`updated_at`), locked `FOR UPDATE`
+/// in `tx`: what an import that replaces it keeps its new upload time after.
+pub async fn lock_updated_at(
+    tx: &mut Transaction<'_, Postgres>,
+    repository_id: i64,
+    path: &str,
+) -> sqlx::Result<Option<DateTime<Utc>>> {
+    sqlx::query_scalar(
+        "SELECT updated_at FROM raw_file WHERE repository_id = $1 AND path = $2 FOR UPDATE",
+    )
+    .bind(repository_id)
+    .bind(path)
+    .fetch_optional(&mut **tx)
+    .await
+}
+
+/// Give file `id`, which an import just stored, the times its source names:
+/// `uploaded_at` (when the source stored the content) becomes its upload
+/// time, which `Last-Modified` and retention's age rules read, and
+/// `downloaded_at` its last download.
+///
+/// A file the import `replaced` keeps an upload time after `replaced`, the
+/// one of the content it replaced: a client that cached that content and
+/// asks `If-Modified-Since` must not be told it is current.
+pub async fn set_imported_times(
+    tx: &mut Transaction<'_, Postgres>,
+    id: i64,
+    uploaded_at: Option<DateTime<Utc>>,
+    downloaded_at: Option<DateTime<Utc>>,
+    replaced: Option<DateTime<Utc>>,
+) -> sqlx::Result<()> {
+    sqlx::query(
+        r#"
+        UPDATE raw_file
+        SET created_at = CASE WHEN $4::timestamptz IS NULL
+                              THEN coalesce($2, created_at) ELSE created_at END,
+            updated_at = CASE WHEN $2::timestamptz IS NULL THEN updated_at
+                              WHEN $4::timestamptz IS NULL THEN $2
+                              ELSE greatest($2, $4 + interval '1 microsecond') END,
+            last_downloaded_at = coalesce($3, last_downloaded_at)
+        WHERE id = $1
+        "#,
+    )
+    .bind(id)
+    .bind(uploaded_at)
+    .bind(downloaded_at)
+    .bind(replaced)
+    .execute(&mut **tx)
+    .await?;
+    Ok(())
+}
+
 /// Delete the file at `path`, releasing its blob reference. Returns whether
 /// there was one.
 pub async fn delete(