Remove legacy manifest publishing and package-set mirroring

.env.example

@@ -45,7 +45,7 @@ SPACES_SECRET="digitalocean_spaces_secret"
 # READONLY=true
 
 # -----------------------------------------------------------------------------
-# Script-only Secrets (not used by server, used by scripts like legacy-importer)
+# Secrets for scripts such as daily-importer, not the server
 # -----------------------------------------------------------------------------
 
 # Personal GitHub token for API requests when running scripts

AGENTS.md

@@ -94,7 +94,6 @@ nixfmt *.nix nix/**/*.nix
 - `scripts/` — Runnable modules for registry tasks (PackageTransferrer, PackageSetUpdater, DailyImporter, etc.). Run via `nix run .#package-transferrer`, etc.
 - `test-utils/` — Shared test utilities.
 - `db/` — SQLite schemas and migrations (use `dbmate up` to initialize).
-- `types/` — Dhall type specifications.
 - `nix/` — Nix build and deployment configuration.
 
 ## Scratch Directory & Caching
@@ -104,7 +103,7 @@ The `scratch/` directory (gitignored) is used by scripts for:
 - `logs/` — Log files
 - `registry/`, `registry-index/` — Local clones for testing, also modified and optionally committed to by scripts
 
-Caching is critical for the legacy importer due to the expense of downloading packages. The `Registry.App.Effect.Cache` module handles caching.
+The `Registry.App.Effect.Cache` module caches API responses and package data to avoid repeated downloads.
 
 ## PureScript Conventions
 

CONTRIBUTING.md

@@ -47,11 +47,10 @@ There is one additional directory for the web dashboard:
 
 - `dashboard` contains the static HTML/CSS/JS dashboard for monitoring registry jobs. It is deployed independently to GitHub Pages and requires no build step.
 
-There are three more directories containing code for the registry.
+There are two more directories containing code for the registry.
 
 - `db` contains schemas and migrations for the sqlite3 database used by the server.
 - `nix` contains Nix code for building and deploying the registry server to Digital Ocean.
-- `types` contains Dhall specifications for the core registry types
 
 Finally, the `flake.nix` file orchestrates builds for the whole repository.
 
@@ -182,7 +181,7 @@ The usual PureScript testing workflow applies in the registry — from within a
 spago test
 ```
 
-There are also a number of checks run by the Nix flake for non-PureScript code, such as verifying Dhall types. Run them:
+The Nix flake also checks formatting, builds the dashboard, and runs integration and deployment tests. Run them:
 
 ```sh
 nix flake check -L
@@ -212,7 +211,7 @@ The registry source code is defined such that most effectful code is abstracted
 
 We obviously don't want to perform these effects in our tests, but we still want to test that the `publish` function behaves as expected. If you are writing a test for a function written in `Run (SOME_EFFECT + r) a`, like `publish`, then you will need to mock effects when writing your tests. You can see the mock implementations in the [`Registry.Test.Assert.Run`](./app/test/Test/Assert/Run.purs) module.
 
-The mock tests use fixtures to represent remote resources. For example, instead of a remote S3 bucket we have the [`registry-storage`](./app/fixtures/registry-storage/) fixtures; this directory is our 'storage backend' and we can 'download' tarballs from it and 'upload' tarballs to it. Instead of accessing arbitrary GitHub repositories we have the [`github-packages`](./app/fixtures/github-packages/) fixtures. Instead of the upstream registry, registry-index, and package-sets repositories, we have e.g. the [`registry-index`](./app/fixtures/registry-index) fixtures.
+The mock tests use fixtures to represent remote resources. For example, instead of a remote S3 bucket we have the [`registry-storage`](./app/fixtures/registry-storage/) fixtures; this directory is our 'storage backend' and we can 'download' tarballs from it and 'upload' tarballs to it. Instead of accessing arbitrary GitHub repositories we have the [`github-packages`](./app/fixtures/github-packages/) fixtures. Instead of the upstream registry and registry-index repositories, we have the [`registry`](./app/fixtures/registry) and [`registry-index`](./app/fixtures/registry-index) fixtures.
 
 The function under test will only have access to data in these fixtures.
 
@@ -256,18 +255,18 @@ services.wiremock-github-api = {
     {
       request = {
         method = "GET";
-        url = "/repos/purescript/package-sets/tags";
+        url = "/repos/purescript/purescript-prelude/tags";
       };
       response = {
         status = 200;
         headers."Content-Type" = "application/json";
-        jsonBody = {
-          name = "psc-0.15.10-20230105";
+        jsonBody = [{
+          name = "v6.0.1";
           commit = {
             sha = "090897c992b2b310b1456506308db789672adac1";
-            url = "https://api.github.com/repos/purescript/package-sets/commits/090897c992b2b310b1456506308db789672adac1";
+            url = "https://api.github.com/repos/purescript/purescript-prelude/commits/090897c992b2b310b1456506308db789672adac1";
           };
-        };
+        }];
       };
     }
   ];

SPEC.md

@@ -225,12 +225,11 @@ Changes that would break forward compatibility (such as removing required fields
 
 ### 3.2 Atomic Data
 
-This section contains schemas for `PackageName`, `Version`, `Range`, `Location`, `Sha256`, `Owner`, `License`, and other simple data used in the registry. Each is a subsection which includes its rules, the representations in Dhall and JSON, and a link to the PureScript implementation in `Schema.purs`.
+This section contains schemas for `PackageName`, `Version`, `Range`, `Location`, `Sha256`, `Owner`, `License`, and other simple data used in the registry. Each subsection includes its rules, JSON representation, and a link to its PureScript implementation.
 
 #### PackageName
 
 **[Source](./lib/src/Registry/PackageName.purs)**
-**[Spec](./types/v1/Manifest.dhall)**
 
 Packages are uniquely identified by their `PackageName`. No two packages in the registry can share the same name. A package name represented as a `string`, with the following restrictions:
 
@@ -247,7 +246,6 @@ We still encourage users to name their repositories with a `purescript-` prefix
 #### Version
 
 **[Source](./lib/src/Registry/Version.purs)**
-**[Spec](./types/v1/Manifest.dhall)**
 
 Packages are associated with one or more versions, representing their source code at a point in time. The registry uses a restricted version of the SemVer spec which only allows versions with major, minor, and patch places (that means no build metadata and no prerelease identifiers). A version is represented as a `string` with the following restrictions:
 
@@ -271,7 +269,6 @@ The `X.Y.Z` form maps on to these three kinds of change with the following rules
 #### Range
 
 **[Source](./lib/src/Registry/Range.purs)**
-**[Spec](./types/v1/Manifest.dhall)**
 
 It is sometimes necessary to refer to a range of versions, such as when specifying dependency versions or supported compiler versions. The registry uses a restricted form of SemVer ranges; a version range is represented as a `string` with the following restrictions:
 
@@ -290,7 +287,6 @@ For example: `">=1.0.1 <2.0.0"`
 #### Location
 
 **[Source](./lib/src/Registry/Location.purs)**
-**[Spec](./types/v1/Location.dhall)**
 
 The registry operates on a pull-based model, in which package authors ask the registry to fetch their code and produce a package instead of uploading their package themselves. Accordingly, every package is associated with the location the registry will use to fetch it. A location is represented in JSON as an `object`.
 
@@ -338,7 +334,6 @@ Or, using a monorepo:
 #### Owner
 
 **[Source](./lib/src/Registry/Owner.purs)**
-**[Spec](./types/v1/Owner.dhall)**
 
 The registry relies on SSH key pairs to verify package ownership for the purposes of sensitive API operations like unpublishing versions or transferring packages. An `Owner` is made up of the three components of an SSH public key in text format ([RFC4253](https://www.rfc-editor.org/rfc/rfc4253#section-6.6)). That format looks like this:
 
@@ -366,7 +361,6 @@ Alternately, this can be written without an id:
 #### License
 
 **[Source](./lib/src/Registry/License.purs)**
-**[Spec](./types/v1/License.dhall)**
 
 All packages in the registry must have a license that grants permission for redistribution of the source code. Concretely, the registry requires that all packages use an SPDX license and specify an [SPDX license identifier](https://spdx.dev/ids/). `AND` and `OR` conjunctions are allowed, and licenses can contain exceptions using the `WITH` preposition. The SPDX specification describes [how licenses can be combined and exceptions applied](https://spdx.dev/ids#how). Newly submitted manifests must use current canonical SPDX identifiers, but registry readers should remain backward-compatible with historical stored manifests that still use deprecated SPDX spellings.
 
@@ -377,7 +371,6 @@ A `License` is represented as a string, which must be a valid SPDX identifier. F
 #### Sha256
 
 **[Source](./lib/src/Registry/Sha256.purs)**
-**[Spec](./types/v1/Sha256.dhall)**
 
 The registry produces an archive file when publishing a package. We use [tarballs](https://wiki.debian.org/TarBall) as our archive format. The registry produces a tarball when publishing a package. The hash of this tarball is recorded so that package managers can verify the integrity of packages they download from the registry. The hash is stored in the [subresource integrity (SRI) format](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity). A Sha256 is represented as a `string` in JSON. For example:
 
@@ -386,7 +379,6 @@ The registry produces an archive file when publishing a package. We use [tarball
 ### 3.3 Manifest
 
 **[Source](./lib/src/Registry/Manifest.purs)**
-**[Spec](./types/v1/Manifest.dhall)**
 
 All packages in the registry contain a `purs.json` manifest file in their root directory. The manifest file specifies information necessary for the registry to package the source code, and it also serves as a lingua franca for package managers: all package managers are expected to support this manifest format. The manifest file is an object with the following fields:
 
@@ -429,7 +421,6 @@ For example:
 ### 3.4 Metadata
 
 **[Source](./lib/src/Registry/Metadata.purs)**
-**[Spec](./types/v1/Metadata.dhall)**
 
 All packages in the registry have an associated metadata file, which is located in the `metadata` directory of the `registry` repository under the package name. For example, the metadata for the `aff` package is located at: https://github.com/purescript/registry/blob/main/metadata/aff.json. Metadata files are the source of truth on all published and unpublished versions for a particular package for what there content is and where the package is located. Metadata files are produced by the registry, not by package authors, though they take some information from package manifests.
 
@@ -623,7 +614,9 @@ First, the registry uses the [`PackageName`](#packagename) listed in the JSON pa
 1. The package name is not registered (ie. no metadata exists) AND a location is provided in the payload. In this case, the registry will create new metadata to register this package at the given location. It is possible to register a package at the same location as one that already exists.
 2. The package name is registered (ie. metadata exists) AND either no location is provided in the payload OR the provided location matches the location recorded in the metadata. When no location is provided, the registry will use the location from the metadata.
 
-Next, the registry will use the provided [`Location`](#location) to fetch the package source at the provided `ref`. Packages must contain a `purs.json` manifest or `spago.yaml` manifest in the package root (in the presence of multiple manifest files, the `purs.json` file takes precedence and others are ignored). The registry will verify that the manifest matches with the JSON payload and metadata:
+Next, the registry will use the provided [`Location`](#location) to fetch the package source at the provided `ref`. Packages must contain a `purs.json` manifest or `spago.yaml` manifest in the package root (in the presence of multiple manifest files, the `purs.json` file takes precedence and others are ignored).
+
+The registry will verify that the manifest matches with the JSON payload and metadata:
 
 1. The manifest MUST be usable to produce a well-formed [`Manifest`](#33-manifest).
 2. The manifest package name and JSON payload package name MUST match.
@@ -632,7 +625,7 @@ Next, the registry will use the provided [`Location`](#location) to fetch the pa
 5. The manifest location MUST match the authoritative registry metadata location (and therefore the JSON payload location, when provided).
 6. The manifest version MUST NOT have been unpublished before, according to the metadata file. A version already present in metadata, the manifest index, storage, and Pursuit is treated as an idempotent success; incomplete prior publications are reconciled.
 7. Either a `resolutions` key was provided in the JSON payload, in which case the resolutions MUST include every dependency listed in the manifest dependencies at a version within the specified version bounds for each dependency, OR the manifest dependencies MUST be solvable by the registry solver. ALL package versions indicated in the resolutions MUST already be registered.
-8. If the package source code includes a LICENSE file and/or a license listed in a bower.json or package.json manifest, then all licenses MUST match with the license specified in the PureScript [`Manifest`](#33-manifest). For example, if the package.json file specifies the "MIT" license and a LICENSE file specifies "BSD-3-Clause", then the PureScript license should admit at least "MIT AND BSD-3-Clause".
+8. If the package source code includes a LICENSE file and/or a license listed in a package.json manifest, then all licenses MUST match with the license specified in the PureScript [`Manifest`](#33-manifest). For example, if the package.json file specifies the "MIT" license and a LICENSE file specifies "BSD-3-Clause", then the PureScript license should admit at least "MIT AND BSD-3-Clause".
 
 Next, the registry will verify the package source code.
 
@@ -648,11 +641,7 @@ Finally, the registry will perform some processing on the source code, package t
 
 ###### Always-Included Files
 
-These files are always included in the tarball, if present:
-
-- The full contents of the `src` directory.
-- The `purs.json`, `spago.yaml`, `spago.dhall` and `packages.dhall`, `bower.json`, and `package.json` manifest formats (in the root of the package).
-- Any README or LICENSE files (in the root of the package).
+The registry always includes the full contents of `src`, package manifests, and root README and LICENSE files when present. The complete filename patterns are defined by `includedGlobs` and `includedInsensitiveGlobs` in [`Registry.Constants`](./lib/src/Constants.purs).
 
 ###### Always-Ignored Files
 

app-e2e/src/Test/E2E/Endpoint/PackageSets.purs

@@ -3,8 +3,11 @@ module Test.E2E.Endpoint.PackageSets (spec) where
 import Registry.App.Prelude
 
 import Control.Monad.Reader (ask)
+import Data.Map as Map
 import Effect.Aff as Aff
+import Node.Path as Path
 import Registry.API.V1 as V1
+import Registry.PackageSet as PackageSet
 import Registry.Test.Assert as Assert
 import Test.E2E.Support.Client as Client
 import Test.E2E.Support.Env (E2ESpec)
@@ -15,7 +18,7 @@ import Test.Spec as Spec
 spec :: E2ESpec
 spec = do
   Spec.describe "Package Sets endpoint" do
-    Spec.it "accepts unauthenticated add/upgrade requests" do
+    Spec.it "releases a registry package set" do
       -- First publish unsafe-coerce to create the tarball in storage
       { jobId: publishJobId } <- Client.publish Fixtures.unsafeCoercePublishData
       _ <- Env.pollJobOrFail publishJobId
@@ -24,6 +27,14 @@ spec = do
       job <- Env.pollJobOrFail jobId
       Assert.shouldSatisfy (V1.jobInfo job).finishedAt isJust
 
+      { stateDir } <- ask
+      let setPath = Path.concat [ stateDir, "scratch", "registry", "package-sets", "0.1.0.json" ]
+      liftAff (readJsonFile PackageSet.codec setPath) >>= case _ of
+        Left err -> Assert.fail $ "Expected a released registry package set: " <> err
+        Right (PackageSet set) -> do
+          Map.lookup Fixtures.unsafeCoerce.name set.packages `Assert.shouldEqual` Just Fixtures.unsafeCoerce.version
+          Map.lookup Fixtures.prelude.name set.packages `Assert.shouldEqual` Just Fixtures.prelude.version
+
     Spec.it "rejects unauthenticated compiler change requests" do
       result <- Client.tryPackageSets Fixtures.packageSetCompilerChangeRequest
       case result of

app-e2e/src/Test/E2E/Endpoint/Publish.purs

@@ -26,7 +26,7 @@ import Test.Spec as Spec
 spec :: E2ESpec
 spec = do
   Spec.describe "Publish workflow" do
-    Spec.it "can publish effect@4.0.0 and verify all state changes" do
+    Spec.it "can publish effect@4.0.1 from purs.json and verify all state changes" do
       { jobId, disposition: submissionDisposition } <- Client.publish Fixtures.effectPublishData
       Assert.shouldEqual submissionDisposition (Just V1.Created)
       job <- Env.pollJobOrFail jobId
@@ -40,7 +40,7 @@ spec = do
       uploadOccurred <- Env.hasStorageUpload Fixtures.effect
       unless uploadOccurred do
         storageRequests <- WireMock.getStorageRequests
-        WireMock.failWithRequests "Expected S3 PUT for effect/4.0.0.tar.gz" storageRequests
+        WireMock.failWithRequests "Expected S3 PUT for effect/4.0.1.tar.gz" storageRequests
 
       Metadata metadata <- Env.readMetadata Fixtures.effect.name
       case Map.lookup Fixtures.effect.version metadata.published of
@@ -76,11 +76,16 @@ spec = do
           let actualCompilers = Set.fromFoldable $ NEA.toArray publishedMetaAfter.compilers
           Assert.shouldEqual actualCompilers expectedCompilers
 
-  Spec.describe "Publish with spago.dhall manifest" do
-    Spec.it "can publish slug@3.0.0 which uses spago.dhall format" do
+  Spec.describe "Publish with spago.yaml manifest" do
+    Spec.it "can publish slug@3.0.0 which uses spago.yaml format" do
       { jobId } <- Client.publish Fixtures.slugPublishData
       job <- Env.pollJobOrFail jobId
       Assert.shouldSatisfy (V1.jobInfo job).finishedAt isJust
+      case job of
+        PublishJob details -> do
+          Assert.shouldEqual details.disposition (Just V1.Published)
+          Assert.shouldEqual details.error Nothing
+        _ -> Assert.fail "Expected a publish job."
 
       uploadOccurred <- Env.hasStorageUpload Fixtures.slug
       unless uploadOccurred do

app-e2e/src/Test/E2E/Support/Fixtures.purs

@@ -56,12 +56,11 @@ console = { name: Utils.unsafePackageName "console", version: Utils.unsafeVersio
 prelude :: PackageFixture
 prelude = { name: Utils.unsafePackageName "prelude", version: Utils.unsafeVersion "6.0.1" }
 
--- | slug@3.0.0 fixture package (uses spago.dhall manifest format)
 slug :: PackageFixture
 slug = { name: Utils.unsafePackageName "slug", version: Utils.unsafeVersion "3.0.0" }
 
 -- | Standard publish data for effect@4.0.1, used by E2E tests.
--- | This matches the fixtures in app/fixtures/github-packages/effect-4.0.1
+-- | The test setup copies effect-4.0.0 and updates its manifest for this release.
 effectPublishData :: Operation.PublishData
 effectPublishData =
   { name: effect.name
@@ -116,9 +115,6 @@ consolePublishData =
   , version: console.version
   }
 
--- | Publish data for slug@3.0.0, used for spago.dhall manifest format test.
--- | Slug depends on prelude only and uses spago.dhall as its manifest format.
--- | This matches the fixtures in app/fixtures/github-packages/slug-3.0.0
 slugPublishData :: Operation.PublishData
 slugPublishData =
   { name: slug.name