Harden distributed file-processing architecture

.env.example

@@ -0,0 +1,6 @@
+POSTGRES_PASSWORD=replace-with-a-strong-local-password
+RABBITMQ_PASSWORD=replace-with-a-strong-local-password
+MINIO_ACCESS_KEY=replace-with-a-local-access-key
+MINIO_SECRET_KEY=replace-with-a-strong-local-secret
+JWT_SIGNING_KEY=replace-with-at-least-32-random-characters
+FILE_PROCESSING_API_KEY=replace-with-a-random-local-api-key

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: nuget
+    directory: /
+    schedule: { interval: weekly }
+    groups: { dotnet: { patterns: ["*"] } }
+  - package-ecosystem: github-actions
+    directory: /
+    schedule: { interval: weekly }
+  - package-ecosystem: docker
+    directory: /
+    schedule: { interval: weekly }

.github/workflows/ci.yml

@@ -0,0 +1,18 @@
+name: ci
+on:
+  push: { branches: [main] }
+  pull_request:
+permissions: { contents: read }
+jobs:
+  dotnet:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-dotnet@v6
+        with: { dotnet-version: 10.0.x }
+      - run: dotnet restore DistributedFileProcessing.slnx
+      - run: dotnet format DistributedFileProcessing.slnx --verify-no-changes --no-restore
+      - run: dotnet build DistributedFileProcessing.slnx -c Release --no-restore
+      - run: dotnet test DistributedFileProcessing.slnx -c Release --no-build --no-restore
+      - run: dotnet list DistributedFileProcessing.slnx package --vulnerable --include-transitive
+      - run: docker compose --env-file .env.example config --quiet

.gitignore

@@ -0,0 +1,8 @@
+**/bin/
+**/obj/
+.env
+.vs/
+.vscode/
+TestResults/
+*.user
+*.suo

CONTRIBUTING.md

@@ -0,0 +1,3 @@
+# Contributing
+
+Keep changes focused and describe their operational failure modes. Run the Release build, tests, formatting check, dependency audit, and Compose configuration check before opening a pull request. Never commit `.env`, credentials, customer files, malware samples, OCR output, or exported object-store data.

README.md

@@ -1,6 +1,6 @@
 # Distributed File Processing Platform
 
-A local-first .NET 10 platform that streams uploads to MinIO, records state in PostgreSQL, publishes durable RabbitMQ jobs, and processes files in horizontally scalable workers with ClamAV, Tesseract OCR, thumbnails, notifications, auditing, retries, Hangfire, Redis, structured logs, metrics, traces, and health checks.
+A local-first .NET 10 reference platform that spools uploads to temporary storage, records state in PostgreSQL, publishes RabbitMQ jobs, and processes files in horizontally scalable workers with ClamAV, Tesseract OCR, thumbnails, notifications, auditing, retries, Hangfire, Redis, structured logs, metrics, traces, and health checks.
 
 ## Architecture and workflow
 
@@ -40,11 +40,13 @@ docker/                            reserved operational assets
 - Optional host development: .NET SDK 10.0.302 or newer
 - About 6 GB free RAM while ClamAV initializes its signatures
 
-The checked-in credentials are local-development values. Change `Authentication`, PostgreSQL, RabbitMQ, and MinIO secrets before sharing a reachable environment.
+Credentials are not committed. Copy `.env.example` to `.env` and replace every placeholder before starting Compose. Never reuse local values in a shared environment.
 
 ## Run everything
 
 ```powershell
+Copy-Item .env.example .env
+# Replace every placeholder in .env.
 docker compose up --build --scale worker=2
 ```
 
@@ -53,8 +55,8 @@ First startup can take several minutes because ClamAV downloads signatures. Wait
 - Swagger: http://localhost:8080/swagger
 - Health: http://localhost:8080/health
 - Hangfire: http://localhost:8080/hangfire (loopback only)
-- RabbitMQ: http://localhost:15672 (`fileprocessing` / `local_dev_password`)
-- MinIO: http://localhost:9001 (`minioadmin` / `minioadmin`)
+- RabbitMQ: http://localhost:15672 (user `fileprocessing`; password from `.env`)
+- MinIO: http://localhost:9001 (credentials from `.env`)
 
 Stop without deleting data with `docker compose down`. To intentionally remove local data, run `docker compose down --volumes`.
 
@@ -82,11 +84,11 @@ DELETE requires an `admin` JWT. Downloads are allowed only after completion and
 
 ## Validation and processing
 
-Allowed types are PDF, DOCX, XLSX, TXT, PNG, JPEG, and TIFF. The API enforces configured size, extension, MIME type, SHA-256 duplicate detection, and signatures for PDF, Office ZIP, PNG, and JPEG before queueing. Upload content is streamed into a bounded buffer for hashing/signature validation and then into MinIO. Set `FileProcessing:MaxFileSizeBytes` to change the 100 MiB default.
+Allowed types are PDF, DOCX, XLSX, TXT, PNG, JPEG, and TIFF. The API enforces configured size, extension, MIME type, SHA-256 duplicate detection, and signatures for PDF, Office ZIP, PNG, and JPEG before queueing. Upload content is spooled to a temporary file for hashing/signature validation and then streamed into MinIO. This bounds managed-memory use but requires temporary disk capacity up to the configured upload limit. Set `FileProcessing:MaxFileSizeBytes` to change the 100 MiB default.
 
 Workers use ClamAV's INSTREAM protocol. Infected files stop at `VirusDetected`. Text files are read directly; other clean files invoke Tesseract. Image thumbnails preserve aspect ratio within `Thumbnails:Width` and `Height`. OCR text and thumbnails are written to MinIO. Configure `Notifications:WebhookUrl` to receive JSON events.
 
-Transient consumer failures are negatively acknowledged to the dead-letter exchange. Operators can call the retry endpoint; Hangfire also republishes up to 100 failed jobs hourly. Attempts are capped by `FileProcessing:MaxRetries`. Completed jobs are idempotent. Increase throughput with `docker compose up --scale worker=4`; RabbitMQ prefetch controls per-consumer concurrency.
+Consumer failures are negatively acknowledged to the dead-letter exchange. Operators can call the retry endpoint; Hangfire also republishes up to 100 failed jobs hourly. Attempts are capped by `FileProcessing:MaxRetries`. Completed jobs are idempotent. Increase throughput with `docker compose up --scale worker=4`; RabbitMQ prefetch controls per-consumer concurrency.
 
 ## Configuration
 
@@ -111,6 +113,8 @@ docker compose config --quiet
 
 Unit tests verify state/history invariants. The integration project is wired to the API and currently checks serialized queue contracts. A live end-to-end check requires the Compose dependencies: upload the EICAR test file and confirm `VirusDetected`, then upload a clean image and observe `Completed`, OCR/thumbnail objects in MinIO, history rows in PostgreSQL, and acknowledged messages in RabbitMQ.
 
+See [architecture](docs/architecture.md), [verification scope](docs/verification.md), and the [transactional-outbox decision](docs/adr/0002-transactional-outbox.md). The current database commit and RabbitMQ publish are not atomic; that gap must be closed before calling the ingestion path production-grade.
+
 ## Operations and troubleshooting
 
 Every transition creates processing history; uploads, retries, failures, completion, and deletion create correlation-indexed audit records. Send `X-Correlation-ID` as a GUID or use the generated value returned in the response header. Serilog emits structured request/worker logs. OpenTelemetry instruments HTTP, runtime, and outbound requests; point `OTEL_EXPORTER_OTLP_ENDPOINT` to a local collector to export traces and metrics.

SECURITY.md

@@ -0,0 +1,5 @@
+# Security policy
+
+Report vulnerabilities through GitHub private vulnerability reporting rather than a public issue. Include impact, reproduction steps, and affected versions without real credentials or sensitive files.
+
+This repository is a reference implementation. The Compose stack is not an internet-ready deployment. Use a managed secret store, TLS, private networking, least-privilege service accounts, malware-signature monitoring, and an authenticated authorization service before production use.

docker-compose.yml

@@ -2,13 +2,13 @@ name: distributed-file-processing
 services:
   postgres:
     image: postgres:18-alpine
-    environment: { POSTGRES_DB: fileprocessing, POSTGRES_USER: fileprocessing, POSTGRES_PASSWORD: local_dev_password }
+    environment: { POSTGRES_DB: fileprocessing, POSTGRES_USER: fileprocessing, POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}" }
     ports: ["5432:5432"]
     volumes: ["postgres-data:/var/lib/postgresql/data"]
     healthcheck: { test: ["CMD-SHELL", "pg_isready -U fileprocessing -d fileprocessing"], interval: 5s, timeout: 5s, retries: 20 }
   rabbitmq:
     image: rabbitmq:4-management-alpine
-    environment: { RABBITMQ_DEFAULT_USER: fileprocessing, RABBITMQ_DEFAULT_PASS: local_dev_password }
+    environment: { RABBITMQ_DEFAULT_USER: fileprocessing, RABBITMQ_DEFAULT_PASS: "${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env}" }
     ports: ["5672:5672", "15672:15672"]
     volumes: ["rabbitmq-data:/var/lib/rabbitmq"]
     healthcheck: { test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"], interval: 5s, timeout: 5s, retries: 20 }
@@ -21,7 +21,7 @@ services:
   minio:
     image: minio/minio:latest
     command: server /data --console-address ":9001"
-    environment: { MINIO_ROOT_USER: minioadmin, MINIO_ROOT_PASSWORD: minioadmin }
+    environment: { MINIO_ROOT_USER: "${MINIO_ACCESS_KEY:?Set MINIO_ACCESS_KEY in .env}", MINIO_ROOT_PASSWORD: "${MINIO_SECRET_KEY:?Set MINIO_SECRET_KEY in .env}" }
     ports: ["9000:9000", "9001:9001"]
     volumes: ["minio-data:/data"]
     healthcheck: { test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"], interval: 5s, timeout: 5s, retries: 20 }
@@ -32,7 +32,18 @@ services:
     healthcheck: { test: ["CMD-SHELL", "echo PING | nc localhost 3310 | grep PONG"], interval: 15s, timeout: 5s, retries: 40, start_period: 60s }
   api:
     build: { context: ., dockerfile: src/FileProcessing.Api/Dockerfile }
-    environment: { ASPNETCORE_URLS: "http://+:8080", ASPNETCORE_ENVIRONMENT: Development, OTEL_EXPORTER_OTLP_ENDPOINT: "" }
+    environment:
+      ASPNETCORE_URLS: "http://+:8080"
+      ASPNETCORE_ENVIRONMENT: Development
+      OTEL_EXPORTER_OTLP_ENDPOINT: ""
+      ConnectionStrings__PostgreSql: "Host=postgres;Port=5432;Database=fileprocessing;Username=fileprocessing;Password=${POSTGRES_PASSWORD}"
+      ConnectionStrings__Redis: "redis:6379"
+      Authentication__JwtKey: "${JWT_SIGNING_KEY:?Set JWT_SIGNING_KEY in .env}"
+      Authentication__ApiKey: "${FILE_PROCESSING_API_KEY:?Set FILE_PROCESSING_API_KEY in .env}"
+      RabbitMq__UserName: fileprocessing
+      RabbitMq__Password: "${RABBITMQ_PASSWORD}"
+      Minio__AccessKey: "${MINIO_ACCESS_KEY}"
+      Minio__SecretKey: "${MINIO_SECRET_KEY}"
     ports: ["8080:8080"]
     depends_on:
       postgres: { condition: service_healthy }
@@ -43,6 +54,13 @@ services:
   worker:
     build: { context: ., dockerfile: src/FileProcessing.Worker/Dockerfile }
     deploy: { replicas: 2 }
+    environment:
+      ConnectionStrings__PostgreSql: "Host=postgres;Port=5432;Database=fileprocessing;Username=fileprocessing;Password=${POSTGRES_PASSWORD}"
+      ConnectionStrings__Redis: "redis:6379"
+      RabbitMq__UserName: fileprocessing
+      RabbitMq__Password: "${RABBITMQ_PASSWORD}"
+      Minio__AccessKey: "${MINIO_ACCESS_KEY}"
+      Minio__SecretKey: "${MINIO_SECRET_KEY}"
     depends_on:
       postgres: { condition: service_healthy }
       rabbitmq: { condition: service_healthy }

docs/adr/0001-temporary-file-spooling.md

@@ -0,0 +1,5 @@
+# ADR 0001: Spool uploads to temporary storage
+
+Status: Accepted
+
+The API copies each accepted upload to an asynchronously opened temporary file, computes its checksum and signature from that seekable stream, then sends it to object storage. This avoids allocating up to the full upload limit in managed memory. The trade-off is temporary-disk capacity and I/O; deployments must size and monitor ephemeral storage.

docs/adr/0002-transactional-outbox.md

@@ -0,0 +1,5 @@
+# ADR 0002: Introduce a transactional outbox before production
+
+Status: Proposed
+
+The current upload flow commits file metadata and then publishes RabbitMQ work. A failure between those operations can leave a queued record without a message. Before production, store an outbox record in the same PostgreSQL transaction, publish it from a background dispatcher with confirms, and mark it delivered idempotently. Reconciliation should detect stale queued records.

docs/architecture.md

@@ -0,0 +1,25 @@
+# Architecture
+
+| Quality attribute | Current mechanism | Evidence or limitation |
+| --- | --- | --- |
+| Scalability | Competing RabbitMQ consumers and object storage | Worker replicas scale independently; load tests are not included. |
+| Reliability | Persistent messages, manual acknowledgements, DLQ, idempotent completed state | Database commit and message publish are not atomic yet; see ADR 0002. |
+| Security | JWT/API-key boundary, signature checks, ClamAV, signed downloads | Production identity and network controls remain deployment concerns. |
+| Observability | Correlation IDs, audit rows, Serilog, OpenTelemetry | No dashboard, alert rules, or SLO validation is included. |
+| Resource efficiency | Uploads spool to a temporary file before hashing and storage | Temporary disk must be sized for concurrent upload limits. |
+
+```mermaid
+flowchart LR
+  Client -->|JWT or API key| API
+  API --> MinIO[(MinIO)]
+  API --> PostgreSQL[(PostgreSQL)]
+  API --> RabbitMQ[(RabbitMQ)]
+  RabbitMQ --> Worker
+  Worker --> ClamAV
+  Worker --> Tesseract
+  Worker --> MinIO
+  Worker --> PostgreSQL
+  Worker -->|optional outbound| Webhook
+```
+
+Uploaded content, filenames, MIME types, webhook destinations, queue payloads, and OCR output are untrusted. Production deployments should isolate workers, restrict egress, enforce object retention, and cap temporary storage.

docs/verification.md

@@ -0,0 +1,5 @@
+# Verification
+
+CI restores, checks formatting, builds in Release, runs deterministic tests, audits NuGet dependencies, and validates the Compose model. Current tests cover domain transitions and contract serialization; they do not prove a live PostgreSQL/RabbitMQ/MinIO/ClamAV/Tesseract workflow.
+
+For an end-to-end check, copy `.env.example` to `.env`, replace every value, start the stack, upload one clean image and the standard EICAR test file, and verify API state, PostgreSQL history, MinIO artifacts, queue acknowledgement, and worker logs. Record the date and environment when publishing runtime results.

src/FileProcessing.Api/Infrastructure.cs

@@ -6,7 +6,8 @@
 using System.Security.Cryptography;
 
 namespace FileProcessing.Api;
-public sealed class CorrelationIdMiddleware(RequestDelegate next){public async Task InvokeAsync(HttpContext c){var id=c.Request.Headers.TryGetValue("X-Correlation-ID",out var supplied)&&Guid.TryParse(supplied,out _)?supplied.ToString():Guid.NewGuid().ToString();c.TraceIdentifier=id;c.Response.Headers["X-Correlation-ID"]=id;await next(c);}}
-public sealed class ApiKeyMiddleware(RequestDelegate next,IConfiguration configuration){public async Task InvokeAsync(HttpContext c){if(c.User.Identity?.IsAuthenticated!=true&&c.Request.Headers.TryGetValue("X-Api-Key",out var supplied)&&CryptographicOperations.FixedTimeEquals(System.Text.Encoding.UTF8.GetBytes(supplied.ToString()),System.Text.Encoding.UTF8.GetBytes(configuration["Authentication:ApiKey"]??"disabled"))){var claims=new[]{new System.Security.Claims.Claim(System.Security.Claims.ClaimTypes.Name,"api-key"),new System.Security.Claims.Claim(System.Security.Claims.ClaimTypes.Role,"processor")};c.User=new(new System.Security.Claims.ClaimsIdentity(claims,"ApiKey"));}await next(c);}}
-public sealed class LocalDashboardAuthorization:IDashboardAuthorizationFilter { public bool Authorize(DashboardContext context){var http=context.GetHttpContext();return http.Connection.RemoteIpAddress is { } ip&&System.Net.IPAddress.IsLoopback(ip);} }
-public sealed class MaintenanceJobs(ProcessingDbContext db,IJobPublisher publisher){public async Task RetryFailedAsync(CancellationToken ct){var files=await db.Files.Where(x=>x.Status==ProcessingStatus.Failed&&x.RetryCount<5).Take(100).ToListAsync(ct);foreach(var file in files){file.Retry();await publisher.PublishAsync(new(file.Id,Guid.NewGuid().ToString(),file.RetryCount),9,ct);}await db.SaveChangesAsync(ct);}public async Task CleanupAsync(CancellationToken ct){var cutoff=DateTimeOffset.UtcNow.AddDays(-30);var logs=await db.AuditLogs.Where(x=>x.OccurredAt<cutoff).ToListAsync(ct);db.AuditLogs.RemoveRange(logs);await db.SaveChangesAsync(ct);}}
+
+public sealed class CorrelationIdMiddleware(RequestDelegate next) { public async Task InvokeAsync(HttpContext c) { var id = c.Request.Headers.TryGetValue("X-Correlation-ID", out var supplied) && Guid.TryParse(supplied, out _) ? supplied.ToString() : Guid.NewGuid().ToString(); c.TraceIdentifier = id; c.Response.Headers["X-Correlation-ID"] = id; await next(c); } }
+public sealed class ApiKeyMiddleware(RequestDelegate next, IConfiguration configuration) { public async Task InvokeAsync(HttpContext c) { if (c.User.Identity?.IsAuthenticated != true && c.Request.Headers.TryGetValue("X-Api-Key", out var supplied) && CryptographicOperations.FixedTimeEquals(System.Text.Encoding.UTF8.GetBytes(supplied.ToString()), System.Text.Encoding.UTF8.GetBytes(configuration["Authentication:ApiKey"] ?? "disabled"))) { var claims = new[] { new System.Security.Claims.Claim(System.Security.Claims.ClaimTypes.Name, "api-key"), new System.Security.Claims.Claim(System.Security.Claims.ClaimTypes.Role, "processor") }; c.User = new(new System.Security.Claims.ClaimsIdentity(claims, "ApiKey")); } await next(c); } }
+public sealed class LocalDashboardAuthorization : IDashboardAuthorizationFilter { public bool Authorize(DashboardContext context) { var http = context.GetHttpContext(); return http.Connection.RemoteIpAddress is { } ip && System.Net.IPAddress.IsLoopback(ip); } }
+public sealed class MaintenanceJobs(ProcessingDbContext db, IJobPublisher publisher) { public async Task RetryFailedAsync(CancellationToken ct) { var files = await db.Files.Where(x => x.Status == ProcessingStatus.Failed && x.RetryCount < 5).Take(100).ToListAsync(ct); foreach (var file in files) { file.Retry(); await publisher.PublishAsync(new(file.Id, Guid.NewGuid().ToString(), file.RetryCount), 9, ct); } await db.SaveChangesAsync(ct); } public async Task CleanupAsync(CancellationToken ct) { var cutoff = DateTimeOffset.UtcNow.AddDays(-30); var logs = await db.AuditLogs.Where(x => x.OccurredAt < cutoff).ToListAsync(ct); db.AuditLogs.RemoveRange(logs); await db.SaveChangesAsync(ct); } }