diff --git a/SMG.md b/SMG.md index a46c055783e..60ac4c8cef4 100644 --- a/SMG.md +++ b/SMG.md @@ -41,6 +41,7 @@ Cap Web's Railway settings live in [`.railway/railway.ts`](.railway/railway.ts) - **Config changes** go through a PR that touches `.railway/`. `SMG Railway plan` comments the plan on the PR; merging applies that exact plan (`SMG Railway apply`). Preview locally with `railway config plan` from a linked checkout. - **Code changes** deploy when they reach `smg`. Railway builds `smg/web.Dockerfile` from `smg` and waits for the `SMG CI` run on that push ("Wait for CI"): if it fails, that commit is not deployed. - **`smg/web.Dockerfile`** is `apps/web/Dockerfile` without the `# syntax=` line (Railway rejects external Dockerfile frontends) and with the cache mount ids Railway requires (`s/-`); we don't patch upstream's file. When a sync changes upstream's Dockerfile, the `scripts` check fails until you run `bun scripts/smg/railway-dockerfile.ts --write` on the sync branch and commit the result. +- **View locations** come from DB-IP City Lite (free, CC BY 4.0, attribution: "IP Geolocation by [DB-IP](https://db-ip.com)"). The generator adds a `geoip` stage to `smg/web.Dockerfile` that downloads the current month's file on every build and sets `CAP_GEOIP_DATABASE_PATH` (read by `feat/self-hosted-geoip`). A failed download doesn't fail the build; views just have no country or city until the next deploy, and the build log says so. - **Migrations** run when Cap Web starts, and a rollback doesn't undo them. That is why a PR that adds one needs the `migrations-approved` label, given after a MySQL backup. - **To pause deploys**, turn off the GitHub autodeploy on Cap Web in Railway. To roll back, redeploy an earlier deployment from the Railway dashboard. @@ -53,6 +54,7 @@ What we carry on top of upstream. `git log --first-parent --oneline main..smg` i | `feat/preview-og-fallback-relative-redirect` | A | [CapSoftware/Cap#3963](https://github.com/CapSoftware/Cap/pull/3963) | In smg ([#10](https://github.com/Solomedia/cap/pull/10)), PR open upstream | `apps/web/app/api/video/preview/route.ts` | | `feat/share-unfurl-ai-summary` | A | Not opened (improvement, not a fix; propose later if wanted) | In smg ([#11](https://github.com/Solomedia/cap/pull/11)) | `apps/web/app/s/[videoId]/page.tsx`, `apps/web/lib/share-video-metadata.ts` | | `feat/google-account-linking` | A | [CapSoftware/Cap#3964](https://github.com/CapSoftware/Cap/pull/3964) | In smg ([#12](https://github.com/Solomedia/cap/pull/12)), PR open upstream | `packages/database/auth/auth-options.ts`, `apps/web/app/(org)/login/form.tsx` | +| `feat/self-hosted-geoip` | A | Not opened yet | PR open into smg ([#21](https://github.com/Solomedia/cap/pull/21)) | `apps/web/app/api/analytics/track/route.ts`, `apps/web/app/api/mobile/[...route]/route.ts`, `packages/env/server.ts`, `apps/web/package.json`, `bun.lock` | Type A: upstream-able (`feat/*`). Type B: SMG-only (`internal/*`). States: `In smg`, `PR open`, `Merged upstream`, `Rejected upstream → SMG-only`. diff --git a/scripts/smg/__tests__/railway-dockerfile.test.ts b/scripts/smg/__tests__/railway-dockerfile.test.ts index 373fa66278d..1dc68fdf222 100644 --- a/scripts/smg/__tests__/railway-dockerfile.test.ts +++ b/scripts/smg/__tests__/railway-dockerfile.test.ts @@ -2,7 +2,10 @@ import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import { CAP_WEB_SERVICE_ID, + GEOIP_DATABASE_PATH, railwayDockerfile, + smgWebDockerfile, + withGeoIpDatabase, } from "../lib/railway-dockerfile.ts"; describe("railwayDockerfile", () => { @@ -51,6 +54,34 @@ describe("railwayDockerfile", () => { test("smg/web.Dockerfile is up to date with apps/web/Dockerfile", () => { const upstream = readFileSync("apps/web/Dockerfile", "utf8"); const ours = readFileSync("smg/web.Dockerfile", "utf8"); - expect(ours).toBe(railwayDockerfile(upstream)); + expect(ours).toBe(smgWebDockerfile(upstream)); + }); +}); + +describe("withGeoIpDatabase", () => { + const dockerfile = + "FROM node AS base\nFROM base AS builder\nRUN build\nFROM base AS runner\nCOPY a b\nUSER nextjs\nCMD node server.js\n"; + + test("downloads the database in its own stage and points the app at it", () => { + const out = withGeoIpDatabase(dockerfile); + expect(out.indexOf("FROM base AS geoip")).toBeLessThan( + out.indexOf("FROM base AS runner"), + ); + expect(out).toContain("ARG RAILWAY_GIT_COMMIT_SHA"); + expect(out).toContain( + "https://download.db-ip.com/free/dbip-city-lite-$month.mmdb.gz", + ); + expect(out).toContain( + `COPY --from=geoip --chown=nextjs:nodejs /geoip /app/geoip\nENV CAP_GEOIP_DATABASE_PATH=${GEOIP_DATABASE_PATH}\n\nUSER nextjs`, + ); + }); + + test("fails when upstream's Dockerfile loses an anchor", () => { + expect(() => + withGeoIpDatabase(dockerfile.replace("USER nextjs\n", "")), + ).toThrow('"USER nextjs"'); + expect(() => + withGeoIpDatabase(dockerfile.replace("FROM base AS runner", "FROM base")), + ).toThrow('"FROM base AS runner"'); }); }); diff --git a/scripts/smg/lib/railway-dockerfile.ts b/scripts/smg/lib/railway-dockerfile.ts index d8496f24f1b..f282aa803df 100644 --- a/scripts/smg/lib/railway-dockerfile.ts +++ b/scripts/smg/lib/railway-dockerfile.ts @@ -5,7 +5,7 @@ export const CAP_WEB_SERVICE_ID = "3035d845-1a7e-43c5-8c15-c61db41ff384"; -const HEADER = `# Generated from apps/web/Dockerfile for Railway: no syntax directive, Railway cache mount ids. +const HEADER = `# Generated from apps/web/Dockerfile for Railway: no syntax directive, Railway cache mount ids, GeoIP database. # Do not edit. Regenerate with: bun scripts/smg/railway-dockerfile.ts --write `; @@ -30,3 +30,56 @@ export function railwayDockerfile( const withoutSyntax = body.replace(/^# syntax=[^\n]*\n/, ""); return `${HEADER}${withoutSyntax}`; } + +// Views get their country and city from this file (CAP_GEOIP_DATABASE_PATH), +// since Railway sends no location headers. DB-IP publishes the free City Lite +// database monthly under CC BY 4.0; the commit SHA busts the build cache so +// every deploy takes the current month. A failed download never fails the +// build: views just keep no location until the next deploy. +const GEOIP_DIR = "/app/geoip"; +export const GEOIP_DATABASE_PATH = `${GEOIP_DIR}/dbip-city-lite.mmdb`; + +const GEOIP_STAGE = `# GeoIP database for view locations (DB-IP City Lite, CC BY 4.0, https://db-ip.com) +FROM base AS geoip +ARG RAILWAY_GIT_COMMIT_SHA +RUN mkdir -p /geoip && cd /geoip && \\ + for month in "$(date -u +%Y-%m)" "$(date -u -d "@$(( $(date -u +%s) - 2678400 ))" +%Y-%m)"; do \\ + if wget -q -O db.mmdb.gz "https://download.db-ip.com/free/dbip-city-lite-$month.mmdb.gz" && gunzip -f db.mmdb.gz; then \\ + mv db.mmdb dbip-city-lite.mmdb && echo "GeoIP: DB-IP City Lite $month (commit \${RAILWAY_GIT_COMMIT_SHA:-unknown})" && break; \\ + fi; \\ + rm -f db.mmdb.gz db.mmdb; \\ + done; \\ + [ -f dbip-city-lite.mmdb ] || echo "GeoIP: download failed, views will have no location" + +`; + +const GEOIP_COPY = `COPY --from=geoip --chown=nextjs:nodejs /geoip ${GEOIP_DIR} +ENV CAP_GEOIP_DATABASE_PATH=${GEOIP_DATABASE_PATH} + +`; + +function insertBefore(text: string, anchor: string, insert: string): string { + const index = text.indexOf(anchor); + if (index === -1 || text.indexOf(anchor, index + 1) !== -1) { + throw new Error( + `expected exactly one "${anchor}" in apps/web/Dockerfile; update scripts/smg/lib/railway-dockerfile.ts`, + ); + } + return text.slice(0, index) + insert + text.slice(index); +} + +export function withGeoIpDatabase(dockerfile: string): string { + const withStage = insertBefore( + dockerfile, + "FROM base AS runner", + GEOIP_STAGE, + ); + return insertBefore(withStage, "USER nextjs", GEOIP_COPY); +} + +export function smgWebDockerfile( + upstream: string, + serviceId = CAP_WEB_SERVICE_ID, +): string { + return withGeoIpDatabase(railwayDockerfile(upstream, serviceId)); +} diff --git a/scripts/smg/railway-dockerfile.ts b/scripts/smg/railway-dockerfile.ts index b2ed84f5f2b..1698fe8c628 100644 --- a/scripts/smg/railway-dockerfile.ts +++ b/scripts/smg/railway-dockerfile.ts @@ -1,11 +1,11 @@ import { readFileSync, writeFileSync } from "node:fs"; import { parseArgs } from "node:util"; -import { railwayDockerfile } from "./lib/railway-dockerfile.ts"; +import { smgWebDockerfile } from "./lib/railway-dockerfile.ts"; const { values } = parseArgs({ options: { write: { type: "boolean", default: false } }, }); -const out = railwayDockerfile(readFileSync("apps/web/Dockerfile", "utf8")); +const out = smgWebDockerfile(readFileSync("apps/web/Dockerfile", "utf8")); if (values.write) { writeFileSync("smg/web.Dockerfile", out); console.log("wrote smg/web.Dockerfile"); diff --git a/smg/web.Dockerfile b/smg/web.Dockerfile index c8ba39057a5..04884a9ea70 100644 --- a/smg/web.Dockerfile +++ b/smg/web.Dockerfile @@ -1,4 +1,4 @@ -# Generated from apps/web/Dockerfile for Railway: no syntax directive, Railway cache mount ids. +# Generated from apps/web/Dockerfile for Railway: no syntax directive, Railway cache mount ids, GeoIP database. # Do not edit. Regenerate with: bun scripts/smg/railway-dockerfile.ts --write FROM oven/bun:1.4.4-alpine AS bun @@ -19,6 +19,18 @@ RUN bun run build:web # 3. Production image, copy all the files and run next +# GeoIP database for view locations (DB-IP City Lite, CC BY 4.0, https://db-ip.com) +FROM base AS geoip +ARG RAILWAY_GIT_COMMIT_SHA +RUN mkdir -p /geoip && cd /geoip && \ + for month in "$(date -u +%Y-%m)" "$(date -u -d "@$(( $(date -u +%s) - 2678400 ))" +%Y-%m)"; do \ + if wget -q -O db.mmdb.gz "https://download.db-ip.com/free/dbip-city-lite-$month.mmdb.gz" && gunzip -f db.mmdb.gz; then \ + mv db.mmdb dbip-city-lite.mmdb && echo "GeoIP: DB-IP City Lite $month (commit ${RAILWAY_GIT_COMMIT_SHA:-unknown})" && break; \ + fi; \ + rm -f db.mmdb.gz db.mmdb; \ + done; \ + [ -f dbip-city-lite.mmdb ] || echo "GeoIP: download failed, views will have no location" + FROM base AS runner WORKDIR /app @@ -34,6 +46,9 @@ COPY --from=builder --chown=nextjs:nodejs /app/apps/web/public ./apps/web/public COPY --from=builder --chown=nextjs:nodejs /app/packages/database/migrations ./apps/web/migrations +COPY --from=geoip --chown=nextjs:nodejs /geoip /app/geoip +ENV CAP_GEOIP_DATABASE_PATH=/app/geoip/dbip-city-lite.mmdb + USER nextjs EXPOSE 3000