From c12074a29867a21286afae7a27728ecd9f61e3c7 Mon Sep 17 00:00:00 2001 From: jmgasper Date: Mon, 28 Sep 2026 09:31:14 +1000 Subject: [PATCH] Add form submission reporting and dev lets-talk definition --- README.md | 4 + deploy/service.yaml | 2 +- docs/api.md | 29 +++ docs/lets-talk.md | 57 ++++++ examples/lets-talk.json | 124 +++++++++++++ integrations/react/TopcoderForm.tsx | 35 ++-- scripts/publish-lets-talk.ts | 43 +++++ src/bootstrap.ts | 4 + src/forms/dto.ts | 23 ++- src/forms/forms.controller.ts | 40 ++++- src/forms/forms.service.ts | 203 ++++++++++++++++++++- src/forms/reporting.ts | 33 +++- test/forms.integration.spec.ts | 264 +++++++++++++++++++++++++++- test/renderer.spec.ts | 8 +- 14 files changed, 843 insertions(+), 26 deletions(-) create mode 100644 docs/lets-talk.md create mode 100644 examples/lets-talk.json create mode 100644 scripts/publish-lets-talk.ts diff --git a/README.md b/README.md index 7fb610a..a498b3f 100644 --- a/README.md +++ b/README.md @@ -102,3 +102,7 @@ TEST_DATABASE_URL=postgresql://forms:forms_local@127.0.0.1:5546/forms?schema=for ``` Coverage includes actual PostgreSQL writes, all field types, SQL views, CSV output, concurrent idempotency, immutable versions, JWT access, required-answer/ownership constraints, Payload synchronization over HTTP, and browser rendering/retry behavior. CI executes the same checks with PostgreSQL 17. + +Forms portal endpoints, inclusive UTC date filtering, and full CSV exports are +specified in [docs/api.md](docs/api.md). The development-only `/lets-talk` migration, +Kafka contract, seed command, and rollout order are in [docs/lets-talk.md](docs/lets-talk.md). diff --git a/deploy/service.yaml b/deploy/service.yaml index 96b9337..1615793 100644 --- a/deploy/service.yaml +++ b/deploy/service.yaml @@ -54,7 +54,7 @@ Parameters: Description: Bootstrap at zero; deploy after database migration succeeds. CorsOrigins: Type: String - Default: https://www.topcoder-dev.com + Default: https://www.topcoder-dev.com,https://reports.topcoder-dev.com,https://platform-ui.topcoder-dev.com ParameterPrefix: Type: String Default: /config/forms-api-v6/appvar diff --git a/docs/api.md b/docs/api.md index d7082ca..715f176 100644 --- a/docs/api.md +++ b/docs/api.md @@ -131,3 +131,32 @@ Envelope/DTO errors use Nest's standard `message` array. Clients should handle b | 500/503 | Service/database failure; preserve input and retry appropriately. | Reports are ordered by creation timestamp and UUID. Pagination does not create a database snapshot; use SQL/reporting jobs when an exact export snapshot is required. CSV is quoted and formula-prefixed text is neutralized for spreadsheets; arrays are JSON-encoded only in the exported CSV cell. SQL data remains fully relational. + +## Forms portal reporting + +All routes below require **Report** access and return `Cache-Control: no-store`. +They do not grant form management privileges. + +| Route (under `/v6`) | Result | +| --- | --- | +| `GET /forms/reports/directory?after=key` | `{ data: [{ key, title }], nextCursor }`; 100 forms per page, most recent published/retired title, no drafts. | +| `GET /forms/:key/submissions?limit=25&after=uuid&startDate=2026-09-01&endDate=2026-09-30` | `{ form, columns, labels, data, total, nextCursor }` across all published/retired revisions. | +| `GET /forms/:key/submissions/export?startDate=2026-09-01&endDate=2026-09-30` | Complete streaming CSV with one header, independently of the table page. Omit both dates for all data. | + +Dates are optional **inclusive UTC calendar dates** in `YYYY-MM-DD`; an end date +includes timestamps through 23:59:59.999. Invalid calendar dates and reversed +ranges return 400. Existing version-specific JSON/CSV endpoints accept these same +filters. A cursor must belong to the selected form/revision and date range. + +The all-revision report unions field keys in first-seen order. `labels` contains +the latest published label for each key; absent answers are null. Existing metadata +columns (`submission_id`, `submitted_at`, `member_id`, `source_page`, `form_version`) +remain present. Rows sort by timestamp then UUID; `total` counts matching rows +before pagination. Decimal values remain strings and multi-select values remain +arrays. A field reused with a different type retains each revision's original value. + +The complete export accepts only date filters, not `limit` or `after`. It fetches +1,000 rows per batch, applies backpressure, stops when the client disconnects, and +uses the established CSV escaping/formula protection. Columns and an upper +submission timestamp are captured at export start. This is not a repeatable-read +database snapshot: transactions already in flight can commit during export. diff --git a/docs/lets-talk.md b/docs/lets-talk.md new file mode 100644 index 0000000..226253f --- /dev/null +++ b/docs/lets-talk.md @@ -0,0 +1,57 @@ +# Development /lets-talk migration + +`examples/lets-talk.json` mirrors the form schema returned by +`https://www.topcoder.com/__api/forms/XpPgFzW8lmX8frlStzzg0` on 2026-09-28. +It preserves the six visible fields, their order and required flags, interest +labels, hidden attribution fields, and the HubSpot success message. +The stable Forms API key is `lets_talk` and the title is “Let’s talk” for reporting discovery. No historical HubSpot submissions are imported. + +Option values are normalized to API-safe keys: + +| HubSpot value | Forms API key | +| --- | --- | +| `Crowdsourcing` | `crowdsourcing` | +| `App Design & Development` | `app_design_development` | +| `Freelancers` | `freelancers` | +| `Landing Page` (hidden lead source) | `landing_page` | + +The website supplies the existing hidden campaign defaults, including +`source__c=/lets-talk`; empty UTM fields remain empty. These are validated answers, +not trusted identity. `sourcePage` is the current pathname. The site submits +`kafka: true` so each accepted attempt publishes `form.submitted` through the +existing Bus API integration. Consumers must deduplicate by `submissionId`. + +## Rollout after review + +1. Deploy this API revision and the existing `SubmissionEvent` migration. Configure + outbound `BUSAPI_URL`, `AUTH0_URL`, `AUTH0_AUDIENCE`, `AUTH0_CLIENT_ID`, + `AUTH0_CLIENT_SECRET`, and `KAFKA_ERROR_TOPIC` as described in operations.md. +2. Ensure `CORS_ORIGINS` / CloudFormation `CorsOrigins` includes the exact website + and reports origins (`https://www.topcoder-dev.com`, + `https://reports.topcoder-dev.com`, `https://platform-ui.topcoder-dev.com` for the + combined portal). Existing stacks retain parameter values: update the parameter + explicitly; changing the template default alone does not update a deployed stack. +3. Publish the reviewed definition from this repository root, supplying an existing + administrator bearer token or M2M token with `manage:forms` through the environment: + + ```sh + nvm use + pnpm exec tsx scripts/publish-lets-talk.ts + ``` + + `FORMS_ADMIN_TOKEN` must be set without putting it in source control. The script + hardcodes the development API, creates no submissions, and fails on differing + immutable content. Re-running an identical version is safe; subsequent schema + changes require a reviewed next version. +4. Deploy the companion `topcoder-website` develop change and the `platform-ui` dev + change. No CMS content mutation is required: the website selects the known form + component only on the development `/lets-talk` route. Production and other + routes continue using their existing forms. +5. Verify the public schema, then submit an explicitly marked development test with + Kafka configured. Confirm the receipt, event, table row, date filtering, and CSV. + A Bus API outage returns 503 after saving; retry unchanged inputs to finish delivery. + +The React integration accepts `kafka` (false by default) and `hiddenAnswers` +(empty by default). Hidden answers are merged into the API payload and omitted +from visible controls. They must match stored field types and validation rules. +The website maintains its analytics adapter around the same rendering contract. diff --git a/examples/lets-talk.json b/examples/lets-talk.json new file mode 100644 index 0000000..aa046bf --- /dev/null +++ b/examples/lets-talk.json @@ -0,0 +1,124 @@ +{ + "title": "Let’s talk", + "description": "", + "successMessage": "Thanks for submitting the form. Someone will reach out to you soon.", + "access": "ANONYMOUS", + "fields": [ + { + "key": "interested_in", + "label": "What are you interested in?", + "type": "SINGLE_SELECT", + "required": true, + "options": [ + { + "key": "crowdsourcing", + "label": "Crowdsourced Innovation" + }, + { + "key": "app_design_development", + "label": "App Design and Development" + }, + { + "key": "freelancers", + "label": "Flexible Talent Access" + } + ] + }, + { + "key": "firstname", + "label": "First name", + "type": "TEXT", + "required": true, + "maxLength": 500 + }, + { + "key": "lastname", + "label": "Last name", + "type": "TEXT", + "required": true, + "maxLength": 500 + }, + { + "key": "email", + "label": "Work Email", + "type": "EMAIL", + "required": true, + "maxLength": 254 + }, + { + "key": "company", + "label": "Company name", + "type": "TEXT", + "required": true, + "maxLength": 500 + }, + { + "key": "briefly_describe_your_inquiry", + "label": "What can we help you with?", + "type": "TEXTAREA", + "required": false, + "maxLength": 10000 + }, + { + "key": "leadsource", + "label": "Leadsource", + "type": "SINGLE_SELECT", + "required": false, + "options": [ + { + "key": "landing_page", + "label": "Landing Page" + } + ] + }, + { + "key": "source__c", + "label": "Landing Page Link", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "lead_source_for_campaign__c", + "label": "Lead Source For Campaign C", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "primary_campaign__c", + "label": "Primary Campaign C", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "insta_page_name__c", + "label": "Insta Page Name C", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "utm_medium__c", + "label": "Utm Medium C", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "utm_campaign__c", + "label": "Utm Campaign C", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "utm_source__c", + "label": "Utm Source C", + "type": "TEXT", + "required": false, + "maxLength": 500 + } + ] +} diff --git a/integrations/react/TopcoderForm.tsx b/integrations/react/TopcoderForm.tsx index c7a2cf6..89c485b 100644 --- a/integrations/react/TopcoderForm.tsx +++ b/integrations/react/TopcoderForm.tsx @@ -13,13 +13,15 @@ import { isPublicForm, type FormField, type PublicForm } from '../contracts'; export interface TopcoderFormProps { formKey: string; apiBaseUrl: string; + kafka?: boolean; + hiddenAnswers?: Readonly>; getAccessToken?: () => Promise; } /** * Renders an API-owned schema referenced by a Payload block, preserving inputs and retry keys on failure. * Cancels pending work when the block reference changes so an old receipt cannot complete a new form. - * @param props Stable form key, public API base ending in /v6, and optional signed-in member token provider. + * @param props Stable form key, public API base ending in /v6, optional token provider, Kafka opt-in, and API-validated hidden answer defaults. * @returns Accessible loading, form, error, or receipt UI; submission failures are displayed without throwing. * @throws No errors during ordinary rendering; fetch/token failures are caught and displayed. */ @@ -27,6 +29,8 @@ export function TopcoderForm({ formKey, apiBaseUrl, getAccessToken, + kafka = false, + hiddenAnswers = {}, }: TopcoderFormProps): ReactNode { const [schema, setSchema] = useState(null); const [state, setState] = useState< @@ -84,7 +88,8 @@ export function TopcoderForm({ const data = new FormData(event.currentTarget); const body = JSON.stringify({ version: schema.version, - answers: collectAnswers(schema.fields, data), + answers: { ...collectAnswers(schema.fields, data), ...hiddenAnswers }, + ...(kafka ? { kafka: true } : {}), sourcePage: window.location.pathname, website: data.get('_website') ?? '', }); @@ -130,9 +135,11 @@ export function TopcoderForm({ ? 'This form changed or this attempt conflicts with a previous submission. Reload the page before submitting again.' : response.status === 401 ? 'Please sign in again to submit this form.' - : response.status === 429 - ? 'Too many requests. Please wait a minute and try again.' - : 'Your submission could not be saved. Check the fields and try again.', + : response.status === 503 + ? 'Your submission may be saved, but delivery is not confirmed. Please retry without changing your answers.' + : response.status === 429 + ? 'Too many requests. Please wait a minute and try again.' + : 'Your submission could not be saved. Check the fields and try again.', ); setState('ready'); return; @@ -187,14 +194,16 @@ export function TopcoderForm({ > {schema.title} - {schema.fields.map((field) => ( - - ))} + {schema.fields + .filter((field) => !Object.hasOwn(hiddenAnswers, field.key)) + .map((field) => ( + + ))}