Add Forms reporting and the dev lets-talk form definition

README.md

@@ -102,3 +102,7 @@ TEST_DATABASE_URL=postgresql://forms:forms_local@127.0.0.1:5546/forms?schema=for
 ```
 
 Coverage includes actual PostgreSQL writes, all field types, SQL views, CSV output, concurrent idempotency, immutable versions, JWT access, required-answer/ownership constraints, Payload synchronization over HTTP, and browser rendering/retry behavior. CI executes the same checks with PostgreSQL 17.
+
+Forms portal endpoints, inclusive UTC date filtering, and full CSV exports are
+specified in [docs/api.md](docs/api.md). The development-only `/lets-talk` migration,
+Kafka contract, seed command, and rollout order are in [docs/lets-talk.md](docs/lets-talk.md).

deploy/service.yaml

@@ -54,7 +54,7 @@ Parameters:
     Description: Bootstrap at zero; deploy after database migration succeeds.
   CorsOrigins:
     Type: String
-    Default: https://www.topcoder-dev.com
+    Default: https://www.topcoder-dev.com,https://reports.topcoder-dev.com,https://platform-ui.topcoder-dev.com
   ParameterPrefix:
     Type: String
     Default: /config/forms-api-v6/appvar

docs/api.md

@@ -131,3 +131,32 @@ Envelope/DTO errors use Nest's standard `message` array. Clients should handle b
 | 500/503 | Service/database failure; preserve input and retry appropriately.   |
 
 Reports are ordered by creation timestamp and UUID. Pagination does not create a database snapshot; use SQL/reporting jobs when an exact export snapshot is required. CSV is quoted and formula-prefixed text is neutralized for spreadsheets; arrays are JSON-encoded only in the exported CSV cell. SQL data remains fully relational.
+
+## Forms portal reporting
+
+All routes below require **Report** access and return `Cache-Control: no-store`.
+They do not grant form management privileges.
+
+| Route (under `/v6`) | Result |
+| --- | --- |
+| `GET /forms/reports/directory?after=key` | `{ data: [{ key, title }], nextCursor }`; 100 forms per page, most recent published/retired title, no drafts. |
+| `GET /forms/:key/submissions?limit=25&after=uuid&startDate=2026-09-01&endDate=2026-09-30` | `{ form, columns, labels, data, total, nextCursor }` across all published/retired revisions. |
+| `GET /forms/:key/submissions/export?startDate=2026-09-01&endDate=2026-09-30` | Complete streaming CSV with one header, independently of the table page. Omit both dates for all data. |
+
+Dates are optional **inclusive UTC calendar dates** in `YYYY-MM-DD`; an end date
+includes timestamps through 23:59:59.999. Invalid calendar dates and reversed
+ranges return 400. Existing version-specific JSON/CSV endpoints accept these same
+filters. A cursor must belong to the selected form/revision and date range.
+
+The all-revision report unions field keys in first-seen order. `labels` contains
+the latest published label for each key; absent answers are null. Existing metadata
+columns (`submission_id`, `submitted_at`, `member_id`, `source_page`, `form_version`)
+remain present. Rows sort by timestamp then UUID; `total` counts matching rows
+before pagination. Decimal values remain strings and multi-select values remain
+arrays. A field reused with a different type retains each revision's original value.
+
+The complete export accepts only date filters, not `limit` or `after`. It fetches
+1,000 rows per batch, applies backpressure, stops when the client disconnects, and
+uses the established CSV escaping/formula protection. Columns and an upper
+submission timestamp are captured at export start. This is not a repeatable-read
+database snapshot: transactions already in flight can commit during export.

docs/lets-talk.md

@@ -0,0 +1,57 @@
+# Development /lets-talk migration
+
+`examples/lets-talk.json` mirrors the form schema returned by
+`https://www.topcoder.com/__api/forms/XpPgFzW8lmX8frlStzzg0` on 2026-09-28.
+It preserves the six visible fields, their order and required flags, interest
+labels, hidden attribution fields, and the HubSpot success message.
+The stable Forms API key is `lets_talk` and the title is “Let’s talk” for reporting discovery. No historical HubSpot submissions are imported.
+
+Option values are normalized to API-safe keys:
+
+| HubSpot value | Forms API key |
+| --- | --- |
+| `Crowdsourcing` | `crowdsourcing` |
+| `App Design & Development` | `app_design_development` |
+| `Freelancers` | `freelancers` |
+| `Landing Page` (hidden lead source) | `landing_page` |
+
+The website supplies the existing hidden campaign defaults, including
+`source__c=/lets-talk`; empty UTM fields remain empty. These are validated answers,
+not trusted identity. `sourcePage` is the current pathname. The site submits
+`kafka: true` so each accepted attempt publishes `form.submitted` through the
+existing Bus API integration. Consumers must deduplicate by `submissionId`.
+
+## Rollout after review
+
+1. Deploy this API revision and the existing `SubmissionEvent` migration. Configure
+   outbound `BUSAPI_URL`, `AUTH0_URL`, `AUTH0_AUDIENCE`, `AUTH0_CLIENT_ID`,
+   `AUTH0_CLIENT_SECRET`, and `KAFKA_ERROR_TOPIC` as described in operations.md.
+2. Ensure `CORS_ORIGINS` / CloudFormation `CorsOrigins` includes the exact website
+   and reports origins (`https://www.topcoder-dev.com`,
+   `https://reports.topcoder-dev.com`, `https://platform-ui.topcoder-dev.com` for the
+   combined portal). Existing stacks retain parameter values: update the parameter
+   explicitly; changing the template default alone does not update a deployed stack.
+3. Publish the reviewed definition from this repository root, supplying an existing
+   administrator bearer token or M2M token with `manage:forms` through the environment:
+
+   ```sh
+   nvm use
+   pnpm exec tsx scripts/publish-lets-talk.ts
+   ```
+
+   `FORMS_ADMIN_TOKEN` must be set without putting it in source control. The script
+   hardcodes the development API, creates no submissions, and fails on differing
+   immutable content. Re-running an identical version is safe; subsequent schema
+   changes require a reviewed next version.
+4. Deploy the companion `topcoder-website` develop change and the `platform-ui` dev
+   change. No CMS content mutation is required: the website selects the known form
+   component only on the development `/lets-talk` route. Production and other
+   routes continue using their existing forms.
+5. Verify the public schema, then submit an explicitly marked development test with
+   Kafka configured. Confirm the receipt, event, table row, date filtering, and CSV.
+   A Bus API outage returns 503 after saving; retry unchanged inputs to finish delivery.
+
+The React integration accepts `kafka` (false by default) and `hiddenAnswers`
+(empty by default). Hidden answers are merged into the API payload and omitted
+from visible controls. They must match stored field types and validation rules.
+The website maintains its analytics adapter around the same rendering contract.

examples/lets-talk.json

@@ -0,0 +1,124 @@
+{
+  "title": "Let’s talk",
+  "description": "",
+  "successMessage": "Thanks for submitting the form. Someone will reach out to you soon.",
+  "access": "ANONYMOUS",
+  "fields": [
+    {
+      "key": "interested_in",
+      "label": "What are you interested in?",
+      "type": "SINGLE_SELECT",
+      "required": true,
+      "options": [
+        {
+          "key": "crowdsourcing",
+          "label": "Crowdsourced Innovation"
+        },
+        {
+          "key": "app_design_development",
+          "label": "App Design and Development"
+        },
+        {
+          "key": "freelancers",
+          "label": "Flexible Talent Access"
+        }
+      ]
+    },
+    {
+      "key": "firstname",
+      "label": "First name",
+      "type": "TEXT",
+      "required": true,
+      "maxLength": 500
+    },
+    {
+      "key": "lastname",
+      "label": "Last name",
+      "type": "TEXT",
+      "required": true,
+      "maxLength": 500
+    },
+    {
+      "key": "email",
+      "label": "Work Email",
+      "type": "EMAIL",
+      "required": true,
+      "maxLength": 254
+    },
+    {
+      "key": "company",
+      "label": "Company name",
+      "type": "TEXT",
+      "required": true,
+      "maxLength": 500
+    },
+    {
+      "key": "briefly_describe_your_inquiry",
+      "label": "What can we help you with?",
+      "type": "TEXTAREA",
+      "required": false,
+      "maxLength": 10000
+    },
+    {
+      "key": "leadsource",
+      "label": "Leadsource",
+      "type": "SINGLE_SELECT",
+      "required": false,
+      "options": [
+        {
+          "key": "landing_page",
+          "label": "Landing Page"
+        }
+      ]
+    },
+    {
+      "key": "source__c",
+      "label": "Landing Page Link",
+      "type": "TEXT",
+      "required": false,
+      "maxLength": 500
+    },
+    {
+      "key": "lead_source_for_campaign__c",
+      "label": "Lead Source For Campaign  C",
+      "type": "TEXT",
+      "required": false,
+      "maxLength": 500
+    },
+    {
+      "key": "primary_campaign__c",
+      "label": "Primary Campaign  C",
+      "type": "TEXT",
+      "required": false,
+      "maxLength": 500
+    },
+    {
+      "key": "insta_page_name__c",
+      "label": "Insta Page Name  C",
+      "type": "TEXT",
+      "required": false,
+      "maxLength": 500
+    },
+    {
+      "key": "utm_medium__c",
+      "label": "Utm Medium  C",
+      "type": "TEXT",
+      "required": false,
+      "maxLength": 500
+    },
+    {
+      "key": "utm_campaign__c",
+      "label": "Utm Campaign  C",
+      "type": "TEXT",
+      "required": false,
+      "maxLength": 500
+    },
+    {
+      "key": "utm_source__c",
+      "label": "Utm Source  C",
+      "type": "TEXT",
+      "required": false,
+      "maxLength": 500
+    }
+  ]
+}

integrations/react/TopcoderForm.tsx

@@ -13,20 +13,24 @@ import { isPublicForm, type FormField, type PublicForm } from '../contracts';
 export interface TopcoderFormProps {
   formKey: string;
   apiBaseUrl: string;
+  kafka?: boolean;
+  hiddenAnswers?: Readonly<Record<string, string>>;
   getAccessToken?: () => Promise<string | undefined>;
 }
 
 /**
  * Renders an API-owned schema referenced by a Payload block, preserving inputs and retry keys on failure.
  * Cancels pending work when the block reference changes so an old receipt cannot complete a new form.
- * @param props Stable form key, public API base ending in /v6, and optional signed-in member token provider.
+ * @param props Stable form key, public API base ending in /v6, optional token provider, Kafka opt-in, and API-validated hidden answer defaults.
  * @returns Accessible loading, form, error, or receipt UI; submission failures are displayed without throwing.
  * @throws No errors during ordinary rendering; fetch/token failures are caught and displayed.
  */
 export function TopcoderForm({
   formKey,
   apiBaseUrl,
   getAccessToken,
+  kafka = false,
+  hiddenAnswers = {},
 }: TopcoderFormProps): ReactNode {
   const [schema, setSchema] = useState<PublicForm | null>(null);
   const [state, setState] = useState<
@@ -84,7 +88,8 @@ export function TopcoderForm({
     const data = new FormData(event.currentTarget);
     const body = JSON.stringify({
       version: schema.version,
-      answers: collectAnswers(schema.fields, data),
+      answers: { ...collectAnswers(schema.fields, data), ...hiddenAnswers },
+      ...(kafka ? { kafka: true } : {}),
       sourcePage: window.location.pathname,
       website: data.get('_website') ?? '',
     });
@@ -130,9 +135,11 @@ export function TopcoderForm({
             ? 'This form changed or this attempt conflicts with a previous submission. Reload the page before submitting again.'
             : response.status === 401
               ? 'Please sign in again to submit this form.'
-              : response.status === 429
-                ? 'Too many requests. Please wait a minute and try again.'
-                : 'Your submission could not be saved. Check the fields and try again.',
+              : response.status === 503
+                ? 'Your submission may be saved, but delivery is not confirmed. Please retry without changing your answers.'
+                : response.status === 429
+                  ? 'Too many requests. Please wait a minute and try again.'
+                  : 'Your submission could not be saved. Check the fields and try again.',
         );
         setState('ready');
         return;
@@ -187,14 +194,16 @@ export function TopcoderForm({
         >
           {schema.title}
         </legend>
-        {schema.fields.map((field) => (
-          <FieldControl
-            key={field.key}
-            field={field}
-            id={`${instance}-${field.key}`}
-            error={fieldErrors[field.key]}
-          />
-        ))}
+        {schema.fields
+          .filter((field) => !Object.hasOwn(hiddenAnswers, field.key))
+          .map((field) => (
+            <FieldControl
+              key={field.key}
+              field={field}
+              id={`${instance}-${field.key}`}
+              error={fieldErrors[field.key]}
+            />
+          ))}
         <div
           aria-hidden="true"
           style={{ position: 'absolute', left: '-10000px' }}

scripts/publish-lets-talk.ts

@@ -0,0 +1,43 @@
+import { readFile } from 'node:fs/promises';
+
+/**
+ * Idempotently publishes the reviewed /lets-talk definition to the development Forms API only.
+ * @returns Completion after registration, immutable version save, and publication; creates no submissions.
+ * @throws Error for a missing FORMS_ADMIN_TOKEN, conflicting revision, or failed API request.
+ */
+async function publishLetsTalk(): Promise<void> {
+  const token = process.env.FORMS_ADMIN_TOKEN;
+  if (!token)
+    throw new Error(
+      'FORMS_ADMIN_TOKEN with manage:forms or Administrator access is required.',
+    );
+  const definition: unknown = JSON.parse(
+    await readFile('examples/lets-talk.json', 'utf8'),
+  );
+  const base = 'https://api.topcoder-dev.com/v6/forms';
+  for (const [method, path, body] of [
+    ['POST', '', { key: 'lets_talk' }],
+    ['PUT', '/lets_talk/versions/1', definition],
+    ['POST', '/lets_talk/versions/1/publish', undefined],
+  ] as const) {
+    const response = await fetch(`${base}${path}`, {
+      method,
+      headers: {
+        Authorization: `Bearer ${token}`,
+        'Content-Type': 'application/json',
+      },
+      ...(body ? { body: JSON.stringify(body) } : {}),
+      signal: AbortSignal.timeout(30000),
+    });
+    if (!response.ok)
+      throw new Error(
+        `${method} ${path} failed (${response.status}); no credentials or response data logged.`,
+      );
+  }
+  console.log('Published lets_talk version 1 to the development Forms API.');
+}
+
+void publishLetsTalk().catch((error: unknown) => {
+  console.error(error instanceof Error ? error.message : 'Publication failed.');
+  process.exitCode = 1;
+});

src/bootstrap.ts

@@ -24,6 +24,10 @@ class SafeExceptionFilter implements ExceptionFilter {
    */
   catch(exception: unknown, host: ArgumentsHost): void {
     const response = host.switchToHttp().getResponse<Response>();
+    if (response.headersSent) {
+      response.destroy();
+      return;
+    }
     if (exception instanceof HttpException) {
       const body = exception.getResponse();
       response

src/forms/dto.ts

@@ -155,8 +155,27 @@ export class SubmissionDto {
   kafka?: boolean;
 }
 
-/** Defines bounded keyset pagination for reporting; only cursors in this version are accepted. */
-export class PageDto {
+/** Validates inclusive UTC calendar dates shared by submission tables and full CSV exports. */
+export class ReportDatesDto {
+  @ApiPropertyOptional({
+    example: '2026-09-01',
+    description: 'Inclusive UTC date (YYYY-MM-DD).',
+  })
+  @IsOptional()
+  @Matches(/^\d{4}-\d{2}-\d{2}$/)
+  startDate?: string;
+
+  @ApiPropertyOptional({
+    example: '2026-09-30',
+    description: 'Inclusive UTC date (YYYY-MM-DD).',
+  })
+  @IsOptional()
+  @Matches(/^\d{4}-\d{2}-\d{2}$/)
+  endDate?: string;
+}
+
+/** Defines bounded keyset pagination and inclusive UTC dates for private reporting. */
+export class PageDto extends ReportDatesDto {
   @ApiPropertyOptional({ default: 100, maximum: 1000 })
   @Type(() => Number)
   @IsInt()