chore(release): promote dev to main

apps/importer/src/shopee/__tests__/entity-matcher.test.ts

@@ -13,6 +13,8 @@ import {
   ENTITY_MATCH_CANDIDATE_FLOOR,
   sanitizeHttpUrl,
   sanitizeHttpUrlList,
+  toBoundedPositiveInt,
+  toBoundedPositiveNumber,
 } from '../entity-matcher.js';
 import {
   ProductNormalizer,
@@ -629,6 +631,50 @@ async function runEntityMatcherSuite() {
   assert.equal(updatedRow821?.battery_mah, 5000);
   console.log('  ✅ [8/8] DOS/Rate#821 EntityMatcher bounds & HTTP URL sanitization passed');
 
+  // 9. DOS/Rate#876: EntityMatcher URL C1 control/whitespace/port bounds, boolean numeric rejection & logger control-char sanitization
+  assert.equal(
+    sanitizeHttpUrl('https://rate.vn/img/\u0085bad.png'),
+    null,
+    'sanitizeHttpUrl must reject C1 control characters'
+  );
+  assert.equal(
+    sanitizeHttpUrl('https://rate.vn/img/\u00A0bad.png'),
+    null,
+    'sanitizeHttpUrl must reject Unicode non-breaking space'
+  );
+  assert.equal(
+    sanitizeHttpUrl('https://rate.vn:8443/img/bad.png'),
+    null,
+    'sanitizeHttpUrl must reject non-default ports'
+  );
+  assert.equal(
+    toBoundedPositiveInt(true, 2048),
+    null,
+    'toBoundedPositiveInt must reject boolean true'
+  );
+  assert.equal(
+    toBoundedPositiveNumber(true, 120),
+    null,
+    'toBoundedPositiveNumber must reject boolean true'
+  );
+  assert.equal(
+    sanitizeAiGeneratedText('Máy\u0085rất\u0000tốt', Number.NaN),
+    'Máy rất tốt',
+    'sanitizeAiGeneratedText must strip C1 control chars and fall back to default maxLen when given NaN'
+  );
+  assert.equal(sanitizeAiGeneratedText('Máy rất tốt', true as unknown as number), 'Máy rất tốt');
+  const ctrlLogger = createImporterLogger('Comp\u0000\nName', {
+    runId: '  run\u0000\r\n123  ',
+    jobId: '  job\u0085456  ',
+    platform: '  shopee\u0007  ',
+  });
+  const ctrlEntry = ctrlLogger.formatEntry('info', '  evt\u0000\nname  ', 'ok');
+  assert.equal(ctrlEntry.component, 'Comp Name');
+  assert.equal(ctrlEntry.run_id, 'run 123');
+  assert.equal(ctrlEntry.job_id, 'job 456');
+  assert.equal(ctrlEntry.platform, 'shopee');
+  assert.equal(ctrlEntry.event, 'evt name');
+  console.log('  ✅ [9/9] DOS/Rate#876 EntityMatcher & Logger control-char/boolean bounds passed');
   console.log('🎉 All EntityMatcher & Open-Core Audit C6/C8/T15/Low tests passed!');
 }
 

apps/importer/src/shopee/__tests__/paging.test.ts

@@ -1,5 +1,6 @@
 import assert from 'node:assert/strict';
-import { fetchAllPages, parseShopeeEnqueueUrl } from '../paging.js';
+import { fetchAllPages, parseShopeeEnqueueUrl, resolveBackfillSupabaseTarget } from '../paging.js';
+import { mergeKeySpecs, sanitizeKeySpecKey } from '../jobs/backfill-specs.js';
 
 // A fake query over `total` rows that, like PostgREST, never returns more than
 // `maxRows` per response whatever range is asked for.
@@ -110,6 +111,46 @@ async function run() {
   );
   assert.deepEqual(clampedCalls, [[0, 999]], 'fetchAllPages must clamp pageSize at 1000');
 
+  // Issue #873: non-array data, boolean pageSize/maxPages, C1 control chars, port check & resolveBackfillSupabaseTarget
+  const nonArrayRes = await fetchAllPages(
+    'non-array\u0000label',
+    () => Promise.resolve({ data: { notAnArray: true } as unknown as never[], error: null }),
+    true as unknown as number,
+    true as unknown as number
+  );
+  assert.deepEqual(nonArrayRes, []);
+
+  assert.equal(
+    parseShopeeEnqueueUrl('https://shopee.vn:8443/product/88201679/27110898667'),
+    null,
+    'must reject non-standard port (#873)'
+  );
+  assert.equal(
+    parseShopeeEnqueueUrl('https://shopee.vn/product/88201679/27110898667\u0085'),
+    null,
+    'must reject C1 Unicode control characters (#873)'
+  );
+
+  assert.deepEqual(resolveBackfillSupabaseTarget('  https://example.supabase.co/rest/v1  '), {
+    url: 'https://example.supabase.co',
+    host: 'example.supabase.co',
+  });
+  assert.throws(
+    () => resolveBackfillSupabaseTarget('https://user:pass@example.supabase.co'),
+    /embedded credentials/,
+    'resolveBackfillSupabaseTarget must reject embedded credentials'
+  );
+  assert.throws(
+    () => resolveBackfillSupabaseTarget('ftp://example.supabase.co'),
+    /http:\/\/ or https:\/\//
+  );
+
+  assert.equal(sanitizeKeySpecKey('chip\u0085name'), 'chip name');
+  assert.deepEqual(mergeKeySpecs({ a: 'val\u0085one' }, { __proto__: 'bad', b: 'ok' }), {
+    a: 'val one',
+    b: 'ok',
+  });
+
   console.log('✅ fetchAllPages & parseShopeeEnqueueUrl suite passed');
 }
 

apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts

@@ -10,6 +10,7 @@ import assert from 'node:assert/strict';
 import { trimTrailingSlashes } from '../ai-spec-enricher.js';
 import {
   ShopeeIngestionPipeline,
+  hasPopulatedSpecificationGroups,
   normalizeScrapedProductPrices,
   platformReviewIdFor,
 } from '../shopee-to-supabase.js';
@@ -709,6 +710,67 @@ async function runTests() {
   );
   console.log('  ✅ [7/7] DOS/Rate#821 HTTP URL sanitization & unpriced rejection passed');
 
+  // 8. DOS/Rate#879: platformReviewIdFor whitespace/control-char/overlong bounds, review videos/profileUrl sanitization & hasPopulatedSpecificationGroups
+  assert.equal(
+    platformReviewIdFor({ ...maskedReviewA, reviewId: '   \u0000\n  ' }),
+    idA,
+    'platformReviewIdFor must fall back to SHA-256 hash when reviewId is whitespace or control characters'
+  );
+  assert.equal(
+    platformReviewIdFor({ ...maskedReviewA, reviewId: '  sp-rev\u0000-879  ' }),
+    'sp-rev-879',
+    'platformReviewIdFor must strip control characters and trim explicit reviewId'
+  );
+  assert.equal(
+    platformReviewIdFor({ ...maskedReviewA, reviewId: 'r'.repeat(161) }),
+    idA,
+    'platformReviewIdFor must fall back to SHA-256 hash when reviewId exceeds 160 chars'
+  );
+  assert.equal(
+    hasPopulatedSpecificationGroups({ design: { colors: ['   ', ''] } }),
+    false,
+    'hasPopulatedSpecificationGroups must reject arrays of blank/whitespace strings'
+  );
+  assert.equal(hasPopulatedSpecificationGroups({ design: { colors: ['Black'] } }), true);
+
+  const sanitized879 = normalizeScrapedProductPrices({
+    ...createBaseProduct(),
+    reviews: [
+      {
+        reviewId: 'rev-879',
+        author: 'Buyer',
+        ratingStar: 5,
+        comment: 'Great',
+        images: [],
+        videos: [
+          'javascript:alert(1)',
+          'https://rate.vn/vid/1.mp4',
+          'https://rate.vn:8443/bad.mp4',
+        ],
+        reviewer: {
+          verifiedPurchase: true,
+          profileUrl: 'javascript:alert(1)',
+        },
+      },
+      {
+        reviewId: 'rev-879-ok',
+        author: 'Buyer 2',
+        ratingStar: 5,
+        comment: 'Nice',
+        images: [],
+        reviewer: {
+          verifiedPurchase: true,
+          profileUrl: 'https://shopee.vn/shop/12345',
+        },
+      },
+    ],
+  });
+  assert.deepEqual(sanitized879.reviews[0]?.videos, ['https://rate.vn/vid/1.mp4']);
+  assert.equal(sanitized879.reviews[0]?.reviewer?.profileUrl, undefined);
+  assert.equal(sanitized879.reviews[1]?.reviewer?.profileUrl, 'https://shopee.vn/shop/12345');
+  console.log(
+    '  ✅ [8/8] DOS/Rate#879 platformReviewIdFor, review videos/profileUrl & spec group bounds passed'
+  );
   console.log('\n🎉 ALL SHOPEE INGESTION PIPELINE TESTS PASSED!');
 }
 

apps/importer/src/shopee/ai-review-analyzer.ts

@@ -1,7 +1,7 @@
 import crypto from 'node:crypto';
 import { z, MAX_VALID_VND_PRICE, seedingRiskPenalty } from '@repo/shared-data';
 import {
-  trimTrailingSlashes,
+  resolveValidHttpBaseUrl,
   extractJsonObjectString,
   type SupportedAiProvider,
   type AiSpecEnricherConfig,
@@ -585,6 +585,7 @@ const RATING_LABEL_RE = /^(?:(?:cực\s*k[ìỳ]|rất)\s+)?(?:không\s+)?hài\s
 export function isRatingOnlyComment(comment: string | null | undefined): boolean {
   const text = (comment || '')
     .normalize('NFC')
+    .replace(/\p{Cc}+/gu, '')
     .replace(/[\s.!,]+$/u, '')
     .trim();
   return text.length === 0 || RATING_LABEL_RE.test(text);
@@ -617,7 +618,10 @@ const PRODUCT_MENTION_RE =
  * any product attribute or usage experience (Section 4 #13 & Section 13 P1).
  */
 export function isShippingOnlyComment(comment: string | null | undefined): boolean {
-  const text = (comment || '').normalize('NFC').trim();
+  const text = (comment || '')
+    .normalize('NFC')
+    .replace(/\p{Cc}+/gu, '')
+    .trim();
   if (text.length < 6 || isRatingOnlyComment(text)) return false;
   return SHIPPING_ASPECT_RE.test(text) && !PRODUCT_MENTION_RE.test(text);
 }
@@ -630,7 +634,10 @@ export function hasStarSentimentContradiction(
   ratingStar: number,
   comment: string | null | undefined
 ): boolean {
-  const text = (comment || '').normalize('NFC').trim();
+  const text = (comment || '')
+    .normalize('NFC')
+    .replace(/\p{Cc}+/gu, '')
+    .trim();
   if (text.length < 10 || isRatingOnlyComment(text)) return false;
 
   if (ratingStar >= 5) {
@@ -686,8 +693,16 @@ export function computeVoterWeight(params: {
 }): number {
   const { accountAgeDays, verifiedReviewCount, trustScore, isVerifiedContact = true } = params;
   if (!isVerifiedContact) return 0;
-  if (accountAgeDays !== null && accountAgeDays < 7) return 0;
-  if ((accountAgeDays !== null && accountAgeDays < 30) || verifiedReviewCount <= 0) {
+  const validAge =
+    typeof accountAgeDays === 'number' && Number.isFinite(accountAgeDays) && accountAgeDays >= 0
+      ? accountAgeDays
+      : null;
+  const validReviews =
+    typeof verifiedReviewCount === 'number' && Number.isFinite(verifiedReviewCount)
+      ? verifiedReviewCount
+      : 0;
+  if (validAge !== null && validAge < 7) return 0;
+  if ((validAge !== null && validAge < 30) || validReviews <= 0) {
     return 0.1;
   }
   const rawT = typeof trustScore === 'number' && Number.isFinite(trustScore) ? trustScore : 0.5;
@@ -728,24 +743,25 @@ function normalizeTextForDedupe(text: string): string {
  * Used to detect paraphrased or reordered seeding templates across distinct accounts/listings.
  */
 export function computeTextShingleSimilarity(textA: string, textB: string, k = 3): number {
+  const safeK = typeof k === 'number' && Number.isInteger(k) && k >= 1 && k <= 10 ? k : 3;
   const normA = normalizeTextForDedupe(textA);
   const normB = normalizeTextForDedupe(textB);
   if (!normA || !normB) return 0;
   if (normA === normB) return 1;
 
   const wordsA = normA.split(' ').filter(Boolean);
   const wordsB = normB.split(' ').filter(Boolean);
-  if (wordsA.length < k || wordsB.length < k) {
+  if (wordsA.length < safeK || wordsB.length < safeK) {
     return 0;
   }
 
   const shinglesA = new Set<string>();
-  for (let i = 0; i <= wordsA.length - k; i++) {
-    shinglesA.add(wordsA.slice(i, i + k).join(' '));
+  for (let i = 0; i <= wordsA.length - safeK; i++) {
+    shinglesA.add(wordsA.slice(i, i + safeK).join(' '));
   }
   const shinglesB = new Set<string>();
-  for (let i = 0; i <= wordsB.length - k; i++) {
-    shinglesB.add(wordsB.slice(i, i + k).join(' '));
+  for (let i = 0; i <= wordsB.length - safeK; i++) {
+    shinglesB.add(wordsB.slice(i, i + safeK).join(' '));
   }
 
   let intersection = 0;
@@ -771,7 +787,10 @@ export function assessReview6Axes(
     isConflictOfInterest?: boolean;
   } = {}
 ): ReviewAssessment {
-  const comment = (review.comment || '').normalize('NFC').trim();
+  const comment = (review.comment || '')
+    .normalize('NFC')
+    .replace(/\p{Cc}+/gu, '')
+    .trim();
   const cleanText = comment.replace(/[\s.!,]+$/u, '').trim();
 
   // 1. Extract aspects & routing
@@ -1007,22 +1026,14 @@ export class AiReviewAnalyzer {
     const defaults = PROVIDER_DEFAULTS[this.provider];
 
     this.apiKey = (config.apiKey ?? this.resolveApiKey(this.provider)).trim();
-    const rawBaseUrl = trimTrailingSlashes(
+    this.baseUrl = resolveValidHttpBaseUrl(
       config.baseUrl ||
         process.env.AI_BASE_URL ||
         process.env.DOS_AI_BASE_URL ||
         process.env.OPENAI_BASE_URL ||
-        defaults.baseUrl
+        defaults.baseUrl,
+      defaults.baseUrl
     );
-    try {
-      const parsedBase = new URL(rawBaseUrl);
-      this.baseUrl =
-        parsedBase.protocol === 'https:' || parsedBase.protocol === 'http:'
-          ? rawBaseUrl
-          : defaults.baseUrl;
-    } catch {
-      this.baseUrl = defaults.baseUrl;
-    }
 
     this.model =
       (config.model || process.env.AI_MODEL || process.env.OPENAI_MODEL || '').trim() ||
@@ -1102,7 +1113,10 @@ export class AiReviewAnalyzer {
     }
 
     return reviews.map(r => {
-      const comment = (r.comment || '').trim();
+      const comment = (r.comment || '')
+        .normalize('NFC')
+        .replace(/\p{Cc}+/gu, '')
+        .trim();
       const norm = normalizeTextForDedupe(comment);
       const distinctAuthorsWithSameText = commentCounts.get(norm)?.size ?? 1;
       const maxShingleSim = maxShingleSimByReviewId.get(r.id) ?? 0;
@@ -1352,13 +1366,10 @@ export class AiReviewAnalyzer {
   } | null> {
     const sanitizePromptText = (raw: string | null | undefined, maxLen = 1500): string => {
       if (!raw) return '';
-      return (
-        raw
-          // eslint-disable-next-line no-control-regex -- stripping control characters is the point
-          .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, ' ')
-          .trim()
-          .slice(0, maxLen)
-      );
+      return raw
+        .replace(/\p{Cc}+/gu, ' ')
+        .trim()
+        .slice(0, maxLen);
     };
 
     const sanitizeOutputText = (raw: unknown, maxLen = 400): string =>