diff --git a/apps/importer/src/shopee/__tests__/entity-matcher.test.ts b/apps/importer/src/shopee/__tests__/entity-matcher.test.ts index e131c5c93..4521e8800 100644 --- a/apps/importer/src/shopee/__tests__/entity-matcher.test.ts +++ b/apps/importer/src/shopee/__tests__/entity-matcher.test.ts @@ -13,6 +13,8 @@ import { ENTITY_MATCH_CANDIDATE_FLOOR, sanitizeHttpUrl, sanitizeHttpUrlList, + toBoundedPositiveInt, + toBoundedPositiveNumber, } from '../entity-matcher.js'; import { ProductNormalizer, @@ -629,6 +631,50 @@ async function runEntityMatcherSuite() { assert.equal(updatedRow821?.battery_mah, 5000); console.log(' ✅ [8/8] DOS/Rate#821 EntityMatcher bounds & HTTP URL sanitization passed'); + // 9. DOS/Rate#876: EntityMatcher URL C1 control/whitespace/port bounds, boolean numeric rejection & logger control-char sanitization + assert.equal( + sanitizeHttpUrl('https://rate.vn/img/\u0085bad.png'), + null, + 'sanitizeHttpUrl must reject C1 control characters' + ); + assert.equal( + sanitizeHttpUrl('https://rate.vn/img/\u00A0bad.png'), + null, + 'sanitizeHttpUrl must reject Unicode non-breaking space' + ); + assert.equal( + sanitizeHttpUrl('https://rate.vn:8443/img/bad.png'), + null, + 'sanitizeHttpUrl must reject non-default ports' + ); + assert.equal( + toBoundedPositiveInt(true, 2048), + null, + 'toBoundedPositiveInt must reject boolean true' + ); + assert.equal( + toBoundedPositiveNumber(true, 120), + null, + 'toBoundedPositiveNumber must reject boolean true' + ); + assert.equal( + sanitizeAiGeneratedText('Máy\u0085rất\u0000tốt', Number.NaN), + 'Máy rất tốt', + 'sanitizeAiGeneratedText must strip C1 control chars and fall back to default maxLen when given NaN' + ); + assert.equal(sanitizeAiGeneratedText('Máy rất tốt', true as unknown as number), 'Máy rất tốt'); + const ctrlLogger = createImporterLogger('Comp\u0000\nName', { + runId: ' run\u0000\r\n123 ', + jobId: ' job\u0085456 ', + platform: ' shopee\u0007 ', + }); + const ctrlEntry = ctrlLogger.formatEntry('info', ' evt\u0000\nname ', 'ok'); + assert.equal(ctrlEntry.component, 'Comp Name'); + assert.equal(ctrlEntry.run_id, 'run 123'); + assert.equal(ctrlEntry.job_id, 'job 456'); + assert.equal(ctrlEntry.platform, 'shopee'); + assert.equal(ctrlEntry.event, 'evt name'); + console.log(' ✅ [9/9] DOS/Rate#876 EntityMatcher & Logger control-char/boolean bounds passed'); console.log('🎉 All EntityMatcher & Open-Core Audit C6/C8/T15/Low tests passed!'); } diff --git a/apps/importer/src/shopee/__tests__/paging.test.ts b/apps/importer/src/shopee/__tests__/paging.test.ts index 88533e470..868521d25 100644 --- a/apps/importer/src/shopee/__tests__/paging.test.ts +++ b/apps/importer/src/shopee/__tests__/paging.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; -import { fetchAllPages, parseShopeeEnqueueUrl } from '../paging.js'; +import { fetchAllPages, parseShopeeEnqueueUrl, resolveBackfillSupabaseTarget } from '../paging.js'; +import { mergeKeySpecs, sanitizeKeySpecKey } from '../jobs/backfill-specs.js'; // A fake query over `total` rows that, like PostgREST, never returns more than // `maxRows` per response whatever range is asked for. @@ -110,6 +111,46 @@ async function run() { ); assert.deepEqual(clampedCalls, [[0, 999]], 'fetchAllPages must clamp pageSize at 1000'); + // Issue #873: non-array data, boolean pageSize/maxPages, C1 control chars, port check & resolveBackfillSupabaseTarget + const nonArrayRes = await fetchAllPages( + 'non-array\u0000label', + () => Promise.resolve({ data: { notAnArray: true } as unknown as never[], error: null }), + true as unknown as number, + true as unknown as number + ); + assert.deepEqual(nonArrayRes, []); + + assert.equal( + parseShopeeEnqueueUrl('https://shopee.vn:8443/product/88201679/27110898667'), + null, + 'must reject non-standard port (#873)' + ); + assert.equal( + parseShopeeEnqueueUrl('https://shopee.vn/product/88201679/27110898667\u0085'), + null, + 'must reject C1 Unicode control characters (#873)' + ); + + assert.deepEqual(resolveBackfillSupabaseTarget(' https://example.supabase.co/rest/v1 '), { + url: 'https://example.supabase.co', + host: 'example.supabase.co', + }); + assert.throws( + () => resolveBackfillSupabaseTarget('https://user:pass@example.supabase.co'), + /embedded credentials/, + 'resolveBackfillSupabaseTarget must reject embedded credentials' + ); + assert.throws( + () => resolveBackfillSupabaseTarget('ftp://example.supabase.co'), + /http:\/\/ or https:\/\// + ); + + assert.equal(sanitizeKeySpecKey('chip\u0085name'), 'chip name'); + assert.deepEqual(mergeKeySpecs({ a: 'val\u0085one' }, { __proto__: 'bad', b: 'ok' }), { + a: 'val one', + b: 'ok', + }); + console.log('✅ fetchAllPages & parseShopeeEnqueueUrl suite passed'); } diff --git a/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts b/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts index 582fc17c5..0fc318860 100644 --- a/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts +++ b/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts @@ -10,6 +10,7 @@ import assert from 'node:assert/strict'; import { trimTrailingSlashes } from '../ai-spec-enricher.js'; import { ShopeeIngestionPipeline, + hasPopulatedSpecificationGroups, normalizeScrapedProductPrices, platformReviewIdFor, } from '../shopee-to-supabase.js'; @@ -709,6 +710,67 @@ async function runTests() { ); console.log(' ✅ [7/7] DOS/Rate#821 HTTP URL sanitization & unpriced rejection passed'); + // 8. DOS/Rate#879: platformReviewIdFor whitespace/control-char/overlong bounds, review videos/profileUrl sanitization & hasPopulatedSpecificationGroups + assert.equal( + platformReviewIdFor({ ...maskedReviewA, reviewId: ' \u0000\n ' }), + idA, + 'platformReviewIdFor must fall back to SHA-256 hash when reviewId is whitespace or control characters' + ); + assert.equal( + platformReviewIdFor({ ...maskedReviewA, reviewId: ' sp-rev\u0000-879 ' }), + 'sp-rev-879', + 'platformReviewIdFor must strip control characters and trim explicit reviewId' + ); + assert.equal( + platformReviewIdFor({ ...maskedReviewA, reviewId: 'r'.repeat(161) }), + idA, + 'platformReviewIdFor must fall back to SHA-256 hash when reviewId exceeds 160 chars' + ); + assert.equal( + hasPopulatedSpecificationGroups({ design: { colors: [' ', ''] } }), + false, + 'hasPopulatedSpecificationGroups must reject arrays of blank/whitespace strings' + ); + assert.equal(hasPopulatedSpecificationGroups({ design: { colors: ['Black'] } }), true); + + const sanitized879 = normalizeScrapedProductPrices({ + ...createBaseProduct(), + reviews: [ + { + reviewId: 'rev-879', + author: 'Buyer', + ratingStar: 5, + comment: 'Great', + images: [], + videos: [ + 'javascript:alert(1)', + 'https://rate.vn/vid/1.mp4', + 'https://rate.vn:8443/bad.mp4', + ], + reviewer: { + verifiedPurchase: true, + profileUrl: 'javascript:alert(1)', + }, + }, + { + reviewId: 'rev-879-ok', + author: 'Buyer 2', + ratingStar: 5, + comment: 'Nice', + images: [], + reviewer: { + verifiedPurchase: true, + profileUrl: 'https://shopee.vn/shop/12345', + }, + }, + ], + }); + assert.deepEqual(sanitized879.reviews[0]?.videos, ['https://rate.vn/vid/1.mp4']); + assert.equal(sanitized879.reviews[0]?.reviewer?.profileUrl, undefined); + assert.equal(sanitized879.reviews[1]?.reviewer?.profileUrl, 'https://shopee.vn/shop/12345'); + console.log( + ' ✅ [8/8] DOS/Rate#879 platformReviewIdFor, review videos/profileUrl & spec group bounds passed' + ); console.log('\n🎉 ALL SHOPEE INGESTION PIPELINE TESTS PASSED!'); } diff --git a/apps/importer/src/shopee/ai-review-analyzer.ts b/apps/importer/src/shopee/ai-review-analyzer.ts index 8a8bb66ee..5aa6eb35c 100644 --- a/apps/importer/src/shopee/ai-review-analyzer.ts +++ b/apps/importer/src/shopee/ai-review-analyzer.ts @@ -1,7 +1,7 @@ import crypto from 'node:crypto'; import { z, MAX_VALID_VND_PRICE, seedingRiskPenalty } from '@repo/shared-data'; import { - trimTrailingSlashes, + resolveValidHttpBaseUrl, extractJsonObjectString, type SupportedAiProvider, type AiSpecEnricherConfig, @@ -585,6 +585,7 @@ const RATING_LABEL_RE = /^(?:(?:cực\s*k[ìỳ]|rất)\s+)?(?:không\s+)?hài\s export function isRatingOnlyComment(comment: string | null | undefined): boolean { const text = (comment || '') .normalize('NFC') + .replace(/\p{Cc}+/gu, '') .replace(/[\s.!,]+$/u, '') .trim(); return text.length === 0 || RATING_LABEL_RE.test(text); @@ -617,7 +618,10 @@ const PRODUCT_MENTION_RE = * any product attribute or usage experience (Section 4 #13 & Section 13 P1). */ export function isShippingOnlyComment(comment: string | null | undefined): boolean { - const text = (comment || '').normalize('NFC').trim(); + const text = (comment || '') + .normalize('NFC') + .replace(/\p{Cc}+/gu, '') + .trim(); if (text.length < 6 || isRatingOnlyComment(text)) return false; return SHIPPING_ASPECT_RE.test(text) && !PRODUCT_MENTION_RE.test(text); } @@ -630,7 +634,10 @@ export function hasStarSentimentContradiction( ratingStar: number, comment: string | null | undefined ): boolean { - const text = (comment || '').normalize('NFC').trim(); + const text = (comment || '') + .normalize('NFC') + .replace(/\p{Cc}+/gu, '') + .trim(); if (text.length < 10 || isRatingOnlyComment(text)) return false; if (ratingStar >= 5) { @@ -686,8 +693,16 @@ export function computeVoterWeight(params: { }): number { const { accountAgeDays, verifiedReviewCount, trustScore, isVerifiedContact = true } = params; if (!isVerifiedContact) return 0; - if (accountAgeDays !== null && accountAgeDays < 7) return 0; - if ((accountAgeDays !== null && accountAgeDays < 30) || verifiedReviewCount <= 0) { + const validAge = + typeof accountAgeDays === 'number' && Number.isFinite(accountAgeDays) && accountAgeDays >= 0 + ? accountAgeDays + : null; + const validReviews = + typeof verifiedReviewCount === 'number' && Number.isFinite(verifiedReviewCount) + ? verifiedReviewCount + : 0; + if (validAge !== null && validAge < 7) return 0; + if ((validAge !== null && validAge < 30) || validReviews <= 0) { return 0.1; } const rawT = typeof trustScore === 'number' && Number.isFinite(trustScore) ? trustScore : 0.5; @@ -728,6 +743,7 @@ function normalizeTextForDedupe(text: string): string { * Used to detect paraphrased or reordered seeding templates across distinct accounts/listings. */ export function computeTextShingleSimilarity(textA: string, textB: string, k = 3): number { + const safeK = typeof k === 'number' && Number.isInteger(k) && k >= 1 && k <= 10 ? k : 3; const normA = normalizeTextForDedupe(textA); const normB = normalizeTextForDedupe(textB); if (!normA || !normB) return 0; @@ -735,17 +751,17 @@ export function computeTextShingleSimilarity(textA: string, textB: string, k = 3 const wordsA = normA.split(' ').filter(Boolean); const wordsB = normB.split(' ').filter(Boolean); - if (wordsA.length < k || wordsB.length < k) { + if (wordsA.length < safeK || wordsB.length < safeK) { return 0; } const shinglesA = new Set(); - for (let i = 0; i <= wordsA.length - k; i++) { - shinglesA.add(wordsA.slice(i, i + k).join(' ')); + for (let i = 0; i <= wordsA.length - safeK; i++) { + shinglesA.add(wordsA.slice(i, i + safeK).join(' ')); } const shinglesB = new Set(); - for (let i = 0; i <= wordsB.length - k; i++) { - shinglesB.add(wordsB.slice(i, i + k).join(' ')); + for (let i = 0; i <= wordsB.length - safeK; i++) { + shinglesB.add(wordsB.slice(i, i + safeK).join(' ')); } let intersection = 0; @@ -771,7 +787,10 @@ export function assessReview6Axes( isConflictOfInterest?: boolean; } = {} ): ReviewAssessment { - const comment = (review.comment || '').normalize('NFC').trim(); + const comment = (review.comment || '') + .normalize('NFC') + .replace(/\p{Cc}+/gu, '') + .trim(); const cleanText = comment.replace(/[\s.!,]+$/u, '').trim(); // 1. Extract aspects & routing @@ -1007,22 +1026,14 @@ export class AiReviewAnalyzer { const defaults = PROVIDER_DEFAULTS[this.provider]; this.apiKey = (config.apiKey ?? this.resolveApiKey(this.provider)).trim(); - const rawBaseUrl = trimTrailingSlashes( + this.baseUrl = resolveValidHttpBaseUrl( config.baseUrl || process.env.AI_BASE_URL || process.env.DOS_AI_BASE_URL || process.env.OPENAI_BASE_URL || - defaults.baseUrl + defaults.baseUrl, + defaults.baseUrl ); - try { - const parsedBase = new URL(rawBaseUrl); - this.baseUrl = - parsedBase.protocol === 'https:' || parsedBase.protocol === 'http:' - ? rawBaseUrl - : defaults.baseUrl; - } catch { - this.baseUrl = defaults.baseUrl; - } this.model = (config.model || process.env.AI_MODEL || process.env.OPENAI_MODEL || '').trim() || @@ -1102,7 +1113,10 @@ export class AiReviewAnalyzer { } return reviews.map(r => { - const comment = (r.comment || '').trim(); + const comment = (r.comment || '') + .normalize('NFC') + .replace(/\p{Cc}+/gu, '') + .trim(); const norm = normalizeTextForDedupe(comment); const distinctAuthorsWithSameText = commentCounts.get(norm)?.size ?? 1; const maxShingleSim = maxShingleSimByReviewId.get(r.id) ?? 0; @@ -1352,13 +1366,10 @@ export class AiReviewAnalyzer { } | null> { const sanitizePromptText = (raw: string | null | undefined, maxLen = 1500): string => { if (!raw) return ''; - return ( - raw - // eslint-disable-next-line no-control-regex -- stripping control characters is the point - .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, ' ') - .trim() - .slice(0, maxLen) - ); + return raw + .replace(/\p{Cc}+/gu, ' ') + .trim() + .slice(0, maxLen); }; const sanitizeOutputText = (raw: unknown, maxLen = 400): string => diff --git a/apps/importer/src/shopee/ai-spec-enricher.ts b/apps/importer/src/shopee/ai-spec-enricher.ts index 35e8445d6..469245eb1 100644 --- a/apps/importer/src/shopee/ai-spec-enricher.ts +++ b/apps/importer/src/shopee/ai-spec-enricher.ts @@ -144,8 +144,11 @@ function toBooleanOrUndefined(val: unknown): boolean | undefined { return undefined; } -function toStringOrUndefined(val: unknown): string | undefined { - if (typeof val === 'string' && val.trim().length > 0) return val.trim(); +function toStringOrUndefined(val: unknown, maxLen = 500): string | undefined { + if (typeof val === 'string') { + const cleaned = sanitizeAiGeneratedText(val, maxLen).trim(); + return cleaned.length > 0 ? cleaned : undefined; + } if (typeof val === 'number' && Number.isFinite(val)) return String(val); return undefined; } @@ -153,8 +156,10 @@ function toStringOrUndefined(val: unknown): string | undefined { function sanitizeColorList(val: unknown): string[] | undefined { if (!Array.isArray(val)) return undefined; const cleaned = Array.from( - new Set(val.map((c: unknown) => toStringOrUndefined(c)).filter((c): c is string => Boolean(c))) - ); + new Set( + val.map((c: unknown) => toStringOrUndefined(c, 80)).filter((c): c is string => Boolean(c)) + ) + ).slice(0, 32); return cleaned.length > 0 ? cleaned : undefined; } @@ -433,12 +438,19 @@ export function trimTrailingSlashes(rawUrl: string): string { return end === rawUrl.length ? rawUrl : rawUrl.slice(0, end); } -function resolveValidHttpBaseUrl(candidateUrl: string, fallbackUrl: string): string { - const cleaned = trimTrailingSlashes((candidateUrl || '').trim()); - if (cleaned) { +export function resolveValidHttpBaseUrl(candidateUrl: string, fallbackUrl: string): string { + const cleaned = trimTrailingSlashes(typeof candidateUrl === 'string' ? candidateUrl.trim() : ''); + if (cleaned && cleaned.length <= 512 && !/\p{Cc}/u.test(cleaned) && !/\s/.test(cleaned)) { try { const parsed = new URL(cleaned); - if ((parsed.protocol === 'http:' || parsed.protocol === 'https:') && parsed.hostname) { + if ( + (parsed.protocol === 'http:' || parsed.protocol === 'https:') && + parsed.hostname && + parsed.username === '' && + parsed.password === '' && + parsed.search === '' && + parsed.hash === '' + ) { return cleaned; } } catch { diff --git a/apps/importer/src/shopee/entity-matcher.ts b/apps/importer/src/shopee/entity-matcher.ts index a6abcb815..d5f8e4798 100644 --- a/apps/importer/src/shopee/entity-matcher.ts +++ b/apps/importer/src/shopee/entity-matcher.ts @@ -30,11 +30,7 @@ const MAX_BATTERY_MAH = 500_000; export const MAX_ENTITY_IMAGES = 50; function hasControlOrSpaceChar(val: string): boolean { - for (let i = 0; i < val.length; i++) { - const code = val.charCodeAt(i); - if (code <= 0x20 || code === 0x7f) return true; - } - return false; + return /[\p{Cc}\s]/u.test(val); } export function sanitizeHttpUrl(raw: unknown): string | null { @@ -46,6 +42,7 @@ export function sanitizeHttpUrl(raw: unknown): string | null { if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null; if (parsed.username || parsed.password) return null; if (!parsed.hostname) return null; + if (parsed.port !== '' && parsed.port !== '80' && parsed.port !== '443') return null; return trimmed; } catch { return null; @@ -67,12 +64,14 @@ export function sanitizeHttpUrlList(urls: unknown): string[] { return out; } -function toBoundedPositiveInt(value: unknown, max: number): number | null { +export function toBoundedPositiveInt(value: unknown, max: number): number | null { + if (typeof value === 'boolean') return null; const n = Number(value); return Number.isInteger(n) && n > 0 && n <= max ? n : null; } -function toBoundedPositiveNumber(value: unknown, max: number): number | null { +export function toBoundedPositiveNumber(value: unknown, max: number): number | null { + if (typeof value === 'boolean') return null; const n = Number(value); return Number.isFinite(n) && n > 0 && n <= max ? n : null; } diff --git a/apps/importer/src/shopee/jobs/backfill-review-risk.ts b/apps/importer/src/shopee/jobs/backfill-review-risk.ts index 3d43bb78a..70fa4d25d 100644 --- a/apps/importer/src/shopee/jobs/backfill-review-risk.ts +++ b/apps/importer/src/shopee/jobs/backfill-review-risk.ts @@ -17,7 +17,7 @@ import { pathToFileURL } from 'node:url'; import dotenv from 'dotenv'; import { AiReviewAnalyzer } from '../ai-review-analyzer.js'; -import { fetchAllPages } from '../paging.js'; +import { fetchAllPages, resolveBackfillSupabaseTarget } from '../paging.js'; import { MERCHANT_RISK_SELECT, PRODUCT_RISK_SELECT, @@ -114,18 +114,19 @@ if (isDirectExecution) { dotenv.config({ path: '../../.env', quiet: true }); (async () => { const { createClient } = await import('@supabase/supabase-js'); - const url = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL || ''; - const key = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_KEY || ''; - if (!url || !key) { + const rawUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL || ''; + const key = (process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_KEY || '').trim(); + if (!rawUrl.trim() || !key) { throw new Error('Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY in environment.'); } + const { url, host } = resolveBackfillSupabaseTarget(rawUrl); const supabase = scopeSupabaseToRate( createClient(url, key, { auth: { persistSession: false } }) ); const apply = process.argv.includes('--apply'); // The host is not secret; it shows which project an --apply run writes to. - logger.info('backfill.target', `Target: ${new URL(url).host} (apply=${apply})`, { - host: new URL(url).host, + logger.info('backfill.target', `Target: ${host} (apply=${apply})`, { + host, apply, }); await runReviewRiskBackfill(supabase, { apply }); diff --git a/apps/importer/src/shopee/jobs/backfill-specs.ts b/apps/importer/src/shopee/jobs/backfill-specs.ts index 6a22bda37..490eab1ff 100644 --- a/apps/importer/src/shopee/jobs/backfill-specs.ts +++ b/apps/importer/src/shopee/jobs/backfill-specs.ts @@ -15,7 +15,7 @@ import { pathToFileURL } from 'node:url'; import dotenv from 'dotenv'; import { ProductSpecificationsSchema, type ProductSpecifications } from '@repo/shared-data'; import { AiSpecEnricher, mergeProductSpecifications } from '../ai-spec-enricher.js'; -import { fetchAllPages } from '../paging.js'; +import { fetchAllPages, resolveBackfillSupabaseTarget } from '../paging.js'; import { CANONICAL_BACKFILL_SPECS_BY_MODEL_KEY, FLAGSHIP_DEVICES_TO_SEED, @@ -105,19 +105,10 @@ export const MAX_KEY_SPEC_OBJECT_KEYS = 100; const UNSAFE_KEY_SPEC_KEYS = new Set(['__proto__', 'constructor', 'prototype']); function stripKeySpecControlChars(input: string): string { - let out = ''; - for (let i = 0; i < input.length; i++) { - const code = input.charCodeAt(i); - if (code > 0x1f && code !== 0x7f) { - out += input[i]; - } else { - out += ' '; - } - } - return out; + return input.replace(/\p{Cc}+/gu, ' '); } -function sanitizeKeySpecKey(key: string): string | null { +export function sanitizeKeySpecKey(key: string): string | null { const cleaned = stripKeySpecControlChars(key).replace(/\s+/g, ' ').trim(); if (!cleaned || cleaned.length > MAX_KEY_SPEC_KEY_LENGTH || UNSAFE_KEY_SPEC_KEYS.has(cleaned)) { return null; @@ -200,11 +191,12 @@ export async function runMasterSpecsBackfill( const { createClient } = await import('@supabase/supabase-js'); const { EntityMatcher } = await import('../entity-matcher.js'); - const url = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL || ''; - const key = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_KEY || ''; - if (!url || !key) { + const rawUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL || ''; + const key = (process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_KEY || '').trim(); + if (!rawUrl.trim() || !key) { throw new Error('Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY in environment.'); } + const { url } = resolveBackfillSupabaseTarget(rawUrl); const supabase = scopeSupabaseToRate(createClient(url, key, { auth: { persistSession: false } })); const enricher = new AiSpecEnricher(); @@ -413,14 +405,17 @@ export async function runQuickSpecSyncAndAudit( ): Promise { const apply = options.apply === true; const { createClient } = await import('@supabase/supabase-js'); - const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL || process.env.SUPABASE_URL; - const supabaseKey = + const rawSupabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL || process.env.SUPABASE_URL; + const supabaseKey = ( process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || - process.env.SUPABASE_ANON_KEY; - if (!supabaseUrl || !supabaseKey) { + process.env.SUPABASE_ANON_KEY || + '' + ).trim(); + if (!rawSupabaseUrl?.trim() || !supabaseKey) { throw new Error('Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY'); } + const { url: supabaseUrl } = resolveBackfillSupabaseTarget(rawSupabaseUrl); const supabase = scopeSupabaseToRate( createClient(supabaseUrl, supabaseKey, { auth: { persistSession: false }, diff --git a/apps/importer/src/shopee/logger.ts b/apps/importer/src/shopee/logger.ts index 2442e07a3..185685c0d 100644 --- a/apps/importer/src/shopee/logger.ts +++ b/apps/importer/src/shopee/logger.ts @@ -75,8 +75,13 @@ export function computeRetryDelayMs( */ export function sanitizeAiGeneratedText(raw: unknown, maxLen = 400): string { if (typeof raw !== 'string') return ''; + const safeMaxLen = + typeof maxLen === 'number' && Number.isFinite(maxLen) && maxLen >= 1 + ? Math.min(Math.floor(maxLen), MAX_LOG_STRING_LENGTH) + : 400; + const boundedInput = raw.slice(0, Math.max(safeMaxLen * 4, 4096)); return ( - raw + boundedInput // `[^<>]` rather than `[^>]`: a run of unclosed '<' would otherwise be // rescanned to the end of the string from each '<' (quadratic). .replace(/<[^<>]*>/g, ' ') @@ -89,11 +94,10 @@ export function sanitizeAiGeneratedText(raw: unknown, maxLen = 400): string { ) // Strip Vietnamese phone / Zalo numbers (e.g. 0912345678, +84 912 345 678, 098.123.4567) .replace(/(?:\+?84|0)\s*[1-9](?:[\s.-]*\d){7,9}\b/g, '[phone-removed]') - // eslint-disable-next-line no-control-regex -- stripping control characters is intentional - .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, ' ') + .replace(/\p{Cc}+/gu, ' ') .replace(/\s+/g, ' ') .trim() - .slice(0, maxLen) + .slice(0, safeMaxLen) ); } @@ -227,17 +231,29 @@ export class ImporterStructuredLogger { message: string, meta?: Record ): ImporterLogEntry { + const cleanField = (val: unknown, fallback = ''): string => + typeof val === 'string' + ? val + .replace(/\p{Cc}+/gu, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 120) || fallback + : fallback; + const cleanNullableField = (val: unknown): string | null => { + const cleaned = cleanField(val, ''); + return cleaned.length > 0 ? cleaned : null; + }; const sanitizedMeta = meta ? (sanitizeLogMeta(meta) as Record) : undefined; return { timestamp: new Date().toISOString(), level, service: '@repo/crawlers', - component: this.context.component, - run_id: this.context.runId ?? process.env.RATE_RUN_ID ?? null, - job_id: this.context.jobId ?? null, - platform: this.context.platform ?? null, - event, - message: redactSensitiveString(message), + component: cleanField(this.context.component, 'Importer'), + run_id: cleanNullableField(this.context.runId ?? process.env.RATE_RUN_ID ?? null), + job_id: cleanNullableField(this.context.jobId ?? null), + platform: cleanNullableField(this.context.platform ?? null), + event: cleanField(event, 'importer.event'), + message: redactSensitiveString(typeof message === 'string' ? message : String(message ?? '')), ...(sanitizedMeta && Object.keys(sanitizedMeta).length > 0 ? { meta: sanitizedMeta } : {}), }; } diff --git a/apps/importer/src/shopee/paging.ts b/apps/importer/src/shopee/paging.ts index db8746942..81c1b3c52 100644 --- a/apps/importer/src/shopee/paging.ts +++ b/apps/importer/src/shopee/paging.ts @@ -21,24 +21,33 @@ export async function fetchAllPages( pageSize = MAX_FETCH_ALL_PAGE_SIZE, maxPages = DEFAULT_FETCH_ALL_MAX_PAGES ): Promise { + const safeLabel = + typeof label === 'string' + ? label + .replace(/\p{Cc}+/gu, ' ') + .trim() + .slice(0, 160) || 'fetchAllPages' + : 'fetchAllPages'; + const rawPageSize = typeof pageSize === 'boolean' ? Number.NaN : Number(pageSize); const safePageSize = - Number.isFinite(pageSize) && pageSize >= 1 - ? Math.min(MAX_FETCH_ALL_PAGE_SIZE, Math.floor(pageSize)) + Number.isFinite(rawPageSize) && rawPageSize >= 1 + ? Math.min(MAX_FETCH_ALL_PAGE_SIZE, Math.floor(rawPageSize)) : MAX_FETCH_ALL_PAGE_SIZE; + const rawMaxPages = typeof maxPages === 'boolean' ? Number.NaN : Number(maxPages); const safeMaxPages = - Number.isFinite(maxPages) && maxPages >= 1 - ? Math.min(MAX_FETCH_ALL_PAGES_CEILING, Math.floor(maxPages)) + Number.isFinite(rawMaxPages) && rawMaxPages >= 1 + ? Math.min(MAX_FETCH_ALL_PAGES_CEILING, Math.floor(rawMaxPages)) : DEFAULT_FETCH_ALL_MAX_PAGES; const rows: T[] = []; let nonEmptyPages = 0; for (let from = 0; ;) { const { data, error } = await buildPage(from, from + safePageSize - 1); - if (error) throw new Error(`${label}: ${error.message}`); - const page = data ?? []; + if (error) throw new Error(`${safeLabel}: ${error.message}`); + const page = Array.isArray(data) ? data : []; if (page.length === 0) return rows; nonEmptyPages += 1; if (nonEmptyPages > safeMaxPages) { - throw new Error(`${label}: exceeded maxPages (${safeMaxPages}) at offset ${from}`); + throw new Error(`${safeLabel}: exceeded maxPages (${safeMaxPages}) at offset ${from}`); } rows.push(...page); from += page.length; @@ -56,10 +65,13 @@ export function parseShopeeEnqueueUrl( ): { shopId: string; itemId: string; normalizedUrl: string } | null { if (typeof rawUrl !== 'string') return null; const trimmed = rawUrl.trim(); - if (!trimmed || trimmed.length > MAX_SHOPEE_ENQUEUE_URL_LENGTH) return null; - for (let i = 0; i < trimmed.length; i++) { - const code = trimmed.charCodeAt(i); - if (code <= 0x20 || code === 0x7f) return null; + if ( + !trimmed || + trimmed.length > MAX_SHOPEE_ENQUEUE_URL_LENGTH || + /\s/.test(trimmed) || + /\p{Cc}/u.test(trimmed) + ) { + return null; } let parsed: URL; try { @@ -73,6 +85,9 @@ export function parseShopeeEnqueueUrl( if (parsed.username || parsed.password) { return null; } + if (parsed.port !== '' && parsed.port !== '443' && parsed.port !== '80') { + return null; + } const host = parsed.hostname.toLowerCase(); if (host !== 'shopee.vn' && !host.endsWith('.shopee.vn')) { return null; @@ -89,3 +104,36 @@ export function parseShopeeEnqueueUrl( normalizedUrl: parsed.toString(), }; } + +/** + * Validates and normalizes the target Supabase project URL for backfill/audit jobs. + * Rejects non-HTTP(S) schemes, embedded credentials, control characters, and whitespace. + */ +export function resolveBackfillSupabaseTarget(rawUrl: string | undefined | null): { + url: string; + host: string; +} { + const trimmed = typeof rawUrl === 'string' ? rawUrl.trim() : ''; + if (!trimmed || trimmed.length > 2048 || /\s/.test(trimmed) || /\p{Cc}/u.test(trimmed)) { + throw new Error('Missing or malformed SUPABASE_URL in environment.'); + } + let parsed: URL; + try { + parsed = new URL(trimmed); + } catch { + throw new Error('Missing or malformed SUPABASE_URL in environment.'); + } + if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { + throw new Error('SUPABASE_URL must use http:// or https://.'); + } + if (parsed.username || parsed.password) { + throw new Error('SUPABASE_URL must not contain embedded credentials.'); + } + if (!parsed.hostname) { + throw new Error('SUPABASE_URL is missing hostname.'); + } + return { + url: parsed.origin, + host: parsed.host, + }; +} diff --git a/apps/importer/src/shopee/shopee-to-supabase.ts b/apps/importer/src/shopee/shopee-to-supabase.ts index 02b20602e..c9d96b4a7 100644 --- a/apps/importer/src/shopee/shopee-to-supabase.ts +++ b/apps/importer/src/shopee/shopee-to-supabase.ts @@ -21,6 +21,7 @@ import { EntityMatcher, sanitizeHttpUrl, sanitizeHttpUrlList } from './entity-ma import { AiSpecEnricher, type AiSpecEnricherConfig } from './ai-spec-enricher.js'; import { AiReviewAnalyzer, type ReviewInputItem } from './ai-review-analyzer.js'; import { createImporterLogger } from './logger.js'; +import { resolveBackfillSupabaseTarget } from './paging.js'; import { resolveClusterPeerFloorPrice, type ClusterPeerPriceCandidate, @@ -36,15 +37,18 @@ function hashEntity(val: string): string { } export function platformReviewIdFor(r: ShopeeReview): string { + const cleanId = typeof r?.reviewId === 'string' ? r.reviewId.replace(/\p{Cc}+/gu, '').trim() : ''; + if (cleanId.length > 0 && cleanId.length <= 160) { + return cleanId; + } return ( - r.reviewId || 'rv-' + - crypto - .createHash('sha256') - .update( - `${r.author || ''}|${r.ratingStar ?? ''}|${r.variation || ''}|${r.createdAt || ''}|${r.comment || ''}` - ) - .digest('hex') + crypto + .createHash('sha256') + .update( + `${r?.author || ''}|${r?.ratingStar ?? ''}|${r?.variation || ''}|${r?.createdAt || ''}|${r?.comment || ''}` + ) + .digest('hex') ); } @@ -73,7 +77,9 @@ export function hasPopulatedSpecificationGroups(specs: unknown): boolean { if (group && typeof group === 'object' && !Array.isArray(group)) { const hasValue = Object.values(group as Record).some(v => { if (v === undefined || v === null) return false; - if (Array.isArray(v)) return v.length > 0; + if (Array.isArray(v)) { + return v.some(item => item !== undefined && item !== null && String(item).trim() !== ''); + } return String(v).trim() !== ''; }); if (hasValue) return true; @@ -123,6 +129,15 @@ export function normalizeScrapedProductPrices( ...r, authorAvatar: sanitizeHttpUrl(r.authorAvatar) ?? undefined, images: sanitizeHttpUrlList(r.images), + ...(r.videos !== undefined ? { videos: sanitizeHttpUrlList(r.videos) } : {}), + ...(r.reviewer + ? { + reviewer: { + ...r.reviewer, + profileUrl: sanitizeHttpUrl(r.reviewer.profileUrl) ?? undefined, + }, + } + : {}), })) : []; return { @@ -181,12 +196,13 @@ export class ShopeeIngestionPipeline { if (aiConfig?.supabaseClient) { this.supabase = aiConfig.supabaseClient; } else { - const url = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL || ''; - const key = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_KEY || ''; + const rawUrl = process.env.SUPABASE_URL || process.env.NEXT_PUBLIC_SUPABASE_URL || ''; + const key = (process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.SUPABASE_KEY || '').trim(); - if (!url || !key) { + if (!rawUrl.trim() || !key) { throw new Error('Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY in environment.'); } + const { url } = resolveBackfillSupabaseTarget(rawUrl); const admin = createClient(url, key, { auth: { autoRefreshToken: false, persistSession: false }, diff --git a/apps/importer/src/shopee/test-ai-enricher.ts b/apps/importer/src/shopee/test-ai-enricher.ts index c3b9d9dbb..b80741e38 100644 --- a/apps/importer/src/shopee/test-ai-enricher.ts +++ b/apps/importer/src/shopee/test-ai-enricher.ts @@ -17,6 +17,7 @@ import { AiSpecEnricher, AiSpecEnrichmentPayloadSchema, extractJsonObjectString, + resolveValidHttpBaseUrl, sanitizeSpecifications, } from './ai-spec-enricher.js'; import { @@ -535,6 +536,44 @@ async function runTests() { assert.equal(enrichedBackfillStorage.ram_gb, 16); assert.equal(enrichedBackfillStorage.storage_gb, 512); + // DOS/Rate#882: resolveValidHttpBaseUrl rejects credentials/query/fragment/control-chars, and sanitizeSpecifications strips HTML/control chars & bounds colors + assert.equal( + resolveValidHttpBaseUrl('https://user:pass@api.example.com/v1', 'https://api.openai.com/v1'), + 'https://api.openai.com/v1', + 'resolveValidHttpBaseUrl must reject embedded URL credentials' + ); + assert.equal( + resolveValidHttpBaseUrl('https://api.example.com/v1?foo=1', 'https://api.openai.com/v1'), + 'https://api.openai.com/v1', + 'resolveValidHttpBaseUrl must reject query strings on baseUrl' + ); + assert.equal( + resolveValidHttpBaseUrl('https://api.example.com/v1#frag', 'https://api.openai.com/v1'), + 'https://api.openai.com/v1', + 'resolveValidHttpBaseUrl must reject fragments on baseUrl' + ); + assert.equal( + resolveValidHttpBaseUrl('https://api.example.com/v1\u0000/', 'https://api.openai.com/v1'), + 'https://api.openai.com/v1', + 'resolveValidHttpBaseUrl must reject control characters in baseUrl' + ); + const sanitizedHtmlSpecs = sanitizeSpecifications({ + hardware: { + chipset: ' Apple A18 Pro\u0000 (3 nm) ', + }, + design: { + colors: [ + 'Titan\u0000 Đen', + 'x'.repeat(120), + ...Array.from({ length: 40 }, (_, idx) => `Color-${idx}`), + ], + }, + }); + assert.equal(sanitizedHtmlSpecs.hardware?.chipset, 'Apple A18 Pro alert(1) (3 nm)'); + assert.equal(sanitizedHtmlSpecs.design?.colors?.[0], 'Titan Đen'); + assert.equal(sanitizedHtmlSpecs.design?.colors?.[1]?.length, 80); + assert.equal(sanitizedHtmlSpecs.design?.colors?.length, 32); + console.log('✅ [AiSpecEnricher] Mock OpenAI-compatible enrichment & caching passed 100%!'); } finally { mockServer.close(); diff --git a/apps/importer/src/shopee/test-review-analyzer.ts b/apps/importer/src/shopee/test-review-analyzer.ts index 0159274a3..4a78cd1fb 100644 --- a/apps/importer/src/shopee/test-review-analyzer.ts +++ b/apps/importer/src/shopee/test-review-analyzer.ts @@ -2396,8 +2396,82 @@ async function run() { 'ProductNormalizer must preserve 1.5TB (1536GB) as 1.5TB' ); + // DOS/Rate#882: computeVoterWeight NaN guards, computeTextShingleSimilarity k bounds, and control-char stripping in review heuristics + assert.equal( + computeVoterWeight({ + accountAgeDays: Number.NaN, + verifiedReviewCount: Number.NaN, + trustScore: 0.9, + isVerifiedContact: true, + }), + 0.1, + 'computeVoterWeight must cap unproven/NaN verifiedReviewCount voters at 0.1' + ); + assert.equal( + computeVoterWeight({ + accountAgeDays: -5, + verifiedReviewCount: 0, + trustScore: 0.9, + isVerifiedContact: true, + }), + 0.1, + 'computeVoterWeight must treat negative accountAgeDays as unknown and cap 0-review voters at 0.1' + ); + assert.equal( + computeTextShingleSimilarity( + 'Máy dùng rất mượt pin trâu màn hình đẹp tuyệt vời', + 'Máy dùng rất mượt pin trâu màn hình sáng rõ nét', + 0 + ), + computeTextShingleSimilarity( + 'Máy dùng rất mượt pin trâu màn hình đẹp tuyệt vời', + 'Máy dùng rất mượt pin trâu màn hình sáng rõ nét', + 3 + ), + 'computeTextShingleSimilarity must fall back to k=3 when k <= 0' + ); + assert.equal( + isRatingOnlyComment('Cực\u0000 kì hài lòng!'), + true, + 'isRatingOnlyComment must strip control characters before matching platform rating labels' + ); + assert.equal( + isShippingOnlyComment('Giao\u0000 hàng nhanh đóng gói cẩn thận'), + true, + 'isShippingOnlyComment must strip control characters before checking shipping/product regexes' + ); + assert.equal( + hasStarSentimentContradiction(5, 'Hàng\u0000 giả lừa đảo không dùng được'), + true, + 'hasStarSentimentContradiction must strip control characters before matching contradiction regex' + ); + const ctrlHeuristicRes = analyzer.analyzeWithHeuristics([ + { + id: 'rev-882-coin', + author_name: 'u1', + rating_star: 5, + comment: 'Đánh giá mang\u0000 tính chất minh họa để nhận\u0000 xu thôi ạ', + }, + { + id: 'rev-882-nulls', + author_name: 'u2', + rating_star: 5, + comment: 'ok\u0000\u0000\u0000\u0000\u0000\u0000', + }, + ]); + assert.equal( + ctrlHeuristicRes[0]?.flag_category, + 'low_value_coin_spam', + 'analyzeWithHeuristics must strip control characters so coin-farming spam cannot evade detection' + ); + assert.equal( + ctrlHeuristicRes[1]?.flag_category, + 'low_value_coin_spam', + 'analyzeWithHeuristics must strip control characters before checking minimum comment length' + ); + console.log( - `✅ [P2/P3 Trust Model & #292/#352/#356/#359/#379/#382/#401/#497/#514/#522/#528/#557/#834/#836] Voter weight, Wilson lower bound (50/52=${wilson50of52} > 1/1=${wilson1of1}), shingle copy seeding (sim=${shingleSim}), affiliate shortlinks, peer floor, ArenaSpecSyncer, Importer & ReviewAnalyzer hardening (#352, #356, #359, #379, #382, #401, #497, #514, #522, #528, #557, #834, #836) verified` + `✅ [P2/P3 Trust Model & #292/#352/#356/#359/#379/#382/#401/#497/#514/#522/#528/#557/#834/#836/#882] Voter weight, Wilson lower bound (50/52=${wilson50of52} > 1/1=${wilson1of1}), shingle copy seeding (sim=${shingleSim}), affiliate shortlinks, peer floor, ArenaSpecSyncer, Importer & ReviewAnalyzer hardening (#352, #356, #359, #379, #382, #401, #497, #514, #522, #528, #557, #834, #836, #882) verified` ); console.log('\n🎉 ALL AI REVIEW ANALYZER TESTS PASSED!'); diff --git a/apps/web/e2e/listing-image-url.spec.ts b/apps/web/e2e/listing-image-url.spec.ts new file mode 100644 index 000000000..a9b152944 --- /dev/null +++ b/apps/web/e2e/listing-image-url.spec.ts @@ -0,0 +1,23 @@ +import { expect, test } from '@playwright/test'; + +// A listing whose stored images start with "[object Object]" (as some crawled +// TGDD rows do) shows its first real image, and the page never requests the +// junk entry as a URL (DOS/Rate#810). +const PROFILE = '/merchants/ffffffff-ffff-4fff-8fff-ffffffff0769'; + +test('a listing thumbnail skips stored entries that are not URLs', async ({ + page, +}) => { + const junk: string[] = []; + page.on('request', (req) => { + if (/object(%20|\s)Object/i.test(req.url())) junk.push(req.url()); + }); + expect((await page.goto(PROFILE))?.status()).toBe(200); + const thumb = page.locator('img[alt="E2E Bud 001 chính hãng"]'); + await expect(thumb).toHaveAttribute( + 'src', + '/img/devices/apple-airpods-pro-3-usb-c.jpg' + ); + await page.waitForLoadState('networkidle'); + expect(junk).toEqual([]); +}); diff --git a/apps/web/e2e/not-found-title.spec.ts b/apps/web/e2e/not-found-title.spec.ts new file mode 100644 index 000000000..e1cc9c8ec --- /dev/null +++ b/apps/web/e2e/not-found-title.spec.ts @@ -0,0 +1,26 @@ +import { expect, test } from '@playwright/test'; + +// Every 404 under a locale is titled as one, not with the site's default +// title (DOS/Rate#877, WCAG 2.4.2). +const PATHS = ['/khong-ton-tai-abc', '/leaderboard/x', '/compare/a-vs-b']; +const TITLES = { + vi: 'Không tìm thấy trang | Rate.vn', + en: 'Page not found | Rate.vn', +}; + +for (const [locale, title] of Object.entries(TITLES)) { + for (const path of PATHS) { + test(`${path} is titled as a 404 in ${locale}`, async ({ + page, + baseURL, + }) => { + await page + .context() + .addCookies([ + { name: 'NEXT_LOCALE', value: locale, url: baseURL ?? '' }, + ]); + expect((await page.goto(path))?.status()).toBe(404); + await expect(page).toHaveTitle(title); + }); + } +} diff --git a/apps/web/e2e/seed.sql b/apps/web/e2e/seed.sql index 163a3733e..9ca76bee0 100644 --- a/apps/web/e2e/seed.sql +++ b/apps/web/e2e/seed.sql @@ -211,10 +211,13 @@ VALUES 'e2e_draft_phone', 'E2E Draft Phone', 'e2e-draft-phone', 4000000, 4000000, 1, 'draft', '{"category_slug": "smartphones"}'); -- The listing has a picture, so the profile renders a thumbnail (DOS/Rate#789). +-- Its stored images start with "[object Object]", as some crawled TGDD rows +-- do; the thumbnail must use the first real URL (DOS/Rate#810). INSERT INTO rate.products (id, merchant_id, platform, platform_item_id, title, category_id, brand, price_current, url, cluster_id, images) SELECT 'cccccccc-cccc-4ccc-8ccc-ccccccc07691', 'ffffffff-ffff-4fff-8fff-ffffffff0769', 'shopee', 'item_draft_links_public', 'E2E Bud 001 chính hãng', 'c0000000-0000-4000-8000-000000000005', 'E2E Audio', 500000, - 'https://shopee.vn/item-draft-links-public', id, ARRAY['/img/devices/apple-airpods-pro-3-usb-c.jpg'] + 'https://shopee.vn/item-draft-links-public', id, + ARRAY['[object Object]', '/img/devices/apple-airpods-pro-3-usb-c.jpg'] FROM rate.master_products WHERE slug = 'e2e-bud-001'; INSERT INTO rate.products (id, merchant_id, platform, platform_item_id, title, category_id, price_current, url, cluster_id) VALUES ('cccccccc-cccc-4ccc-8ccc-ccccccc07692', 'ffffffff-ffff-4fff-8fff-ffffffff0769', 'shopee', 'item_draft_links_draft', diff --git a/apps/web/locales/cs.json b/apps/web/locales/cs.json index 311907934..c4dc95852 100644 --- a/apps/web/locales/cs.json +++ b/apps/web/locales/cs.json @@ -119,7 +119,6 @@ } }, "compare": { - "versusMetaFallbackTitle": "Souboj 1v1 | Rate.vn", "versusMetaTitle": "{nameA} vs {nameB}: 10 skupin parametrů a hodnocení | Rate.vn", "versusMetaTitleShort": "{nameA} vs {nameB} | Rate.vn", "versusMetaDescription": "Přímé porovnání {nameA} a {nameB}: hodnocení v 5 kritériích, 10 skupin parametrů a skutečné ceny v obchodech.", diff --git a/apps/web/locales/en.json b/apps/web/locales/en.json index 15aea7d9f..b94397201 100644 --- a/apps/web/locales/en.json +++ b/apps/web/locales/en.json @@ -119,7 +119,6 @@ } }, "compare": { - "versusMetaFallbackTitle": "1v1 Compare | Rate.vn", "versusMetaTitle": "{nameA} vs {nameB}: 10 spec groups and review scores | Rate.vn", "versusMetaTitleShort": "{nameA} vs {nameB} | Rate.vn", "versusMetaDescription": "Compare {nameA} and {nameB} head to head: review scores across 5 criteria, 10 spec groups and real store prices.", diff --git a/apps/web/locales/vi.json b/apps/web/locales/vi.json index 8839b2d86..934bde130 100644 --- a/apps/web/locales/vi.json +++ b/apps/web/locales/vi.json @@ -119,7 +119,6 @@ } }, "compare": { - "versusMetaFallbackTitle": "Đối đầu 1v1 | Rate.vn", "versusMetaTitle": "So sánh {nameA} và {nameB}: 10 nhóm thông số và điểm đánh giá | Rate.vn", "versusMetaTitleShort": "{nameA} vs {nameB} | Rate.vn", "versusMetaDescription": "So sánh trực tiếp {nameA} và {nameB}: điểm đánh giá theo 5 tiêu chí, 10 nhóm thông số kỹ thuật và giá thật trên các sàn.", diff --git a/apps/web/package.json b/apps/web/package.json index 4134535ed..e08cba598 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -8,7 +8,7 @@ "lint": "eslint . --max-warnings 0", "preview": "next build && next start", "start": "next start", - "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts", + "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/image-url.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts", "typecheck": "tsc --noEmit" }, "browserslist": [ diff --git a/apps/web/src/app/[locale]/[...rest]/page.tsx b/apps/web/src/app/[locale]/[...rest]/page.tsx index 2ed7c37fd..d56896379 100644 --- a/apps/web/src/app/[locale]/[...rest]/page.tsx +++ b/apps/web/src/app/[locale]/[...rest]/page.tsx @@ -1,9 +1,26 @@ +import type { Metadata } from 'next'; +import { getTranslations } from 'next-intl/server'; import { notFound } from 'next/navigation'; +import { normalizeRateLocale } from '@/lib/rate-i18n'; // Rendered per request: caching would store one 404 entry for every path a // scanner tries (DOS/Rate#491). export const dynamic = 'force-dynamic'; +// Titled as a 404 rather than with the site's default title (DOS/Rate#877). +export async function generateMetadata({ + params, +}: { + params: Promise<{ locale: string }>; +}): Promise { + const { locale } = await params; + const t = await getTranslations({ + locale: normalizeRateLocale(locale), + namespace: 'errors.notFound', + }); + return { title: `${t('title')} | Rate.vn` }; +} + // Unknown paths get the localized not-found page of this segment instead of // the bare root one. export default function UnknownPage() { diff --git a/apps/web/src/app/[locale]/compare/[versus]/page.tsx b/apps/web/src/app/[locale]/compare/[versus]/page.tsx index cd18f8bc4..6ce9bd951 100644 --- a/apps/web/src/app/[locale]/compare/[versus]/page.tsx +++ b/apps/web/src/app/[locale]/compare/[versus]/page.tsx @@ -51,16 +51,25 @@ export async function generateMetadata({ const { locale, versus } = await params; const lang = normalizeRateLocale(locale); const t = await getTranslations({ locale: lang, namespace: 'compare' }); + // Both cases below render the 404 page, so they are titled as one + // (DOS/Rate#877). + const notFoundMeta = async (): Promise => { + const tNotFound = await getTranslations({ + locale: lang, + namespace: 'errors.notFound', + }); + return { title: `${tNotFound('title')} | Rate.vn` }; + }; const parsed = parseVersusSlugs(versus); if (!parsed) { - return { title: t('versusMetaFallbackTitle') }; + return notFoundMeta(); } const compared = await getCachedCompareMasterProducts(parsed.join(',')); const a = compared[0]; const b = compared[1]; if (!a || !b) { - return { title: t('versusMetaFallbackTitle') }; + return notFoundMeta(); } const canonicalUrl = `https://rate.vn/compare/${a.slug}-vs-${b.slug}`; diff --git a/apps/web/src/app/[locale]/page.tsx b/apps/web/src/app/[locale]/page.tsx index eee3867d6..3a5c5cddc 100644 --- a/apps/web/src/app/[locale]/page.tsx +++ b/apps/web/src/app/[locale]/page.tsx @@ -53,6 +53,7 @@ import ContentText from '@/components/elementary/ContentText'; import { contentLang } from '@/lib/content-lang'; import ClientMessages from '@/components/providers/ClientMessages'; import { siteJsonLd } from '@/lib/seo'; +import { productImageUrl } from '@/lib/rate-domain-core'; // Cached after the first request, refreshed every 5 minutes or when the // worker revalidates its cache tags (DOS/Rate#491). @@ -465,11 +466,7 @@ export default async function Home({
{masterProducts.slice(0, 6).map((item) => { - const thumb = - item.thumbnail || - (item.images && item.images.length > 0 - ? item.images[0] - : null); + const thumb = productImageUrl(item); const keySpecs = item.key_specs || {}; const productHref = getMasterProductHref(item, catSlugById); const rawUser = Number( diff --git a/apps/web/src/app/[locale]/smartphones/[slug]/page.tsx b/apps/web/src/app/[locale]/smartphones/[slug]/page.tsx index 807f1290d..ce83d5efa 100644 --- a/apps/web/src/app/[locale]/smartphones/[slug]/page.tsx +++ b/apps/web/src/app/[locale]/smartphones/[slug]/page.tsx @@ -68,6 +68,11 @@ import ProductOffersExplorer from '@/components/leaderboard/ProductOffersExplore import ProductSpecsExportToolbar from '@/components/leaderboard/ProductSpecsExportToolbar'; import ClientMessages from '@/components/providers/ClientMessages'; import { plausibleMinPrice } from '@/lib/offer-plausibility'; +import { + firstImageUrl, + usableImageUrls, + productImageUrl, +} from '@/lib/rate-domain-core'; // Cached after the first request, refreshed every 5 minutes or when the // worker revalidates its cache tags (DOS/Rate#491). @@ -125,10 +130,7 @@ export async function generateCategoryProductMetadata( title: t('ogTitle', { name: product.name }), description: t('ogDesc', { minPrice }), url: canonicalUrl, - images: - product.images && product.images.length > 0 - ? [product.images[0]!] - : [SHARE_IMAGE], + images: [firstImageUrl(product.images) ?? SHARE_IMAGE], }, }; } @@ -237,6 +239,7 @@ export async function ProductDetailView({ : rawUserRating; const keySpecLine = buildKeySpecLine(product); + const mainImage = firstImageUrl(product.images); const canonicalUrl = `https://rate.vn${categoryBasePath}/${product.slug}`; @@ -280,7 +283,7 @@ export async function ProductDetailView({ brand: product.brand, merchantCount: product.merchant_count || 0, }), - image: (product.images || []).map(absoluteUrl), + image: usableImageUrls(product.images).map(absoluteUrl), offers: { '@type': 'AggregateOffer', priceCurrency: 'VND', @@ -341,11 +344,11 @@ export async function ProductDetailView({
- {product.images && product.images.length > 0 ? ( + {mainImage ? ( // The main product image is the page's largest paint: fetch it // ahead of the below-the-fold peer and offer thumbnails. {product.name} 0 - ? product.images[0] || null - : null), + thumbnail: productImageUrl(product), min_price: currentMinPrice, categoryPath: categoryBasePath, }} @@ -862,12 +861,7 @@ export async function ProductDetailView({ name={product.name} brand={product.brand} categoryPath={categoryBasePath} - thumbnail={ - product.thumbnail || - (product.images && product.images.length > 0 - ? product.images[0] || null - : null) - } + thumbnail={productImageUrl(product)} currentMinPrice={currentMinPrice} size="sm" /> @@ -1589,11 +1583,7 @@ export async function ProductDetailView({ {relatedCompetitors.map((peer) => { const peerPrice = Number(peer.min_price || 0); const priceDelta = peerPriceDelta(peerPrice, currentMinPrice); - const peerThumb = - peer.thumbnail || - (peer.images && peer.images.length > 0 - ? peer.images[0] - : null); + const peerThumb = productImageUrl(peer); const peerRating = resolveCompareEffectiveUserRating(peer); const peerScore10 = peerRating * 2; const currentScore10 = displayUserRating * 2; diff --git a/apps/web/src/app/[locale]/smartphones/page.tsx b/apps/web/src/app/[locale]/smartphones/page.tsx index dc6e51ba8..ec0f8f61c 100644 --- a/apps/web/src/app/[locale]/smartphones/page.tsx +++ b/apps/web/src/app/[locale]/smartphones/page.tsx @@ -43,6 +43,7 @@ import { import type { KeySpecs, ProductSpecifications } from '@repo/shared-data'; import ClientMessages from '@/components/providers/ClientMessages'; import BootstrapClient from '@/app/BootstrapClient'; +import { productImageUrl } from '@/lib/rate-domain-core'; // Reads searchParams, so it renders per request. The [locale] layout's // generateStaticParams would otherwise make it static, and the searchParams @@ -928,11 +929,7 @@ export async function CategoryCatalogContent({ const keySpecs = (item.key_specs || {}) as KeySpecs; const specs = (item.specifications || {}) as ProductSpecifications; - const thumb = - item.thumbnail || - (item.images && item.images.length > 0 - ? item.images[0] - : null); + const thumb = productImageUrl(item); const effectiveRating = item.user_rating_weighted && diff --git a/apps/web/src/components/compare/CompareWorkspace.tsx b/apps/web/src/components/compare/CompareWorkspace.tsx index 12482020b..2801865d4 100644 --- a/apps/web/src/components/compare/CompareWorkspace.tsx +++ b/apps/web/src/components/compare/CompareWorkspace.tsx @@ -56,6 +56,7 @@ import { import { upsertWatchlistItem } from '@/components/watchlist/watchlist-store'; import { getSpecUiDictionary, localizeSpecValue } from '@/lib/spec-localizer'; import ContentText from '@/components/elementary/ContentText'; +import { productImageUrl } from '@/lib/rate-domain-core'; interface CompareWorkspaceProps { initialProducts: MasterProduct[]; @@ -851,11 +852,7 @@ export default function CompareWorkspace({ slug: p.slug, name: p.name, brand: p.brand, - thumbnail: resolveProductImageUrl( - p.slug, - p.thumbnail || - (p.images && p.images.length > 0 ? p.images[0] || null : null) - ), + thumbnail: resolveProductImageUrl(p.slug, productImageUrl(p)), min_price: Number(p.min_price || 0), categorySlug: targetCat, })) @@ -1213,11 +1210,7 @@ export default function CompareWorkspace({ minP > 0 ? Math.max(100_000, Math.round((minP * 0.95) / 100_000) * 100_000) : 10_000_000; - const thumb = resolveProductImageUrl( - p.slug, - p.thumbnail || - (p.images && p.images.length > 0 ? p.images[0] || null : null) - ); + const thumb = resolveProductImageUrl(p.slug, productImageUrl(p)); const resolvedHref = getMasterProductHref(p); const resolvedCategoryPath = resolvedHref.replace(`/${p.slug}`, '') || '/smartphones'; @@ -1409,10 +1402,7 @@ export default function CompareWorkspace({ const alreadySelected = selectedSlugs.has(item.slug); const thumb = resolveProductImageUrl( item.slug, - item.thumbnail || - (item.images && item.images.length > 0 - ? item.images[0] - : null) + productImageUrl(item) ); const priceFormatted = Number( item.min_price || 0 @@ -1966,8 +1956,7 @@ export default function CompareWorkspace({ {products.map((p, idx) => { const thumb = resolveProductImageUrl( p.slug, - p.thumbnail || - (p.images && p.images.length > 0 ? p.images[0] : null) + productImageUrl(p) ); const minPrice = Number(p.min_price || 0); const minPriceFormatted = minPrice.toLocaleString('vi-VN'); diff --git a/apps/web/src/components/home/HeroSearchInspector.tsx b/apps/web/src/components/home/HeroSearchInspector.tsx index d05adc4a0..533cd2490 100644 --- a/apps/web/src/components/home/HeroSearchInspector.tsx +++ b/apps/web/src/components/home/HeroSearchInspector.tsx @@ -34,6 +34,7 @@ import { BsXLg, } from 'react-icons/bs'; import ContentText from '@/components/elementary/ContentText'; +import { firstImageUrl, productImageUrl } from '@/lib/rate-domain-core'; interface HeroSearchInspectorProps { initialProducts: HeroProductSuggestion[]; @@ -392,11 +393,7 @@ export default function HeroSearchInspector({
{filteredProducts.map((item, idx) => { const keySpecs = (item.key_specs || {}) as KeySpecs; - const thumb = - item.thumbnail || - (item.images && item.images.length > 0 - ? item.images[0] - : null); + const thumb = productImageUrl(item); const minPrice = Number( item.min_price || 0 ).toLocaleString('vi-VN'); @@ -669,9 +666,13 @@ export default function HeroSearchInspector({
- {inspectResult.masterProduct.images?.[0] && ( + {firstImageUrl(inspectResult.masterProduct.images) && ( {inspectResult.masterProduct.name} diff --git a/apps/web/src/components/watchlist/WatchlistWorkspace.tsx b/apps/web/src/components/watchlist/WatchlistWorkspace.tsx index 00c121010..83d76cfba 100644 --- a/apps/web/src/components/watchlist/WatchlistWorkspace.tsx +++ b/apps/web/src/components/watchlist/WatchlistWorkspace.tsx @@ -35,6 +35,7 @@ import { authorizedJsonHeaders, jsonWriteHeaders, } from '@/lib/auth/access-token'; +import { productImageUrl } from '@/lib/rate-domain-core'; interface WatchlistWorkspaceProps { catalogProducts: MasterProduct[]; @@ -125,9 +126,7 @@ export default function WatchlistWorkspace({ const handleQuickAddCatalog = (p: MasterProduct) => { const minP = Number(p.min_price || 0); const targetP = computeWatchlistTargetPrice(minP, 5); - const thumb = - p.thumbnail || - (p.images && p.images.length > 0 ? p.images[0] || null : null); + const thumb = productImageUrl(p); const resolvedHref = getMasterProductHref(p); const resolvedCategoryPath = resolvedHref.replace(`/${p.slug}`, '') || '/smartphones'; @@ -685,11 +684,7 @@ export default function WatchlistWorkspace({
{catalogProducts.slice(0, 3).map((p) => { const minP = Number(p.min_price || 0); - const thumb = - p.thumbnail || - (p.images && p.images.length > 0 - ? p.images[0] - : '/img/placeholder.png'); + const thumb = productImageUrl(p) || '/img/placeholder.png'; return (
diff --git a/apps/web/src/lib/__tests__/image-url.test.ts b/apps/web/src/lib/__tests__/image-url.test.ts new file mode 100644 index 000000000..f0ce24a11 --- /dev/null +++ b/apps/web/src/lib/__tests__/image-url.test.ts @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict'; +import { + firstImageUrl, + isImageUrl, + productImageUrl, + resolveProductImageUrl, + usableImageUrls, +} from '../rate-domain-core'; + +function main() { + console.warn('🧪 Image URL suite'); + + assert.equal(isImageUrl('https://cdn.tgdd.vn/a.jpg'), true); + assert.equal(isImageUrl('http://example.com/a.png'), true); + assert.equal(isImageUrl('//cf.shopee.vn/file/abc'), true); + assert.equal(isImageUrl('/img/devices/iphone.jpg'), true); + assert.equal(isImageUrl('[object Object]'), false); + assert.equal(isImageUrl(''), false); + assert.equal(isImageUrl('abc.jpg'), false); + assert.equal(isImageUrl({ url: 'https://x/a.jpg' }), false); + assert.equal(isImageUrl(null), false); + + // TGDD rows store "[object Object]" before their real URLs. + assert.equal( + firstImageUrl([ + '[object Object]', + '[object Object]', + 'https://cdn.tgdd.vn/a.jpg', + ]), + 'https://cdn.tgdd.vn/a.jpg' + ); + assert.equal( + firstImageUrl(['https://a/1.jpg', 'https://a/2.jpg']), + 'https://a/1.jpg' + ); + assert.equal(firstImageUrl(['[object Object]']), null); + assert.equal(firstImageUrl([]), null); + assert.equal(firstImageUrl(null), null); + assert.equal(firstImageUrl(undefined), null); + + // A protocol-relative or absolute URL needs a host. + assert.equal(isImageUrl('//'), false); + assert.equal(isImageUrl('/'), false); + assert.equal(isImageUrl('https://'), false); + assert.equal(firstImageUrl(['//', '/img/good.jpg']), '/img/good.jpg'); + + // A junk thumbnail gives way to the first usable stored image. + assert.equal( + productImageUrl({ + thumbnail: '[object Object]', + images: ['[object Object]', 'https://a/1.jpg'], + }), + 'https://a/1.jpg' + ); + assert.equal( + productImageUrl({ thumbnail: '/t.jpg', images: ['https://a/1.jpg'] }), + '/t.jpg' + ); + assert.equal(productImageUrl({ thumbnail: null, images: null }), null); + + // Surrounding whitespace is trimmed rather than rejected. + assert.equal(isImageUrl(' https://a/1.jpg\n'), true); + assert.equal(firstImageUrl([' https://a/1.jpg ']), 'https://a/1.jpg'); + assert.deepEqual( + usableImageUrls(['[object Object]', 'https://a/1.jpg', '', '/b.png']), + ['https://a/1.jpg', '/b.png'] + ); + + // A junk thumbnail or fallback gives way to the stored images. + assert.equal( + resolveProductImageUrl({ + slug: 'e2e-not-local', + thumbnail: '[object Object]', + images: ['[object Object]', 'https://a/1.jpg'], + }), + 'https://a/1.jpg' + ); + assert.equal( + resolveProductImageUrl('e2e-not-local', '[object Object]'), + null + ); + + console.warn('✅ Image URL suite passed'); +} + +main(); diff --git a/apps/web/src/lib/hero-products.ts b/apps/web/src/lib/hero-products.ts index b05c3c795..2df1d80b5 100644 --- a/apps/web/src/lib/hero-products.ts +++ b/apps/web/src/lib/hero-products.ts @@ -1,4 +1,5 @@ import type { MasterProduct } from '@repo/shared-data'; +import { usableImageUrls } from './rate-domain-core'; /** * The product fields the hero search box shows, filters on and links with. @@ -77,7 +78,9 @@ export function toHeroProductSuggestions( variant_name: p.variant_name, key_specs: searchKeySpecs(p.key_specs), storage_gb: p.storage_gb, - images: p.images?.slice(0, 1) ?? p.images, + // The first usable image, not the first entry, which can be junk such as + // "[object Object]" (DOS/Rate#810). + images: usableImageUrls(p.images).slice(0, 1), thumbnail: p.thumbnail, min_price: p.min_price, merchant_count: p.merchant_count, diff --git a/apps/web/src/lib/rate-api.ts b/apps/web/src/lib/rate-api.ts index 2ebdfdf7f..ac0bab995 100644 --- a/apps/web/src/lib/rate-api.ts +++ b/apps/web/src/lib/rate-api.ts @@ -25,6 +25,7 @@ import { type ParsedEcommerceUrl, parseEcommerceUrlOrQuery, extractProductKeywordFromUrl, + firstImageUrl, } from './rate-domain-core'; export { computeWilsonLowerBound, @@ -2773,7 +2774,7 @@ export async function getMerchantProfileById( rating_star: Number(p.rating_star || 0), rating_count: Number(p.rating_count || 0), url: p.url, - image: p.images && p.images.length > 0 ? p.images[0] || null : null, + image: firstImageUrl(p.images), master_product: masterInfo, discount_analysis: discountByOfferId.get(p.id) || null, }; diff --git a/apps/web/src/lib/rate-domain-core.ts b/apps/web/src/lib/rate-domain-core.ts index c7ebfb613..d96c7daba 100644 --- a/apps/web/src/lib/rate-domain-core.ts +++ b/apps/web/src/lib/rate-domain-core.ts @@ -837,6 +837,42 @@ export const KNOWN_LOCAL_DEVICE_IMAGES = new Set([ 'xiaomi-pad-7-pro-256gb', ]); +/** + * Whether a stored image entry can be used as an `src`: an absolute + * http(s) URL, a protocol-relative URL or a site path. Crawled rows can hold + * other strings, such as "[object Object]" from a stringified image object + * (DOS/Rate#810). + */ +export function isImageUrl(value: unknown): value is string { + // A host after "http(s)://" or "//"; otherwise one slash and a path. + return ( + typeof value === 'string' && + /^(?:(?:https?:)?\/\/[^\s/?#]+|\/(?!\/)\S)\S*$/i.test(value.trim()) + ); +} + +/** The usable images of a stored `images` list, trimmed, in order. */ +export function usableImageUrls( + images: readonly unknown[] | null | undefined +): string[] { + return (images ?? []).filter(isImageUrl).map((url) => url.trim()); +} + +/** The first usable image of a stored `images` list, or null. */ +export function firstImageUrl( + images: readonly unknown[] | null | undefined +): string | null { + return usableImageUrls(images)[0] ?? null; +} + +/** A product's thumbnail if usable, else its first usable stored image. */ +export function productImageUrl(product: { + readonly thumbnail?: string | null; + readonly images?: readonly unknown[] | null; +}): string | null { + return firstImageUrl([product.thumbnail, ...(product.images ?? [])]); +} + export function resolveProductImageUrl( slugOrProduct?: | string @@ -854,19 +890,20 @@ export function resolveProductImageUrl( if (slug && KNOWN_LOCAL_DEVICE_IMAGES.has(slug)) { return `/img/devices/${slug}.jpg`; } + // Stored thumbnails and listing images can be junk such as + // "[object Object]" (DOS/Rate#810). return ( - slugOrProduct.thumbnail || - (Array.isArray(slugOrProduct.images) && slugOrProduct.images.length > 0 - ? slugOrProduct.images[0] || null - : null) || - fallbackUrl || - null + firstImageUrl([slugOrProduct.thumbnail]) || + firstImageUrl( + Array.isArray(slugOrProduct.images) ? slugOrProduct.images : null + ) || + firstImageUrl([fallbackUrl]) ); } if (slugOrProduct && KNOWN_LOCAL_DEVICE_IMAGES.has(slugOrProduct)) { return `/img/devices/${slugOrProduct}.jpg`; } - return fallbackUrl || null; + return firstImageUrl([fallbackUrl]); } export type CanonicalCategorySlug =