From f7973908e27b6af2474d7cb3bb1fb2d9bdd6858f Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:49:43 +0700 Subject: [PATCH 01/19] fix(importer): harden AiReviewAnalyzer config, summary, and vote bounds (#834) (#835) Co-authored-by: JOY --- .../importer/src/shopee/ai-review-analyzer.ts | 88 ++++++++++++++----- .../src/shopee/test-review-analyzer.ts | 47 +++++++++- 2 files changed, 114 insertions(+), 21 deletions(-) diff --git a/apps/importer/src/shopee/ai-review-analyzer.ts b/apps/importer/src/shopee/ai-review-analyzer.ts index fba34ec8e..8a8bb66ee 100644 --- a/apps/importer/src/shopee/ai-review-analyzer.ts +++ b/apps/importer/src/shopee/ai-review-analyzer.ts @@ -1006,19 +1006,33 @@ export class AiReviewAnalyzer { this.provider = this.resolveProvider(config.provider); const defaults = PROVIDER_DEFAULTS[this.provider]; - this.apiKey = config.apiKey ?? this.resolveApiKey(this.provider); - this.baseUrl = trimTrailingSlashes( + this.apiKey = (config.apiKey ?? this.resolveApiKey(this.provider)).trim(); + const rawBaseUrl = trimTrailingSlashes( config.baseUrl || process.env.AI_BASE_URL || process.env.DOS_AI_BASE_URL || process.env.OPENAI_BASE_URL || defaults.baseUrl ); + try { + const parsedBase = new URL(rawBaseUrl); + this.baseUrl = + parsedBase.protocol === 'https:' || parsedBase.protocol === 'http:' + ? rawBaseUrl + : defaults.baseUrl; + } catch { + this.baseUrl = defaults.baseUrl; + } this.model = - config.model || process.env.AI_MODEL || process.env.OPENAI_MODEL || defaults.defaultModel; - - this.timeoutMs = config.timeoutMs ?? Number(process.env.AI_TIMEOUT_MS || 25000); + (config.model || process.env.AI_MODEL || process.env.OPENAI_MODEL || '').trim() || + defaults.defaultModel; + + const rawTimeout = config.timeoutMs ?? Number(process.env.AI_TIMEOUT_MS || 25000); + this.timeoutMs = + Number.isFinite(rawTimeout) && rawTimeout > 0 + ? Math.min(Math.max(Math.round(rawTimeout), 1000), 120000) + : 25000; } private resolveProvider(explicit?: SupportedAiProvider): SupportedAiProvider { @@ -1806,6 +1820,14 @@ Return ONLY valid JSON matching: ? `Đã phân tích ${reviews.length} đánh giá: giữ lại ${authenticCount} đánh giá thực chất (điểm JReviews có trọng số: ${weightedUserRating}/5★) và loại bỏ ${ignoredCount} đánh giá rác/nhận xu/seeding/bom hàng khỏi điểm tổng.` : `Chưa có đánh giá thực chất sau khi lọc ${ignoredCount} bình luận rác/nhận xu.`; + const cleanPros = Array.from( + new Set(aiPros.map(p => sanitizeAiGeneratedText(p, 160)).filter(Boolean)) + ).slice(0, 6); + const cleanCons = Array.from( + new Set(aiCons.map(c => sanitizeAiGeneratedText(c, 160)).filter(Boolean)) + ).slice(0, 6); + const cleanSummaryText = sanitizeAiGeneratedText(aiSummaryText, 600); + return { total_reviews_analyzed: reviews.length, authentic_review_count: authenticCount, @@ -1817,13 +1839,13 @@ Return ONLY valid JSON matching: authenticity_score: authenticityScore, sentiment_positive_ratio: sentimentPositiveRatio, pros: - aiPros.length > 0 - ? aiPros + cleanPros.length > 0 + ? cleanPros : positiveAuthenticCount > 0 ? ['Nhận phản hồi tích cực từ người dùng thực tế'] : [], - cons: aiCons.length > 0 ? aiCons : [], - ai_summary: aiSummaryText || defaultSummary, + cons: cleanCons, + ai_summary: cleanSummaryText || defaultSummary, breakdown_by_category: breakdown, bayesian_rating: bayesianRating, }; @@ -1990,13 +2012,29 @@ Return ONLY valid JSON matching: `[AiReviewAnalyzer] Failed to read existing review_flags: ${error.message}` ); } + const validVerdicts = new Set([ + 'authentic', + 'suspicious', + 'fake', + 'unanalyzed', + ]); + const validCategories = new Set([ + 'authentic', + 'low_value_coin_spam', + 'seeding_fake', + 'competitor_sabotage', + 'review_bombing', + ]); for (const row of (data || []) as Array>) { - if (!row.review_id) continue; - preserved.set(String(row.review_id), { - verdict: row.verdict as ReviewVerdict, - flag_category: row.flag_category as ReviewFlagCategory, + const revId = typeof row.review_id === 'string' ? row.review_id.trim() : ''; + if (!revId) continue; + const rawVerdict = row.verdict as ReviewVerdict; + const rawFlagCat = row.flag_category as ReviewFlagCategory; + preserved.set(revId, { + verdict: validVerdicts.has(rawVerdict) ? rawVerdict : 'fake', + flag_category: validCategories.has(rawFlagCat) ? rawFlagCat : 'seeding_fake', include_in_rating: Boolean(row.include_in_rating), - fake_probability: Number(row.fake_probability ?? 0), + fake_probability: clamp01(Number(row.fake_probability ?? 0)), }); } } @@ -2237,16 +2275,26 @@ Return ONLY valid JSON matching: reportReason?: 'spam' | 'fake' | 'offensive' | 'off_topic' | 'other' | null; } ): Promise<{ action: 'upserted' | 'deleted' }> { - const vote = params.vote ?? null; + const reviewId = (params.reviewId || '').trim(); + const voterId = (params.voterId || '').trim(); + if (!reviewId || !voterId) { + throw new Error('[AiReviewAnalyzer] reviewId and voterId must be non-empty strings'); + } + const vote = params.vote === 'up' || params.vote === 'down' ? params.vote : null; const reported = Boolean(params.reported); - const reportReason = reported ? (params.reportReason ?? 'other') : null; + const validReasons = new Set(['spam', 'fake', 'offensive', 'off_topic', 'other']); + const reportReason = reported + ? params.reportReason && validReasons.has(params.reportReason) + ? params.reportReason + : 'other' + : null; if (!vote && !reported) { const { error } = await supabase .from('review_votes') .delete() - .eq('review_id', params.reviewId) - .eq('voter_id', params.voterId); + .eq('review_id', reviewId) + .eq('voter_id', voterId); if (error) { throw new Error(`[AiReviewAnalyzer] Failed to delete review_vote: ${error.message}`); } @@ -2255,8 +2303,8 @@ Return ONLY valid JSON matching: const { error } = await supabase.from('review_votes').upsert( { - review_id: params.reviewId, - voter_id: params.voterId, + review_id: reviewId, + voter_id: voterId, vote, reported, report_reason: reportReason, diff --git a/apps/importer/src/shopee/test-review-analyzer.ts b/apps/importer/src/shopee/test-review-analyzer.ts index 5bb1d0c48..cfb9e6d04 100644 --- a/apps/importer/src/shopee/test-review-analyzer.ts +++ b/apps/importer/src/shopee/test-review-analyzer.ts @@ -2255,8 +2255,53 @@ async function run() { 'ingestProduct must skip Step 6 review re-analysis when pricesOnly/skipReviewAnalysis is set' ); + // 16k. DOS/Rate#834: AiReviewAnalyzer config bounds, computeJReviewsSummary sanitization, and castReviewVote validation + const analyzer834 = new AiReviewAnalyzer({ + provider: 'openai', + baseUrl: 'javascript:alert(1)', + timeoutMs: -500, + }); + const analyzer834Internal = analyzer834 as unknown as { baseUrl: string; timeoutMs: number }; + assert.equal( + analyzer834Internal.baseUrl, + 'https://api.openai.com/v1', + 'AiReviewAnalyzer must fall back to provider default baseUrl when baseUrl is non-HTTP(S)' + ); + assert.equal( + analyzer834Internal.timeoutMs, + 25000, + 'AiReviewAnalyzer must fall back to 25000ms when timeoutMs is negative or non-finite' + ); + + const summary834 = analyzer834.computeJReviewsSummary( + [{ id: 'r-834', rating_star: 5, comment: 'Máy rất tốt' }], + analyzer834.analyzeWithHeuristics([{ id: 'r-834', rating_star: 5, comment: 'Máy rất tốt' }]), + null, + ['Pin trâu', 'Pin trâu', ' '], + [''], + '

Tổng kết đánh giá

' + ); + assert.deepEqual( + summary834.pros, + ['Pin trâu'], + 'computeJReviewsSummary must strip HTML tags and deduplicate pros' + ); + assert.deepEqual(summary834.cons, ['Nóng máy']); + assert.equal(summary834.ai_summary, 'Tổng kết đánh giá'); + + await assert.rejects( + () => + analyzer834.castReviewVote(pricesOnlyStub!, { + reviewId: ' ', + voterId: 'voter-1', + vote: 'up', + }), + /reviewId and voterId must be non-empty strings/, + 'castReviewVote must reject blank reviewId or voterId' + ); + console.log( - `✅ [P2/P3 Trust Model & #292/#352/#356/#359/#379/#382/#401/#497/#514/#522/#528/#557] Voter weight, Wilson lower bound (50/52=${wilson50of52} > 1/1=${wilson1of1}), shingle copy seeding (sim=${shingleSim}), affiliate shortlinks, peer floor, ArenaSpecSyncer, Importer & ReviewAnalyzer hardening (#352, #356, #359, #379, #382, #401, #497, #514, #522, #528, #557) verified` + `✅ [P2/P3 Trust Model & #292/#352/#356/#359/#379/#382/#401/#497/#514/#522/#528/#557/#834] Voter weight, Wilson lower bound (50/52=${wilson50of52} > 1/1=${wilson1of1}), shingle copy seeding (sim=${shingleSim}), affiliate shortlinks, peer floor, ArenaSpecSyncer, Importer & ReviewAnalyzer hardening (#352, #356, #359, #379, #382, #401, #497, #514, #522, #528, #557, #834) verified` ); console.log('\n🎉 ALL AI REVIEW ANALYZER TESTS PASSED!'); From ad89f35c4b95a1094dbb7eae4551ed04ede43487 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:14:11 +0700 Subject: [PATCH 02/19] fix(importer): enforce integer RAM/storage bounds and asset checks (#836) (#837) Co-authored-by: JOY --- .../importer/src/shopee/product-normalizer.ts | 77 ++++++++++++--- .../src/shopee/test-review-analyzer.ts | 98 ++++++++++++++++++- 2 files changed, 162 insertions(+), 13 deletions(-) diff --git a/apps/importer/src/shopee/product-normalizer.ts b/apps/importer/src/shopee/product-normalizer.ts index 84cb012a4..6a78bd725 100644 --- a/apps/importer/src/shopee/product-normalizer.ts +++ b/apps/importer/src/shopee/product-normalizer.ts @@ -3,6 +3,42 @@ import type { ShopeeProduct } from './types.js'; export const MAX_VALID_RAM_GB = 2048; export const MAX_VALID_STORAGE_GB = 65536; +export const MAX_NORMALIZED_COLOR_LENGTH = 80; +export const MAX_NORMALIZED_COLOR_COUNT = 24; +export const MAX_NORMALIZED_IMAGE_URL_LENGTH = 2048; +export const MAX_NORMALIZED_IMAGE_COUNT = 50; +export const MAX_NORMALIZED_DESCRIPTION_LENGTH = 8000; + +function stripControlChars(input: string): string { + let out = ''; + for (let i = 0; i < input.length; i++) { + const code = input.charCodeAt(i); + if (code === 0x09 || code === 0x0a || code === 0x0d || (code > 0x1f && code !== 0x7f)) { + out += input[i]; + } else { + out += ' '; + } + } + return out; +} + +function normalizeHttpAssetUrl(raw: unknown): string | null { + if (typeof raw !== 'string') return null; + const trimmed = raw.trim(); + if (!trimmed || trimmed.length > MAX_NORMALIZED_IMAGE_URL_LENGTH) return null; + for (let i = 0; i < trimmed.length; i++) { + const code = trimmed.charCodeAt(i); + if (code <= 0x20 || code === 0x7f) return null; + } + try { + const parsed = new URL(trimmed); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null; + if (!parsed.hostname) return null; + return parsed.toString(); + } catch { + return null; + } +} export type CanonicalCategorySlug = 'smartphones' | 'tablets' | 'laptops' | 'smartwatches' | 'earbuds'; @@ -1373,14 +1409,16 @@ export class ProductNormalizer { } const storageCandidates = claimable.filter( - t => !t.claimed && t.value > 0 && t.value <= MAX_VALID_STORAGE_GB + t => !t.claimed && Number.isInteger(t.value) && t.value > 0 && t.value <= MAX_VALID_STORAGE_GB ); const baseKey = `${slugify(brand)}:${slugify(model)}`; const benchmark = CANONICAL_PHONE_REGISTRY[baseKey]; const toValidStorageGb = (val: number): number | null => - Number.isFinite(val) && val > 0 && val <= MAX_VALID_STORAGE_GB ? val : null; + Number.isFinite(val) && Number.isInteger(val) && val > 0 && val <= MAX_VALID_STORAGE_GB + ? val + : null; let storageGb: number | null = null; let storageConfident = false; @@ -1419,9 +1457,17 @@ export class ProductNormalizer { const colors: string[] = []; if (Array.isArray(raw.models)) { for (const m of raw.models) { - const trimmedColor = typeof m?.name === 'string' ? m.name.trim() : ''; - if (trimmedColor.length > 0 && !colors.includes(trimmedColor)) { - colors.push(trimmedColor); + if (colors.length >= MAX_NORMALIZED_COLOR_COUNT) break; + const normalizedColor = + typeof m?.name === 'string' + ? stripControlChars(m.name) + .replace(/\s+/g, ' ') + .trim() + .slice(0, MAX_NORMALIZED_COLOR_LENGTH) + .trim() + : ''; + if (normalizedColor.length > 0 && !colors.includes(normalizedColor)) { + colors.push(normalizedColor); } } } @@ -1429,9 +1475,10 @@ export class ProductNormalizer { const images: string[] = []; if (Array.isArray(raw.images)) { for (const img of raw.images) { - const trimmedImg = typeof img === 'string' ? img.trim() : ''; - if (trimmedImg.length > 0 && !images.includes(trimmedImg)) { - images.push(trimmedImg); + if (images.length >= MAX_NORMALIZED_IMAGE_COUNT) break; + const normalizedImg = normalizeHttpAssetUrl(img); + if (normalizedImg && !images.includes(normalizedImg)) { + images.push(normalizedImg); } } } @@ -1444,7 +1491,7 @@ export class ProductNormalizer { ? 'Standard' : storageGb === null ? 'Không rõ dung lượng' - : storageGb >= 1024 + : storageGb >= 1024 && storageGb % 512 === 0 ? `${storageGb / 1024}TB` : `${storageGb}GB`; const canonicalName = @@ -1476,6 +1523,7 @@ export class ProductNormalizer { const validClaimedRam = claimedRamValue !== null && Number.isFinite(claimedRamValue) && + Number.isInteger(claimedRamValue) && claimedRamValue > 0 && claimedRamValue <= MAX_VALID_RAM_GB ? claimedRamValue @@ -1792,8 +1840,12 @@ export class ProductNormalizer { ram: ramGb ? `${ramGb}GB` : undefined, }; - const condition = detectProductCondition(title, raw.description); - const origin = detectProductOrigin(title, raw.description); + const sanitizedDescription = + typeof raw.description === 'string' + ? stripControlChars(raw.description).trim().slice(0, MAX_NORMALIZED_DESCRIPTION_LENGTH) + : undefined; + const condition = detectProductCondition(title, sanitizedDescription); + const origin = detectProductOrigin(title, sanitizedDescription); return { category_slug: categorySlug, @@ -1815,7 +1867,8 @@ export class ProductNormalizer { images, specifications, key_specs: keySpecs, - description: raw.description, + description: + sanitizedDescription && sanitizedDescription.length > 0 ? sanitizedDescription : undefined, is_benchmark_enriched: Boolean(benchmark), spec_source: benchmark ? 'benchmark_registry' : 'crawled_title', }; diff --git a/apps/importer/src/shopee/test-review-analyzer.ts b/apps/importer/src/shopee/test-review-analyzer.ts index cfb9e6d04..0159274a3 100644 --- a/apps/importer/src/shopee/test-review-analyzer.ts +++ b/apps/importer/src/shopee/test-review-analyzer.ts @@ -2300,8 +2300,104 @@ async function run() { 'castReviewVote must reject blank reviewId or voterId' ); + // 16l. DOS/Rate#836: ProductNormalizer integer RAM/storage bounds, TB formatting, and asset/color sanitization + const normalizer836 = new ProductNormalizer(); + const baseRaw836 = { + platform: 'shopee' as const, + itemId: 'item-836', + shopId: 'shop-836', + url: 'https://shopee.vn/product/836/836', + priceMin: 12_990_000, + priceMax: 12_990_000, + currency: 'VND', + historicalSold: 10, + ratingStar: 4.8, + ratingCount: 5, + attributes: [], + shop: { shopId: 'shop-836', name: 'Shop 836', isMall: false }, + reviews: [], + }; + const fractionalRom836 = normalizer836.normalize({ + ...baseRaw836, + title: 'Điện thoại Samsung Galaxy S24 FE 8.5GB RAM ROM 12.5GB', + images: [ + 'https://cf.shopee.vn/file/valid.jpg', + 'javascript:alert(1)', + 'ftp://files.example.com/img.jpg', + ' ', + ], + models: [ + { modelId: 'm-1', name: ' Xanh Mint\u0007 ', price: 12_990_000, stock: 5 }, + { modelId: 'm-2', name: 'Xanh Mint', price: 12_990_000, stock: 3 }, + { modelId: 'm-3', name: 'Đ'.repeat(120), price: 12_990_000, stock: 2 }, + ], + description: ' Máy chính hãng\u0000 bảo hành 12 tháng ', + }); + assert.equal( + fractionalRom836.storage_gb, + null, + 'ProductNormalizer must reject non-integer GB storage values like 12.5GB' + ); + assert.equal( + fractionalRom836.ram_gb, + null, + 'ProductNormalizer must reject non-integer GB RAM values like 8.5GB' + ); + assert.deepEqual( + fractionalRom836.images, + ['https://cf.shopee.vn/file/valid.jpg'], + 'ProductNormalizer must keep only valid http/https asset URLs in images' + ); + assert.equal( + fractionalRom836.colors[0], + 'Xanh Mint', + 'ProductNormalizer must strip control chars, collapse whitespace, and deduplicate colors' + ); + assert.equal( + fractionalRom836.colors.length, + 2, + 'ProductNormalizer must deduplicate collapsed color names' + ); + assert.equal( + fractionalRom836.colors[1]?.length, + 80, + 'ProductNormalizer must cap color name length at 80 characters' + ); + assert.equal( + fractionalRom836.description, + 'Máy chính hãng bảo hành 12 tháng', + 'ProductNormalizer must strip control characters and trim description' + ); + + const nonMultipleTb836 = normalizer836.normalize({ + ...baseRaw836, + title: 'Laptop Asus Vivobook 15 16GB RAM ROM 1280GB', + images: [], + models: [], + }); + assert.equal(nonMultipleTb836.storage_gb, 1280); + assert.equal( + nonMultipleTb836.variant_name, + '1280GB', + 'ProductNormalizer must format non-512-multiple storage >= 1024 as GB rather than fractional TB' + ); + assert.equal(nonMultipleTb836.key_specs.storage, '1280GB'); + + const binaryTb836 = normalizer836.normalize({ + ...baseRaw836, + title: 'Laptop Asus Vivobook 15 16GB RAM 1.5TB SSD', + images: [], + models: [], + }); + assert.equal(binaryTb836.storage_gb, 1536); + assert.equal( + binaryTb836.variant_name, + '1.5TB', + 'ProductNormalizer must preserve 1.5TB (1536GB) as 1.5TB' + ); + console.log( - `✅ [P2/P3 Trust Model & #292/#352/#356/#359/#379/#382/#401/#497/#514/#522/#528/#557/#834] Voter weight, Wilson lower bound (50/52=${wilson50of52} > 1/1=${wilson1of1}), shingle copy seeding (sim=${shingleSim}), affiliate shortlinks, peer floor, ArenaSpecSyncer, Importer & ReviewAnalyzer hardening (#352, #356, #359, #379, #382, #401, #497, #514, #522, #528, #557, #834) verified` + `✅ [P2/P3 Trust Model & #292/#352/#356/#359/#379/#382/#401/#497/#514/#522/#528/#557/#834/#836] Voter weight, Wilson lower bound (50/52=${wilson50of52} > 1/1=${wilson1of1}), shingle copy seeding (sim=${shingleSim}), affiliate shortlinks, peer floor, ArenaSpecSyncer, Importer & ReviewAnalyzer hardening (#352, #356, #359, #379, #382, #401, #497, #514, #522, #528, #557, #834, #836) verified` ); console.log('\n🎉 ALL AI REVIEW ANALYZER TESTS PASSED!'); From f1a13528c0b4bdd9b859c0bcec7b96240d74d779 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 17:49:05 +0700 Subject: [PATCH 03/19] fix(importer): clamp fetchAllPages bounds and harden merchant risk (#838) (#839) * fix(importer): clamp fetchAllPages bounds and harden merchant risk (#838) * fix(importer): require pageSize and maxPages >= 1 in fetchAllPages (#838) --------- Co-authored-by: JOY --- .../__tests__/backfill-review-risk.test.ts | 57 ++++++++++++++++++ .../src/shopee/__tests__/paging.test.ts | 32 ++++++++++ .../src/shopee/jobs/backfill-specs.ts | 59 ++++++++++++++++--- .../src/shopee/merchant-review-risk.ts | 54 +++++++++++++---- apps/importer/src/shopee/paging.ts | 30 +++++++--- 5 files changed, 206 insertions(+), 26 deletions(-) diff --git a/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts b/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts index 154805141..6afbfc61f 100644 --- a/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts +++ b/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts @@ -3,6 +3,7 @@ import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { AiReviewAnalyzer } from '../ai-review-analyzer.js'; import { loadAndScoreMerchant, rescoreMerchantReviewRisk } from '../merchant-review-risk.js'; +import { mergeKeySpecs } from '../jobs/backfill-specs.js'; import { mapStoredReview, runReviewRiskBackfill, @@ -392,6 +393,62 @@ async function run() { 'ingestion awaits the seller rescore after persistAnalysisToSupabase' ); + // DOS/Rate#838: mapStoredReview vocabulary validation, follower_count ceiling, and mergeKeySpecs sanitization + const badVocab = mapStoredReview({ + id: 'r-838', + product_id: 'p-1', + rating_star: -10, + comment: 'Tốt', + review_flags: { verdict: 'unknown_verdict', flag_category: 'unknown_category' }, + })!; + assert.equal(badVocab.result.verdict, 'authentic'); + assert.equal(badVocab.result.flag_category, 'authentic'); + assert.equal(badVocab.raw.rating_star, 5); + + const overflowMerchants: MerchantRow[] = [ + { + id: 'm-overflow', + platform: 'shopee', + is_official: false, + follower_count: 999_999_999, + review_risk_score: 0, + }, + ]; + const overflowScores = scoreMerchants( + new AiReviewAnalyzer(), + overflowMerchants, + [{ id: 'p-ov', merchant_id: 'm-overflow', cluster_id: 'c1', price_current: 10_000_000 }], + [review('r-ov', 'p-ov', { verdict: 'authentic', flag_category: 'authentic' })], + new Map() + ); + assert.equal( + overflowScores[0]?.risk.risk_score, + 15, + 'scoreMerchants must ignore overflowed follower_count > 100M rather than granting the 50k+ follower discount' + ); + + const mergedSpecs = mergeKeySpecs( + { + screen: ' 6.9"\u0007 OLED ', + price_spread_audit: { + verdict: ' tight_market_consensus ', + spread_vnd: Number.NaN, + platform_count: 3, + }, + }, + { + chip: 'A'.repeat(600), + ram: ' ', + } + ); + assert.equal(mergedSpecs.screen, '6.9" OLED'); + assert.equal((mergedSpecs.chip as string).length, 500); + assert.equal('ram' in mergedSpecs, false); + assert.deepEqual(mergedSpecs.price_spread_audit, { + verdict: 'tight_market_consensus', + platform_count: 3, + }); + console.log('✅ Review risk backfill suite passed'); } diff --git a/apps/importer/src/shopee/__tests__/paging.test.ts b/apps/importer/src/shopee/__tests__/paging.test.ts index 35958e02e..88533e470 100644 --- a/apps/importer/src/shopee/__tests__/paging.test.ts +++ b/apps/importer/src/shopee/__tests__/paging.test.ts @@ -77,6 +77,38 @@ async function run() { null, 'must reject non-HTTP(S) scheme' ); + assert.equal( + parseShopeeEnqueueUrl('https://user:pass@shopee.vn/product/88201679/27110898667'), + null, + 'must reject URLs with embedded userinfo credentials (#838)' + ); + assert.equal( + parseShopeeEnqueueUrl('https://shopee.vn/product/0/27110898667'), + null, + 'must reject zero shopId (#838)' + ); + assert.equal( + parseShopeeEnqueueUrl('https://shopee.vn/product/88201679/0'), + null, + 'must reject zero itemId (#838)' + ); + assert.equal( + parseShopeeEnqueueUrl('https://shopee.vn/product/88201679/123456789012345678901'), + null, + 'must reject oversized >20-digit itemId (#838)' + ); + + // Oversized pageSize is clamped to 1000 (#838) + const clampedCalls: Array<[number, number]> = []; + await fetchAllPages( + 'clamped', + (from, to) => { + clampedCalls.push([from, to]); + return Promise.resolve({ data: [], error: null }); + }, + 999_999 + ); + assert.deepEqual(clampedCalls, [[0, 999]], 'fetchAllPages must clamp pageSize at 1000'); console.log('✅ fetchAllPages & parseShopeeEnqueueUrl suite passed'); } diff --git a/apps/importer/src/shopee/jobs/backfill-specs.ts b/apps/importer/src/shopee/jobs/backfill-specs.ts index 7ffb48d5d..942fc1cab 100644 --- a/apps/importer/src/shopee/jobs/backfill-specs.ts +++ b/apps/importer/src/shopee/jobs/backfill-specs.ts @@ -98,6 +98,54 @@ export async function auditMasterSpecs(supabase: RateSupabaseClient): Promise 0x1f && code !== 0x7f) { + out += input[i]; + } else { + out += ' '; + } + } + return out; +} + +function sanitizeKeySpecValue(value: unknown, depth = 0): unknown { + if (depth > 4 || value === undefined || value === null) return undefined; + if (typeof value === 'boolean') return value; + if (typeof value === 'number') { + return Number.isFinite(value) ? value : undefined; + } + if (typeof value === 'string') { + const cleaned = stripKeySpecControlChars(value) + .replace(/\s+/g, ' ') + .trim() + .slice(0, MAX_KEY_SPEC_STRING_LENGTH) + .trim(); + return cleaned.length > 0 ? cleaned : undefined; + } + if (Array.isArray(value)) { + return value + .slice(0, 50) + .map(item => sanitizeKeySpecValue(item, depth + 1)) + .filter(item => item !== undefined); + } + if (typeof value === 'object') { + const objOut: Record = {}; + for (const [k, v] of Object.entries(value as Record)) { + const sanitized = sanitizeKeySpecValue(v, depth + 1); + if (sanitized !== undefined) { + objOut[k] = sanitized; + } + } + return objOut; + } + return undefined; +} + export function mergeKeySpecs( existing: unknown, incoming: Record | null | undefined @@ -110,15 +158,10 @@ export function mergeKeySpecs( const out: Record = {}; for (const source of [base, next]) { for (const [k, v] of Object.entries(source)) { - if (v === undefined || v === null) continue; - if (typeof v === 'number' && !Number.isFinite(v)) continue; - if (typeof v === 'string') { - const trimmed = v.trim(); - if (!trimmed) continue; - out[k] = trimmed; - continue; + const sanitized = sanitizeKeySpecValue(v, 0); + if (sanitized !== undefined) { + out[k] = sanitized; } - out[k] = v; } } return out; diff --git a/apps/importer/src/shopee/merchant-review-risk.ts b/apps/importer/src/shopee/merchant-review-risk.ts index 0db309d2d..0bbe68ee5 100644 --- a/apps/importer/src/shopee/merchant-review-risk.ts +++ b/apps/importer/src/shopee/merchant-review-risk.ts @@ -25,6 +25,20 @@ export const MERCHANT_RISK_SELECT = 'id, platform, is_official, follower_count, export const PRODUCT_RISK_SELECT = 'id, merchant_id, cluster_id, price_current'; export const REVIEW_RISK_SELECT = 'id, product_id, rating_star, comment, review_flags(verdict, flag_category, include_in_rating)'; +export const MAX_MERCHANT_FOLLOWER_COUNT = 100_000_000; + +const VALID_STORED_VERDICTS = new Set([ + 'authentic', + 'suspicious', + 'fake', +]); +const VALID_STORED_FLAG_CATEGORIES = new Set([ + 'authentic', + 'low_value_coin_spam', + 'seeding_fake', + 'competitor_sabotage', + 'review_bombing', +]); export interface MerchantRow { id: string; @@ -65,15 +79,23 @@ export function mapStoredReview( row: ReviewRow ): { result: AnalyzedReviewResult; raw: ReviewInputItem } | null { const vf = Array.isArray(row.review_flags) ? row.review_flags[0] : row.review_flags; - const verdict = vf?.verdict ?? null; - if (verdict === 'unanalyzed') return null; - let category = - vf?.flag_category || - (verdict === 'fake' + const rawVerdict = typeof vf?.verdict === 'string' ? vf.verdict.trim().toLowerCase() : null; + if (rawVerdict === 'unanalyzed') return null; + const verdict: AnalyzedReviewResult['verdict'] = + rawVerdict && VALID_STORED_VERDICTS.has(rawVerdict as AnalyzedReviewResult['verdict']) + ? (rawVerdict as AnalyzedReviewResult['verdict']) + : 'authentic'; + const rawCategory = + typeof vf?.flag_category === 'string' ? vf.flag_category.trim().toLowerCase() : ''; + let category: AnalyzedReviewResult['flag_category'] = VALID_STORED_FLAG_CATEGORIES.has( + rawCategory as AnalyzedReviewResult['flag_category'] + ) + ? (rawCategory as AnalyzedReviewResult['flag_category']) + : verdict === 'fake' ? 'seeding_fake' : verdict === 'suspicious' ? 'low_value_coin_spam' - : 'authentic'); + : 'authentic'; // The web counts a 'fake' verdict as seeding unless the category says // sabotage or bombing (DOS/Rate#645). if (verdict === 'fake' && category !== 'competitor_sabotage' && category !== 'review_bombing') { @@ -82,11 +104,14 @@ export function mapStoredReview( const includeInRating = Boolean( vf?.include_in_rating ?? (category === 'authentic' && verdict !== 'fake') ); + const parsedStar = Number(row.rating_star); + const ratingStar = + Number.isFinite(parsedStar) && parsedStar >= 1 && parsedStar <= 5 ? Math.round(parsedStar) : 5; return { result: { review_id: row.id, - verdict: (verdict ?? 'authentic') as AnalyzedReviewResult['verdict'], - flag_category: category as AnalyzedReviewResult['flag_category'], + verdict, + flag_category: category, fake_probability: 0, include_in_rating: includeInRating, helpfulness_weight: includeInRating ? 1 : 0, @@ -95,8 +120,8 @@ export function mapStoredReview( }, raw: { id: row.id, - rating_star: Number(row.rating_star || 5), - comment: row.comment || '', + rating_star: ratingStar, + comment: typeof row.comment === 'string' ? row.comment.slice(0, 4000) : '', }, }; } @@ -200,11 +225,18 @@ export function scoreMerchants( officialByMerchant, masterMinByCluster ); + const rawFollowers = Number(merchant.follower_count); + const safeFollowerCount = + Number.isFinite(rawFollowers) && + rawFollowers >= 0 && + rawFollowers <= MAX_MERCHANT_FOLLOWER_COUNT + ? Math.floor(rawFollowers) + : 0; const risk = analyzer.evaluateMerchantRisk({ merchantId, platform: merchant.platform, isOfficial: Boolean(merchant.is_official), - followerCount: Number(merchant.follower_count) || 0, + followerCount: safeFollowerCount, listingPrice: anomaly?.listingPrice, canonicalMinPrice: anomaly?.canonicalMinPrice, reviewResults: bucket.results, diff --git a/apps/importer/src/shopee/paging.ts b/apps/importer/src/shopee/paging.ts index fad9ab04c..db8746942 100644 --- a/apps/importer/src/shopee/paging.ts +++ b/apps/importer/src/shopee/paging.ts @@ -1,4 +1,7 @@ export const DEFAULT_FETCH_ALL_MAX_PAGES = 500; +export const MAX_FETCH_ALL_PAGE_SIZE = 1000; +export const MAX_FETCH_ALL_PAGES_CEILING = 10_000; +export const MAX_SHOPEE_ENQUEUE_URL_LENGTH = 2048; /** * Reads every row of a query in pages. One PostgREST response stops at the @@ -15,12 +18,17 @@ export async function fetchAllPages( from: number, to: number ) => PromiseLike<{ data: T[] | null; error: { message: string } | null }>, - pageSize = 1000, + pageSize = MAX_FETCH_ALL_PAGE_SIZE, maxPages = DEFAULT_FETCH_ALL_MAX_PAGES ): Promise { - const safePageSize = Number.isFinite(pageSize) && pageSize > 0 ? Math.floor(pageSize) : 1000; + const safePageSize = + Number.isFinite(pageSize) && pageSize >= 1 + ? Math.min(MAX_FETCH_ALL_PAGE_SIZE, Math.floor(pageSize)) + : MAX_FETCH_ALL_PAGE_SIZE; const safeMaxPages = - Number.isFinite(maxPages) && maxPages > 0 ? Math.floor(maxPages) : DEFAULT_FETCH_ALL_MAX_PAGES; + Number.isFinite(maxPages) && maxPages >= 1 + ? Math.min(MAX_FETCH_ALL_PAGES_CEILING, Math.floor(maxPages)) + : DEFAULT_FETCH_ALL_MAX_PAGES; const rows: T[] = []; let nonEmptyPages = 0; for (let from = 0; ;) { @@ -40,14 +48,19 @@ export async function fetchAllPages( /** * Validates and extracts `{ shopId, itemId, normalizedUrl }` from a Shopee Vietnam * product URL (`https://shopee.vn/...-i..`, `/product//`, - * or `/opaanlp//`). Rejects non-HTTP(S) schemes and non-Shopee hosts. + * or `/opaanlp//`). Rejects non-HTTP(S) schemes, credentials, control + * characters, and non-Shopee hosts. */ export function parseShopeeEnqueueUrl( rawUrl: string ): { shopId: string; itemId: string; normalizedUrl: string } | null { if (typeof rawUrl !== 'string') return null; const trimmed = rawUrl.trim(); - if (!trimmed) return null; + if (!trimmed || trimmed.length > MAX_SHOPEE_ENQUEUE_URL_LENGTH) return null; + for (let i = 0; i < trimmed.length; i++) { + const code = trimmed.charCodeAt(i); + if (code <= 0x20 || code === 0x7f) return null; + } let parsed: URL; try { parsed = new URL(trimmed); @@ -57,13 +70,16 @@ export function parseShopeeEnqueueUrl( if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { return null; } + if (parsed.username || parsed.password) { + return null; + } const host = parsed.hostname.toLowerCase(); if (host !== 'shopee.vn' && !host.endsWith('.shopee.vn')) { return null; } const match = - parsed.pathname.match(/(?:a-)?i\.(\d+)\.(\d+)/i) || - parsed.pathname.match(/\/(?:product|opaanlp)\/(\d+)\/(\d+)/i); + parsed.pathname.match(/(?:a-)?i\.([1-9]\d{0,19})\.([1-9]\d{0,19})(?!\d)/i) || + parsed.pathname.match(/\/(?:product|opaanlp)\/([1-9]\d{0,19})\/([1-9]\d{0,19})(?!\d)/i); if (!match || !match[1] || !match[2]) { return null; } From 5cca9e6e89e3979a16f75d160afd084dce1041b4 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:18:55 +0700 Subject: [PATCH 04/19] fix(importer): harden computeRetryDelayMs, logger redaction, and mergeKeySpecs (#840) (#841) * fix(importer): harden computeRetryDelayMs, logger redaction, and mergeKeySpecs (#840) * fix(importer): redact depth-5 leaves, preserve DAG refs, and scan boundary secrets (#840) --------- Co-authored-by: JOY --- .../__tests__/backfill-review-risk.test.ts | 85 +++++++++++++++++++ .../src/shopee/jobs/backfill-specs.ts | 28 +++++- apps/importer/src/shopee/logger.ts | 79 ++++++++++++----- 3 files changed, 168 insertions(+), 24 deletions(-) diff --git a/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts b/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts index 6afbfc61f..3b0636043 100644 --- a/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts +++ b/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { AiReviewAnalyzer } from '../ai-review-analyzer.js'; +import { computeRetryDelayMs, redactSensitiveString, sanitizeLogMeta } from '../logger.js'; import { loadAndScoreMerchant, rescoreMerchantReviewRisk } from '../merchant-review-risk.js'; import { mergeKeySpecs } from '../jobs/backfill-specs.js'; import { @@ -449,6 +450,90 @@ async function run() { platform_count: 3, }); + // DOS/Rate#840: computeRetryDelayMs bounds, redactSensitiveString & sanitizeLogMeta hardening, and mergeKeySpecs key safety + assert.equal( + Number.isInteger(computeRetryDelayMs(Number.NaN, Number.NaN, Number.NaN, () => Number.NaN)), + true, + 'computeRetryDelayMs must return a finite integer when inputs are NaN' + ); + assert.equal( + computeRetryDelayMs(0, 400, 400, () => 1.0), + 400, + 'computeRetryDelayMs must never exceed maxDelayMs when randomFn returns 1.0' + ); + + const fakeGeminiToken = `AIza${'x'.repeat(24)}`; + const fakeGhToken = `ghp_${'y'.repeat(24)}`; + const redacted = redactSensitiveString( + `GET https://api.example.com/v1?api_key=${'s'.repeat(12)}&token=${'t'.repeat(12)} ${fakeGeminiToken} ${fakeGhToken}` + ); + assert.equal(redacted.includes('s'.repeat(12)), false); + assert.equal(redacted.includes('t'.repeat(12)), false); + assert.equal(redacted.includes(fakeGeminiToken), false); + assert.equal(redacted.includes(fakeGhToken), false); + + const circularObj: Record = { + ok: 'fine', + nanVal: Number.NaN, + apiKey: 'secret-val', + }; + circularObj.self = circularObj; + const sharedChild = { status: 'active' }; + circularObj.firstRef = sharedChild; + circularObj.secondRef = sharedChild; + circularObj.d1 = { + d2: { + d3: { + d4: { + leafNote: `Bearer ${'z'.repeat(24)}`, + tooDeepObj: { nope: true }, + }, + }, + }, + }; + const sanitizedMeta = sanitizeLogMeta(circularObj) as Record; + assert.equal(sanitizedMeta.ok, 'fine'); + assert.equal(sanitizedMeta.nanVal, null); + assert.equal(sanitizedMeta.apiKey, '[REDACTED]'); + assert.equal(sanitizedMeta.self, '[Circular]'); + assert.deepEqual(sanitizedMeta.firstRef, { status: 'active' }); + assert.deepEqual( + sanitizedMeta.secondRef, + { status: 'active' }, + 'Shared non-circular DAG object references must not be marked [Circular]' + ); + const d4Out = ( + ((sanitizedMeta.d1 as Record).d2 as Record).d3 as Record< + string, + unknown + > + ).d4 as Record; + assert.equal(d4Out.leafNote, 'Bearer [REDACTED]'); + assert.equal(d4Out.tooDeepObj, '[MaxDepth]'); + + const boundaryStraddle = `${'a'.repeat(7990)} http://admin:${'p'.repeat(16)}@proxy.example.com`; + const redactedBoundary = redactSensitiveString(boundaryStraddle); + assert.equal(redactedBoundary.includes('p'.repeat(16)), false); + assert.equal(redactedBoundary.length <= 8000, true); + + const keyBoundedSpecs = mergeKeySpecs( + { + [' display type ']: 'LTPO OLED', + ['constructor']: 'polluted', + ['K'.repeat(120)]: 'too-long-key', + }, + { + nested: { + ['prototype']: 'blocked', + [' refresh rate ']: '120Hz', + }, + } + ); + assert.equal(keyBoundedSpecs['display type'], 'LTPO OLED'); + assert.equal(Object.hasOwn(keyBoundedSpecs, 'constructor'), false); + assert.equal(Object.hasOwn(keyBoundedSpecs, 'K'.repeat(120)), false); + assert.deepEqual(keyBoundedSpecs.nested, { 'refresh rate': '120Hz' }); + console.log('✅ Review risk backfill suite passed'); } diff --git a/apps/importer/src/shopee/jobs/backfill-specs.ts b/apps/importer/src/shopee/jobs/backfill-specs.ts index 942fc1cab..6a22bda37 100644 --- a/apps/importer/src/shopee/jobs/backfill-specs.ts +++ b/apps/importer/src/shopee/jobs/backfill-specs.ts @@ -99,6 +99,10 @@ export async function auditMasterSpecs(supabase: RateSupabaseClient): Promise MAX_KEY_SPEC_KEY_LENGTH || UNSAFE_KEY_SPEC_KEYS.has(cleaned)) { + return null; + } + return cleaned; +} + function sanitizeKeySpecValue(value: unknown, depth = 0): unknown { if (depth > 4 || value === undefined || value === null) return undefined; if (typeof value === 'boolean') return value; @@ -135,10 +147,15 @@ function sanitizeKeySpecValue(value: unknown, depth = 0): unknown { } if (typeof value === 'object') { const objOut: Record = {}; + let count = 0; for (const [k, v] of Object.entries(value as Record)) { + const safeKey = sanitizeKeySpecKey(k); + if (!safeKey) continue; + if (count >= MAX_KEY_SPEC_OBJECT_KEYS && !Object.hasOwn(objOut, safeKey)) continue; const sanitized = sanitizeKeySpecValue(v, depth + 1); if (sanitized !== undefined) { - objOut[k] = sanitized; + if (!Object.hasOwn(objOut, safeKey)) count += 1; + objOut[safeKey] = sanitized; } } return objOut; @@ -156,11 +173,16 @@ export function mergeKeySpecs( : {}; const next = incoming && typeof incoming === 'object' && !Array.isArray(incoming) ? incoming : {}; const out: Record = {}; + let count = 0; for (const source of [base, next]) { for (const [k, v] of Object.entries(source)) { + const safeKey = sanitizeKeySpecKey(k); + if (!safeKey) continue; + if (count >= MAX_KEY_SPEC_OBJECT_KEYS && !Object.hasOwn(out, safeKey)) continue; const sanitized = sanitizeKeySpecValue(v, 0); if (sanitized !== undefined) { - out[k] = sanitized; + if (!Object.hasOwn(out, safeKey)) count += 1; + out[safeKey] = sanitized; } } } @@ -226,7 +248,7 @@ export async function runMasterSpecsBackfill( battery_mah: device.battery_mah, has_5g: device.has_5g, specifications: device.specifications, - key_specs: device.key_specs, + key_specs: mergeKeySpecs(undefined, device.key_specs as unknown as Record), colors: device.colors, ...(device.images.length > 0 ? { images: device.images, thumbnail: device.images[0] } : {}), updated_at: new Date().toISOString(), diff --git a/apps/importer/src/shopee/logger.ts b/apps/importer/src/shopee/logger.ts index 76742978e..2442e07a3 100644 --- a/apps/importer/src/shopee/logger.ts +++ b/apps/importer/src/shopee/logger.ts @@ -41,6 +41,9 @@ export function computeImporterAlgorithmMetadata( }; } +export const MAX_LOG_STRING_LENGTH = 8000; +export const MAX_LOG_META_KEYS = 60; + /** * Computes exponential backoff with full jitter for LLM / network retries on 429 or 5xx (Audit C8). */ @@ -50,11 +53,19 @@ export function computeRetryDelayMs( maxDelayMs = 5000, randomFn: () => number = Math.random ): number { - const safeAttempt = Math.max(0, Math.min(10, Math.floor(attempt))); - const exp = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, safeAttempt)); + const safeAttempt = Number.isFinite(attempt) ? Math.max(0, Math.min(10, Math.floor(attempt))) : 0; + const safeBaseDelayMs = + Number.isFinite(baseDelayMs) && baseDelayMs >= 1 ? Math.floor(baseDelayMs) : 400; + const safeMaxDelayMs = + Number.isFinite(maxDelayMs) && maxDelayMs >= safeBaseDelayMs + ? Math.floor(maxDelayMs) + : Math.max(5000, safeBaseDelayMs); + const exp = Math.min(safeMaxDelayMs, safeBaseDelayMs * Math.pow(2, safeAttempt)); const half = Math.floor(exp / 2); - const jitter = Math.floor(Math.max(0, Math.min(1, randomFn())) * (exp - half + 1)); - return half + jitter; + const rawRand = randomFn(); + const clampedRand = Number.isFinite(rawRand) ? Math.max(0, Math.min(0.999999999, rawRand)) : 0.5; + const jitter = Math.floor(clampedRand * (exp - half + 1)); + return Math.min(safeMaxDelayMs, half + jitter); } /** @@ -92,19 +103,26 @@ export function sanitizeAiGeneratedText(raw: unknown, maxLen = 400): string { */ export function redactSensitiveString(raw: string): string { if (!raw) return raw; + const maxScanLen = MAX_LOG_STRING_LENGTH + 512; + const bounded = raw.length > maxScanLen ? raw.slice(0, maxScanLen) : raw; return ( - raw + bounded // Proxy credentials in URLs: http://user:pass@host -> http://u***:p***@host .replace(/(\bhttps?:\/\/)([^/\s:@]+):([^/\s@]+)@/gi, '$1u***:p***@') + // Query-parameter secrets: ?api_key=... / &token=... -> ?api_key=[REDACTED] + .replace( + /([?&](?:api_?key|token|secret|password|access_token|service_role_key)=)[^&\s#]+/gi, + '$1[REDACTED]' + ) // Bearer / eyJ JWT tokens .replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi, 'Bearer [REDACTED]') .replace( /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, '[REDACTED_JWT]' ) - // API keys (sk-..., sbp_..., sb_secret_...) + // API keys (sk-..., sbp_..., sb_secret_..., AIza..., ghp_..., github_pat_...) .replace( - /\b(?:sk-[A-Za-z0-9_-]{12,}|sbp_[A-Za-z0-9_-]{12,}|sb_secret_[A-Za-z0-9_-]{12,})\b/g, + /\b(?:sk-[A-Za-z0-9_-]{12,}|sbp_[A-Za-z0-9_-]{12,}|sb_secret_[A-Za-z0-9_-]{12,}|AIza[A-Za-z0-9_-]{20,}|gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b/g, '[REDACTED_KEY]' ) // Email addresses: alice@example.com -> a***@example.com @@ -112,37 +130,56 @@ export function redactSensitiveString(raw: string): string { /\b([A-Za-z0-9._%+-])[A-Za-z0-9._%+-]*@([A-Za-z0-9.-]+\.[A-Za-z]{2,})\b/g, '$1***@$2' ) + .slice(0, MAX_LOG_STRING_LENGTH) ); } const SENSITIVE_META_KEY_RE = /(password|secret|token|api_?key|service_role|authorization|cookie|credential|jwt)/i; -export function sanitizeLogMeta(value: unknown, depth = 0): unknown { - if (depth > 4 || value === null || value === undefined) return value; +const UNSAFE_OBJECT_KEYS = new Set(['__proto__', 'constructor', 'prototype']); + +export function sanitizeLogMeta( + value: unknown, + depth = 0, + seen: WeakSet = new WeakSet() +): unknown { + if (value === null || value === undefined) return value; if (typeof value === 'string') return redactSensitiveString(value); - if (typeof value === 'number' || typeof value === 'boolean') return value; + if (typeof value === 'boolean') return value; + if (typeof value === 'number') return Number.isFinite(value) ? value : null; if (value instanceof Error) { return { name: value.name || 'Error', message: redactSensitiveString(value.message || ''), }; } - if (Array.isArray(value)) { - return value.slice(0, 50).map(item => sanitizeLogMeta(item, depth + 1)); - } if (typeof value === 'object') { - const out: Record = {}; - for (const [k, v] of Object.entries(value as Record)) { - if (SENSITIVE_META_KEY_RE.test(k)) { - out[k] = '[REDACTED]'; - } else { - out[k] = sanitizeLogMeta(v, depth + 1); + if (seen.has(value)) return '[Circular]'; + if (depth > 4) return '[MaxDepth]'; + seen.add(value); + try { + if (Array.isArray(value)) { + return value.slice(0, 50).map(item => sanitizeLogMeta(item, depth + 1, seen)); + } + const out: Record = {}; + let count = 0; + for (const [k, v] of Object.entries(value as Record)) { + if (UNSAFE_OBJECT_KEYS.has(k)) continue; + if (count >= MAX_LOG_META_KEYS) break; + count += 1; + if (SENSITIVE_META_KEY_RE.test(k)) { + out[k] = '[REDACTED]'; + } else { + out[k] = sanitizeLogMeta(v, depth + 1, seen); + } } + return out; + } finally { + seen.delete(value); } - return out; } - return String(value); + return redactSensitiveString(String(value)); } export type ImporterLogLevel = 'info' | 'warn' | 'error'; From e5b0fc7a98dac8b1af23e93036a8c2500abc8462 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:44:46 +0700 Subject: [PATCH 05/19] fix(web): harden cache-tags targets, api-handler bounds, and cookie token parsing (#842) (#843) * fix(web): harden cache-tags targets, api-handler bounds, and cookie token parsing (#842) * fix(web): allow composite platform:shopId merchant identifiers in cache tags (#842) --------- Co-authored-by: JOY --- .../__tests__/core-rate-architecture.test.ts | 87 +++++++++++++++++++ apps/web/src/server/rate/api-handler.ts | 28 ++++-- apps/web/src/server/rate/cache-tags.ts | 80 +++++++++++++---- apps/web/src/server/rate/request-auth.ts | 24 +++-- 4 files changed, 189 insertions(+), 30 deletions(-) diff --git a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts index c6f9fec67..2472e9f9e 100644 --- a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts +++ b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts @@ -8471,6 +8471,93 @@ async function runCoreArchitectureSuite() { ' ✅ [94/94] /api/health reports a database the app cannot read (#829) passed' ); + // 95. cache-tags revalidation targets, api-handler bounds & cookie token parsing (#842) + { + const { canonicalizeCacheParams, resolveRevalidationTargets } = + await import('../cache-tags'); + const { BoundedSlidingWindowRateLimiter, parseBoundedStringParam } = + await import('../api-handler'); + const { readBearerToken, readCookieAccessToken } = + await import('../request-auth'); + + const sharedSub = { a: 1 }; + const cyclicCacheObj: Record = { + first: sharedSub, + second: sharedSub, + nanVal: Number.NaN, + }; + cyclicCacheObj.self = cyclicCacheObj; + const serialized = canonicalizeCacheParams(cyclicCacheObj); + assert.equal(serialized.includes('"[Circular]"'), true); + assert.equal(serialized.includes('"nanVal":null'), true); + + const targets = resolveRevalidationTargets({ + tags: [' RATE:PRODUCTS ', 'arbitrary-tag', 'rate:bad space'], + slugs: ['iphone-16-pro', '../bad/slug', 'a'.repeat(130)], + categories: ['smartphones', 'bad/category'], + merchants: ['m-123', 'shopee:88201679', 'bad merchant id'], + paths: ['/smartphones/iphone-16-pro', '//evil.com'], + }); + assert.equal(targets.tags.includes('rate:products'), true); + assert.equal(targets.tags.includes('arbitrary-tag'), false); + assert.equal(targets.tags.includes('rate:product:iphone-16-pro'), true); + assert.equal(targets.tags.includes('rate:category:smartphones'), true); + assert.equal(targets.tags.includes('rate:merchant:m-123'), true); + assert.equal(targets.tags.includes('rate:merchant:shopee:88201679'), true); + assert.deepEqual(targets.paths, ['/smartphones/iphone-16-pro']); + + assert.equal( + parseBoundedStringParam(' hello\u0000\u0007world ', 11), + 'hello world', + 'parseBoundedStringParam must strip control chars and trim after slicing' + ); + assert.equal( + parseBoundedStringParam('\u0000\u0007 ', 20), + null, + 'parseBoundedStringParam must return null when only control/whitespace chars remain' + ); + + const nanLimiter = new BoundedSlidingWindowRateLimiter({ + windowMs: Number.NaN, + maxRequests: Number.NaN, + maxBuckets: Number.NaN, + }); + assert.equal(nanLimiter.allow('ip-842', 1000), true); + + assert.equal( + readBearerToken( + new Request('https://rate.vn/api', { + headers: { Authorization: 'Bearer tok\u0007en' }, + }) + ), + '', + 'readBearerToken must reject tokens containing control characters' + ); + assert.equal( + await readCookieAccessToken([ + { + name: 'sb-ref-auth-token', + value: JSON.stringify({ access_token: 'bad\u0000token' }), + }, + ]), + null, + 'readCookieAccessToken must reject tokens containing control characters' + ); + assert.equal( + await readCookieAccessToken([ + { + name: 'sb-ref-auth-token', + value: JSON.stringify({ access_token: 't'.repeat(5000) }), + }, + ]), + null, + 'readCookieAccessToken must reject oversized access tokens > 4096 chars' + ); + } + console.log( + ' ✅ [95/95] cache-tags revalidation targets, api-handler bounds & cookie token parsing (#842) passed' + ); + console.log('🎉 All Rate.vn Core Architecture tests passed!'); } diff --git a/apps/web/src/server/rate/api-handler.ts b/apps/web/src/server/rate/api-handler.ts index 6230f17eb..0aac9cb89 100644 --- a/apps/web/src/server/rate/api-handler.ts +++ b/apps/web/src/server/rate/api-handler.ts @@ -91,9 +91,15 @@ export function parseBoundedStringParam( maxLength: number ): string | null { if (raw == null) return null; - const trimmed = raw.trim(); - if (!trimmed) return null; - return trimmed.length > maxLength ? trimmed.slice(0, maxLength) : trimmed; + const safeMaxLength = + Number.isFinite(maxLength) && maxLength >= 1 ? Math.floor(maxLength) : 256; + const cleaned = raw.replace(/\p{Cc}+/gu, ' ').trim(); + if (!cleaned) return null; + const sliced = + cleaned.length > safeMaxLength + ? cleaned.slice(0, safeMaxLength).trim() + : cleaned; + return sliced || null; } /** @@ -245,9 +251,19 @@ export class BoundedSlidingWindowRateLimiter { private readonly maxBuckets: number; constructor(options: SlidingWindowRateLimiterOptions) { - this.windowMs = Math.max(1_000, options.windowMs); - this.maxRequests = Math.max(1, options.maxRequests); - this.maxBuckets = Math.max(1, options.maxBuckets ?? 2_000); + this.windowMs = + Number.isFinite(options.windowMs) && options.windowMs >= 1_000 + ? Math.min(86_400_000, Math.floor(options.windowMs)) + : 60_000; + this.maxRequests = + Number.isFinite(options.maxRequests) && options.maxRequests >= 1 + ? Math.min(100_000, Math.floor(options.maxRequests)) + : 30; + const rawMaxBuckets = options.maxBuckets ?? 2_000; + this.maxBuckets = + Number.isFinite(rawMaxBuckets) && rawMaxBuckets >= 1 + ? Math.min(50_000, Math.floor(rawMaxBuckets)) + : 2_000; } public get size(): number { diff --git a/apps/web/src/server/rate/cache-tags.ts b/apps/web/src/server/rate/cache-tags.ts index bd0295ecd..2719d709a 100644 --- a/apps/web/src/server/rate/cache-tags.ts +++ b/apps/web/src/server/rate/cache-tags.ts @@ -40,24 +40,51 @@ export interface RevalidationTargetsInput { paths?: readonly string[]; } -export function canonicalizeCacheParams(value: unknown): string { +const SAFE_CACHE_TAG_REGEX = /^rate:[a-z0-9:._-]{1,150}$/i; +const SAFE_ENTITY_TOKEN_REGEX = /^[a-z0-9._-]+$/i; +const SAFE_MERCHANT_IDENTIFIER_REGEX = /^[a-z0-9:._-]+$/i; + +export function canonicalizeCacheParams( + value: unknown, + depth = 0, + seen: WeakSet = new WeakSet() +): string { if (value === null || value === undefined) { - return '{}'; + return depth === 0 ? '{}' : 'null'; + } + if (typeof value === 'number') { + return Number.isFinite(value) ? JSON.stringify(value) : 'null'; + } + if (typeof value !== 'object') { + return JSON.stringify(value); } - if (Array.isArray(value)) { - return `[${value.map((item) => canonicalizeCacheParams(item)).join(',')}]`; + if (seen.has(value)) { + return '"[Circular]"'; } - if (typeof value === 'object') { + if (depth > 6) { + return '"[MaxDepth]"'; + } + seen.add(value); + try { + if (Array.isArray(value)) { + return `[${value + .slice(0, 100) + .map((item) => canonicalizeCacheParams(item, depth + 1, seen)) + .join(',')}]`; + } const entries = Object.entries(value as Record) .filter(([, val]) => val !== undefined) - .sort(([keyA], [keyB]) => keyA.localeCompare(keyB)); + .sort(([keyA], [keyB]) => keyA.localeCompare(keyB)) + .slice(0, 60); return `{${entries .map( - ([key, val]) => `${JSON.stringify(key)}:${canonicalizeCacheParams(val)}` + ([key, val]) => + `${JSON.stringify(key)}:${canonicalizeCacheParams(val, depth + 1, seen)}` ) .join(',')}}`; + } finally { + seen.delete(value); } - return JSON.stringify(value); } export function isSafeRevalidationPath(rawPath: string): boolean { @@ -80,13 +107,19 @@ export function resolveRevalidationTargets(input: RevalidationTargetsInput): { const tagSet = new Set(); const pathSet = new Set(); - for (const tag of input.tags ?? []) { - const clean = tag.trim(); - if (clean) tagSet.add(clean); + for (const tag of (input.tags ?? []).slice(0, 50)) { + if (typeof tag !== 'string') continue; + const clean = tag.trim().toLowerCase(); + if (clean && SAFE_CACHE_TAG_REGEX.test(clean)) { + tagSet.add(clean); + } } - for (const slug of input.slugs ?? []) { + for (const slug of (input.slugs ?? []).slice(0, 50)) { + if (typeof slug !== 'string') continue; const clean = slug.trim(); - if (!clean) continue; + if (!clean || clean.length > 120 || !SAFE_ENTITY_TOKEN_REGEX.test(clean)) { + continue; + } tagSet.add(productCacheTag(clean)); tagSet.add(RATE_CACHE_TAGS.PRODUCTS); tagSet.add(RATE_CACHE_TAGS.OFFERS); @@ -95,18 +128,28 @@ export function resolveRevalidationTargets(input: RevalidationTargetsInput): { tagSet.add(RATE_CACHE_TAGS.DEALS); tagSet.add(RATE_CACHE_TAGS.STATS); } - for (const cat of input.categories ?? []) { + for (const cat of (input.categories ?? []).slice(0, 20)) { + if (typeof cat !== 'string') continue; const clean = cat.trim(); - if (!clean) continue; + if (!clean || clean.length > 60 || !SAFE_ENTITY_TOKEN_REGEX.test(clean)) { + continue; + } tagSet.add(categoryCacheTag(clean)); tagSet.add(RATE_CACHE_TAGS.CATEGORIES); tagSet.add(RATE_CACHE_TAGS.PRODUCTS); tagSet.add(RATE_CACHE_TAGS.DEALS); tagSet.add(RATE_CACHE_TAGS.STATS); } - for (const merchantId of input.merchants ?? []) { + for (const merchantId of (input.merchants ?? []).slice(0, 50)) { + if (typeof merchantId !== 'string') continue; const clean = merchantId.trim(); - if (!clean) continue; + if ( + !clean || + clean.length > 120 || + !SAFE_MERCHANT_IDENTIFIER_REGEX.test(clean) + ) { + continue; + } tagSet.add(merchantCacheTag(clean)); tagSet.add(RATE_CACHE_TAGS.MERCHANTS); tagSet.add(RATE_CACHE_TAGS.OFFERS); @@ -114,7 +157,8 @@ export function resolveRevalidationTargets(input: RevalidationTargetsInput): { tagSet.add(RATE_CACHE_TAGS.DEALS); tagSet.add(RATE_CACHE_TAGS.STATS); } - for (const p of input.paths ?? []) { + for (const p of (input.paths ?? []).slice(0, 50)) { + if (typeof p !== 'string') continue; const clean = p.trim(); if (isSafeRevalidationPath(clean)) { pathSet.add(clean); diff --git a/apps/web/src/server/rate/request-auth.ts b/apps/web/src/server/rate/request-auth.ts index a13c7438f..688cffbbc 100644 --- a/apps/web/src/server/rate/request-auth.ts +++ b/apps/web/src/server/rate/request-auth.ts @@ -44,7 +44,11 @@ export function readBearerToken(request: Request): string { const match = /^Bearer\s+(.+)$/i.exec(header); if (!match || !match[1]) return ''; const token = match[1].trim(); - if (!token || token.length > MAX_BEARER_TOKEN_LENGTH || /\s/.test(token)) { + if ( + !token || + token.length > MAX_BEARER_TOKEN_LENGTH || + /[\s\p{Cc}]/u.test(token) + ) { return ''; } return token; @@ -59,6 +63,7 @@ function readCookies(request: Request): CookieRecord[] { const AUTH_COOKIE_RE = /^(sb-[^.]+-auth-token)(?:\.\d+)?$/; const BASE64_PREFIX = 'base64-'; +const MAX_COOKIE_SESSION_LENGTH = 16_384; /** * Reads the access token from the @supabase/ssr session cookie @@ -74,16 +79,23 @@ export async function readCookieAccessToken( .find((k): k is string => Boolean(k)); if (!key) return null; const raw = await combineChunks(key, (name) => byName.get(name)); - if (!raw) return null; + if (!raw || raw.length > MAX_COOKIE_SESSION_LENGTH) return null; try { const json = raw.startsWith(BASE64_PREFIX) ? stringFromBase64URL(raw.slice(BASE64_PREFIX.length)) : raw; + if (!json || json.length > MAX_COOKIE_SESSION_LENGTH) return null; const session = JSON.parse(json) as { access_token?: unknown }; - return typeof session.access_token === 'string' && - session.access_token.length > 0 - ? session.access_token - : null; + if (typeof session.access_token !== 'string') return null; + const token = session.access_token.trim(); + if ( + !token || + token.length > MAX_BEARER_TOKEN_LENGTH || + /[\s\p{Cc}]/u.test(token) + ) { + return null; + } + return token; } catch { return null; } From cf37bcc581e22d0a2801655a08bf4233b65adca8 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:05:58 +0700 Subject: [PATCH 06/19] fix(web): harden structured logger redaction and extension URL/payload bounds (#844) (#845) Co-authored-by: JOY --- apps/web/src/lib/__tests__/logger.test.ts | 20 ++++++ apps/web/src/lib/logger.ts | 53 +++++++++++--- .../__tests__/core-rate-architecture.test.ts | 72 +++++++++++++++++++ apps/web/src/server/rate/extension.ts | 53 +++++++++++--- 4 files changed, 178 insertions(+), 20 deletions(-) diff --git a/apps/web/src/lib/__tests__/logger.test.ts b/apps/web/src/lib/__tests__/logger.test.ts index 5d5d79933..95e76d79d 100644 --- a/apps/web/src/lib/__tests__/logger.test.ts +++ b/apps/web/src/lib/__tests__/logger.test.ts @@ -85,6 +85,26 @@ function main() { } ); + // Circular references, shared DAG sub-objects, BigInt, NaN, and boundary-straddling emails/tokens (#844). + const sharedDag = { ok: true }; + const cyclic: Record = { + first: sharedDag, + second: sharedDag, + big: BigInt('9007199254740993'), + nanVal: Number.NaN, + bearerMsg: `Authorization: Bearer ${'s'.repeat(16)}`, + boundaryEmail: `${'a'.repeat(1995)}victim@example.com`, + }; + cyclic.self = cyclic; + const redactedCyclic = redact(cyclic) as Record; + assert.equal(redactedCyclic.self, '[Circular]'); + assert.deepEqual(redactedCyclic.first, { ok: true }); + assert.deepEqual(redactedCyclic.second, { ok: true }); + assert.equal(redactedCyclic.big, '9007199254740993'); + assert.equal(redactedCyclic.nanVal, null); + assert.equal(redactedCyclic.bearerMsg, 'Authorization: Bearer [redacted]'); + assert.equal(String(redactedCyclic.boundaryEmail).includes('victim'), false); + console.warn('✅ Structured logger suite passed'); } diff --git a/apps/web/src/lib/logger.ts b/apps/web/src/lib/logger.ts index 1d94830fd..f25fcb868 100644 --- a/apps/web/src/lib/logger.ts +++ b/apps/web/src/lib/logger.ts @@ -12,12 +12,25 @@ const SECRET_KEY = // Postgres error text can echo user input ("Key (email)=(a@b.c) ..."); emails // are personal data, so they are masked inside any logged string. const EMAIL_IN_TEXT = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi; +const BEARER_IN_TEXT = /\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/gi; +const URL_CREDENTIALS_IN_TEXT = /(https?:\/\/[^/\s:@]+:)([^/\s@]+)(@)/gi; +const JWT_IN_TEXT = + /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g; -function scrubText(text: string): string { - return text.slice(0, MAX_STRING).replace(EMAIL_IN_TEXT, '[email]'); -} const MAX_DEPTH = 4; const MAX_STRING = 2000; +const MAX_KEYS = 60; +const UNSAFE_OBJECT_KEYS = new Set(['__proto__', 'constructor', 'prototype']); + +function scrubText(text: string): string { + const bounded = text.slice(0, MAX_STRING + 256); + return bounded + .replace(EMAIL_IN_TEXT, '[email]') + .replace(BEARER_IN_TEXT, 'Bearer [redacted]') + .replace(URL_CREDENTIALS_IN_TEXT, '$1[redacted]$3') + .replace(JWT_IN_TEXT, '[redacted]') + .slice(0, MAX_STRING); +} function isProduction(): boolean { return process.env.NODE_ENV === 'production'; @@ -41,18 +54,36 @@ export function serializeError(err: unknown): LogFields { return { message: scrubText(String(err)) }; } -export function redact(value: unknown, depth = 0): unknown { +export function redact( + value: unknown, + depth = 0, + seen: WeakSet = new WeakSet() +): unknown { if (value instanceof Error) return serializeError(value); if (typeof value === 'string') return scrubText(value); + if (typeof value === 'number') return Number.isFinite(value) ? value : null; + if (typeof value === 'bigint') return value.toString(); if (value === null || typeof value !== 'object') return value; + if (seen.has(value)) return '[Circular]'; if (depth >= MAX_DEPTH) return '[truncated]'; - if (Array.isArray(value)) - return value.slice(0, 50).map((v) => redact(v, depth + 1)); - const out: LogFields = {}; - for (const [key, v] of Object.entries(value)) { - out[key] = SECRET_KEY.test(key) ? '[redacted]' : redact(v, depth + 1); + seen.add(value); + try { + if (Array.isArray(value)) { + return value.slice(0, 50).map((v) => redact(v, depth + 1, seen)); + } + const out: LogFields = {}; + for (const [key, v] of Object.entries(value).slice(0, MAX_KEYS)) { + if (UNSAFE_OBJECT_KEYS.has(key)) continue; + const safeKey = key.slice(0, 120); + if (!safeKey) continue; + out[safeKey] = SECRET_KEY.test(safeKey) + ? '[redacted]' + : redact(v, depth + 1, seen); + } + return out; + } finally { + seen.delete(value); } - return out; } export interface Logger { @@ -69,7 +100,7 @@ export function createLogger(scope: string): Logger { ts: new Date().toISOString(), level, scope, - msg, + msg: scrubText(String(msg ?? '')), ...((fields ? redact(fields) : {}) as LogFields), }); if (level === 'error' || level === 'warn') { diff --git a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts index 2472e9f9e..07d9b54ae 100644 --- a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts +++ b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts @@ -8558,6 +8558,78 @@ async function runCoreArchitectureSuite() { ' ✅ [95/95] cache-tags revalidation targets, api-handler bounds & cookie token parsing (#842) passed' ); + // ========================================================================= + // 96. Extension Server URL Credential/Length & Crowd Ingest Bounds (#844) + // ========================================================================= + { + const { + canonicalizeProductUrl, + detectPlatformFromInput, + ingestExtensionCrowdSignal, + } = await import('../extension'); + + assert.equal( + canonicalizeProductUrl('https://user:pass@shopee.vn/product/123/456'), + null, + 'canonicalizeProductUrl must reject embedded URL credentials' + ); + assert.equal( + canonicalizeProductUrl( + `https://shopee.vn/product/123/${'a'.repeat(2100)}` + ), + null, + 'canonicalizeProductUrl must reject URLs > 2048 chars' + ); + assert.equal( + detectPlatformFromInput( + null, + 'https://user:pass@shopee.vn/product/123/456' + ), + null, + 'detectPlatformFromInput must reject embedded URL credentials' + ); + + let capturedRpcArgs: Record | null = null; + const ingestRes844 = await ingestExtensionCrowdSignal({ + platform: 'shopee', + externalItemId: ' 18093055535\u0007 ', + externalShopId: ' 88201679\u0000 ', + url: 'https://shopee.vn/Apple-iPhone-15-128GB-i.88201679.18093055535', + title: ' Apple\u0000 iPhone 15 128GB Chính Hãng ', + selectedVariant: ' 128GB\u0007 Đen ', + price: 18_990_000, + userConsentOptIn: true, + rateClient: { + rpc: async (_fn, args) => { + capturedRpcArgs = args as unknown as Record; + return { + data: [{ job_id: 'job-844', state: 'enqueued' }], + error: null, + }; + }, + }, + }); + assert.equal(ingestRes844.accepted, true); + assert.equal( + (capturedRpcArgs as unknown as { p_platform_item_id: string }) + ?.p_platform_item_id, + '88201679_18093055535' + ); + const payload844 = ( + capturedRpcArgs as unknown as { + p_payload: { observed_title: string; selected_variant: string }; + } + )?.p_payload; + assert.equal( + payload844?.observed_title, + 'Apple iPhone 15 128GB Chính Hãng' + ); + assert.equal(payload844?.selected_variant, '128GB Đen'); + console.log( + ' ✅ [96/96] Extension server URL credential/length & crowd ingest bounds (#844) passed' + ); + } + console.log('🎉 All Rate.vn Core Architecture tests passed!'); } diff --git a/apps/web/src/server/rate/extension.ts b/apps/web/src/server/rate/extension.ts index d83e8f40b..853c84719 100644 --- a/apps/web/src/server/rate/extension.ts +++ b/apps/web/src/server/rate/extension.ts @@ -278,19 +278,34 @@ function isDomainOrSubdomain(host: string, domain: string): boolean { return host === domain || host.endsWith(`.${domain}`); } +const MAX_EXTENSION_URL_LENGTH = 2048; + +function sanitizeExtensionField(raw: unknown, maxLen: number): string { + if (typeof raw !== 'string') return ''; + return raw + .replace(/\p{Cc}+/gu, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, maxLen) + .trim(); +} + /** * Strip all tracking & session parameters (sp_atk, xptdk, utm_*, fbclid, etc.) * to comply with Chrome Web Store User Data Privacy policies. */ export function canonicalizeProductUrl(rawUrl?: string | null): string | null { if (!rawUrl || typeof rawUrl !== 'string') return null; - const trimmed = rawUrl.trim(); - if (!trimmed) return null; + const trimmed = rawUrl.replace(/\p{Cc}+/gu, '').trim(); + if (!trimmed || trimmed.length > MAX_EXTENSION_URL_LENGTH) return null; try { const parsed = new URL(trimmed); if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { return null; } + if (parsed.username !== '' || parsed.password !== '') { + return null; + } return `${parsed.origin}${parsed.pathname}`; } catch { return null; @@ -321,12 +336,17 @@ export function detectPlatformFromInput( return norm; } if (!rawUrl || typeof rawUrl !== 'string') return null; + const trimmedUrl = rawUrl.replace(/\p{Cc}+/gu, '').trim(); + if (!trimmedUrl || trimmedUrl.length > MAX_EXTENSION_URL_LENGTH) return null; let host: string; try { - const parsed = new URL(rawUrl.trim()); + const parsed = new URL(trimmedUrl); if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { return null; } + if (parsed.username !== '' || parsed.password !== '') { + return null; + } host = parsed.hostname.toLowerCase(); } catch { return null; @@ -1321,9 +1341,20 @@ export async function ingestExtensionCrowdSignal( rpcClient = getRateClient(); } - const normalizedItemId = input.externalShopId - ? `${input.externalShopId}_${input.externalItemId}` - : input.externalItemId; + const cleanItemId = sanitizeExtensionField(input.externalItemId, 128); + const cleanShopId = sanitizeExtensionField(input.externalShopId, 128); + if (!cleanItemId) { + return { + accepted: false, + quarantinedForWorker: false, + jobState: 'rejected_invalid_url', + jobId: null, + reason: 'Mã định danh sản phẩm không hợp lệ.', + }; + } + const normalizedItemId = cleanShopId + ? `${cleanShopId}_${cleanItemId}` + : cleanItemId; const validOriginalPrice = typeof input.originalPrice === 'number' && Number.isFinite(input.originalPrice) && @@ -1346,16 +1377,20 @@ export async function ingestExtensionCrowdSignal( ? Math.round(input.ratingCount) : null; + const cleanObservedTitle = sanitizeExtensionField(input.title, 280); + const cleanSelectedVariant = + sanitizeExtensionField(input.selectedVariant, 120) || null; + const { data, error } = await rpcClient.rpc('enqueue_crowd_verify_signal', { p_platform: input.platform, p_target_url: canonicalUrl, p_platform_item_id: normalizedItemId, - p_platform_shop_id: input.externalShopId || undefined, + p_platform_shop_id: cleanShopId || undefined, p_payload: { source: 'extension_mv3_quarantine', canonical_url: canonicalUrl, - observed_title: input.title.slice(0, 280), - selected_variant: input.selectedVariant?.slice(0, 120) || null, + observed_title: cleanObservedTitle, + selected_variant: cleanSelectedVariant, observed_price: safeObservedPrice, observed_original_price: validOriginalPrice, observed_rating_star: validRatingStar, From fa86dcf6e1f040de0d81a1bb2935371f841c3525 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:25:08 +0700 Subject: [PATCH 07/19] fix(extension): harden security-guards and detector JSON-LD/ID bounds (#846) (#847) Co-authored-by: JOY --- .../src/extractors/ecommerce-detector.ts | 70 +++++++--- apps/extension/src/security-guards.ts | 62 ++++++++- apps/extension/src/test-extension-pipeline.ts | 121 ++++++++++++++++++ 3 files changed, 232 insertions(+), 21 deletions(-) diff --git a/apps/extension/src/extractors/ecommerce-detector.ts b/apps/extension/src/extractors/ecommerce-detector.ts index d738d68a3..a54de710f 100644 --- a/apps/extension/src/extractors/ecommerce-detector.ts +++ b/apps/extension/src/extractors/ecommerce-detector.ts @@ -7,11 +7,17 @@ import type { * Strip all tracking query strings (sp_atk, xptdk, utm_*, fbclid, etc.) and hashes. */ export function sanitizeProductUrl(rawUrl: string): string | null { + if (typeof rawUrl !== 'string') return null; + const cleaned = rawUrl.replace(/\p{Cc}+/gu, '').trim(); + if (!cleaned || cleaned.length > 2048) return null; try { - const parsed = new URL(rawUrl); + const parsed = new URL(cleaned); if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { return null; } + if (parsed.username || parsed.password) { + return null; + } return `${parsed.origin}${parsed.pathname}`; } catch { return null; @@ -25,12 +31,18 @@ function isDomainOrSubdomain(host: string, domain: string): boolean { export function detectEcommercePlatform( url: string ): SupportedEcommercePlatform | null { + if (typeof url !== 'string') return null; + const cleaned = url.replace(/\p{Cc}+/gu, '').trim(); + if (!cleaned || cleaned.length > 2048) return null; let host: string; try { - const parsed = new URL(url); + const parsed = new URL(cleaned); if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { return null; } + if (parsed.username || parsed.password) { + return null; + } host = parsed.hostname.toLowerCase(); } catch { return null; @@ -114,10 +126,10 @@ export function extractIdsFromUrl( const m = canonicalUrl.match(/(?:a-)?i\.(\d+)\.(\d+)/i) || canonicalUrl.match(/\/(?:product|opaanlp)\/(\d+)\/(\d+)/i); - if (m) { + if (m && m[1] && m[2] && m[1].length <= 24 && m[2].length <= 24) { return { - externalShopId: m[1] || null, - externalItemId: m[2] || null, + externalShopId: m[1], + externalItemId: m[2], }; } return { externalItemId: null, externalShopId: null }; @@ -125,9 +137,9 @@ export function extractIdsFromUrl( if (platform === 'lazada') { const m = canonicalUrl.match(/-i(\d+)(?:-s(\d+))?\.html/i); - if (m) { + if (m && m[1] && m[1].length <= 24 && (!m[2] || m[2].length <= 24)) { return { - externalItemId: m[1] || null, + externalItemId: m[1], externalShopId: m[2] || null, }; } @@ -136,9 +148,9 @@ export function extractIdsFromUrl( if (platform === 'tiki') { const m = canonicalUrl.match(/-p(\d+)\.html/i); - if (m) { + if (m && m[1] && m[1].length <= 24) { return { - externalItemId: m[1] || null, + externalItemId: m[1], externalShopId: null, }; } @@ -153,9 +165,13 @@ export function extractIdsFromUrl( canonicalUrl.match(/\/product\/(\d+)/i) || canonicalUrl.match(/\/view\/product\/(\d+)/i); const shopMatch = canonicalUrl.match(/\/@([a-zA-Z0-9_.-]+)/); + const itemId = + prodMatch?.[1] && prodMatch[1].length <= 24 ? prodMatch[1] : null; + const shopId = + shopMatch?.[1] && shopMatch[1].length <= 64 ? shopMatch[1] : null; return { - externalItemId: prodMatch?.[1] || null, - externalShopId: shopMatch?.[1] || null, + externalItemId: itemId, + externalShopId: shopId, }; } @@ -179,6 +195,8 @@ export function extractIdsFromUrl( const isValidItemSlug = Boolean(lastSegment) && lastSegment!.length >= 2 && + lastSegment!.length <= 120 && + /^[a-zA-Z0-9_.-]+$/.test(lastSegment!) && !RETAIL_NON_PRODUCT_FIRST_SEGMENTS.has(firstSegLower) && !RETAIL_NON_PRODUCT_FIRST_SEGMENTS.has(lastSegLower) && !(segments.length === 1 && RETAIL_CATEGORY_ROOT_SLUGS.has(firstSegLower)) && @@ -225,6 +243,9 @@ const JSON_LD_PRODUCT_TYPES = new Set([ 'productmodel', ]); +const MAX_JSON_LD_ENTITIES = 100; +const MAX_JSON_LD_OFFERS = 100; + export function flattenJsonLdEntities( raw: unknown, depth = 0 @@ -233,7 +254,12 @@ export function flattenJsonLdEntities( if (Array.isArray(raw)) { const out: Record[] = []; for (const entry of raw) { - out.push(...flattenJsonLdEntities(entry, depth + 1)); + if (out.length >= MAX_JSON_LD_ENTITIES) break; + const nested = flattenJsonLdEntities(entry, depth + 1); + for (const item of nested) { + if (out.length >= MAX_JSON_LD_ENTITIES) break; + out.push(item); + } } return out; } @@ -241,7 +267,11 @@ export function flattenJsonLdEntities( const obj = raw as Record; const out: Record[] = [obj]; if (Array.isArray(obj['@graph'])) { - out.push(...flattenJsonLdEntities(obj['@graph'], depth + 1)); + const nested = flattenJsonLdEntities(obj['@graph'], depth + 1); + for (const item of nested) { + if (out.length >= MAX_JSON_LD_ENTITIES) break; + out.push(item); + } } return out; } @@ -270,11 +300,18 @@ export function extractPriceFromJsonLdOffers(offersRaw: unknown): { } const candidates: Record[] = []; const pushOffer = (entry: unknown) => { - if (!entry || typeof entry !== 'object') return; + if ( + !entry || + typeof entry !== 'object' || + candidates.length >= MAX_JSON_LD_OFFERS + ) { + return; + } const obj = entry as Record; candidates.push(obj); if (Array.isArray(obj.offers)) { for (const nested of obj.offers) { + if (candidates.length >= MAX_JSON_LD_OFFERS) break; if (nested && typeof nested === 'object') { candidates.push(nested as Record); } @@ -283,7 +320,10 @@ export function extractPriceFromJsonLdOffers(offersRaw: unknown): { }; if (Array.isArray(offersRaw)) { - for (const entry of offersRaw) pushOffer(entry); + for (const entry of offersRaw) { + if (candidates.length >= MAX_JSON_LD_OFFERS) break; + pushOffer(entry); + } } else { pushOffer(offersRaw); } diff --git a/apps/extension/src/security-guards.ts b/apps/extension/src/security-guards.ts index 7fdf7b8d0..9179ad356 100644 --- a/apps/extension/src/security-guards.ts +++ b/apps/extension/src/security-guards.ts @@ -122,6 +122,35 @@ export function selectFreshestSessionSignal( return tabSignal ?? latestSignal ?? null; } +const SAFE_EXTERNAL_ID_REGEX = /^[a-zA-Z0-9_.-]{1,120}$/; + +function sanitizeExternalIdentifier(val: unknown): string | null { + if (typeof val !== 'string') return null; + const cleaned = val.replace(/\p{Cc}+/gu, '').trim(); + if (!cleaned || !SAFE_EXTERNAL_ID_REGEX.test(cleaned)) { + return null; + } + return cleaned; +} + +function sanitizeCanonicalSignalUrl(rawUrl: unknown): string { + if (typeof rawUrl !== 'string') return ''; + const cleaned = rawUrl.replace(/\p{Cc}+/gu, '').trim(); + if (!cleaned || cleaned.length > 2048) return ''; + try { + const parsed = new URL(cleaned); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + return ''; + } + if (parsed.username || parsed.password) { + return ''; + } + return `${parsed.origin}${parsed.pathname}`; + } catch { + return ''; + } +} + export function sanitizeExtensionProductSignal( signal: ExtractedPageProductSignal ): ExtractedPageProductSignal { @@ -159,14 +188,27 @@ export function sanitizeExtensionProductSignal( : null; const trimmedTitle = - typeof signal.title === 'string' ? signal.title.trim().slice(0, 500) : ''; + typeof signal.title === 'string' + ? signal.title + .replace(/\p{Cc}+/gu, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 500) + : ''; const trimmedVariant = - typeof signal.selectedVariant === 'string' && signal.selectedVariant.trim() - ? signal.selectedVariant.trim().slice(0, 200) + typeof signal.selectedVariant === 'string' + ? signal.selectedVariant + .replace(/\p{Cc}+/gu, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 200) || null : null; return { ...signal, + canonicalUrl: sanitizeCanonicalSignalUrl(signal.canonicalUrl), + externalItemId: sanitizeExternalIdentifier(signal.externalItemId), + externalShopId: sanitizeExternalIdentifier(signal.externalShopId), title: trimmedTitle, selectedVariant: trimmedVariant, price: validPrice, @@ -181,13 +223,16 @@ export function escapeSafeExternalHref( fallbackUrl = 'https://rate.vn' ): string { if (typeof rawUrl !== 'string') return fallbackUrl; - const trimmed = rawUrl.trim(); - if (!trimmed) return fallbackUrl; + const trimmed = rawUrl.replace(/\p{Cc}+/gu, '').trim(); + if (!trimmed || trimmed.length > 2048) return fallbackUrl; try { const parsed = new URL(trimmed, 'https://rate.vn'); if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { return fallbackUrl; } + if (parsed.username || parsed.password) { + return fallbackUrl; + } return parsed.href .replace(/&/g, '&') .replace(/ | null | undefined ): number[] { if (!Array.isArray(points)) return []; - return points + const valid = points .map((p) => Number(p?.price)) .filter( (p) => Number.isFinite(p) && p > 0 && p <= MAX_VALID_EXTENSION_VND_PRICE ); + return valid.length > MAX_SPARKLINE_POINTS + ? valid.slice(-MAX_SPARKLINE_POINTS) + : valid; } export function shouldRenderTrustPill( diff --git a/apps/extension/src/test-extension-pipeline.ts b/apps/extension/src/test-extension-pipeline.ts index 1905f0d32..ecb751762 100644 --- a/apps/extension/src/test-extension-pipeline.ts +++ b/apps/extension/src/test-extension-pipeline.ts @@ -16,6 +16,7 @@ import { } from './extractors/ecommerce-detector.js'; import { DEFAULT_API_BASE_URL, + MAX_SPARKLINE_POINTS, escapeSafeExternalHref, isPrivilegedExtensionSender, normalizeAllowedApiBaseUrl, @@ -2442,6 +2443,126 @@ async function runTests() { ); } + // --- DOS/Rate#846: security-guards signal/URL sanitization & ecommerce-detector JSON-LD/ID bounds --- + { + const sanitizedSig = sanitizeExtensionProductSignal({ + platform: 'shopee', + canonicalUrl: + 'https://shopee.vn/iPhone-16-i.88201679.29512841920?utm_source=ads#frag', + externalItemId: ' 29512841920\u0000 ', + externalShopId: 'bad'), '#'); + assert.equal(formatHref('//evil.example/phish'), '#'); + assert.equal(formatHref('/\\evil.example/phish'), '#'); + assert.equal(formatHref('https://user:pass@rate.vn/deals'), '#'); + assert.equal(formatHref('httpfoo:bar'), '#'); + assert.equal(formatHref(`https://rate.vn/${'a'.repeat(2100)}`), '#'); + assert.equal(formatHref('products/iphone-16\u0000'), '/products/iphone-16'); + assert.equal(formatHref('#reviews\u0007'), '#reviews'); + assert.equal( + formatHref('mailto:support@rate.vn'), + 'mailto:support@rate.vn' + ); + assert.equal(formatHref('mailto:bad address@rate.vn'), '#'); + + assert.equal(isAppLink('https://rate.vn/smartphones'), true); + assert.equal(isAppLink('/smartphones/iphone-16'), true); + assert.equal(isAppLink('//evil.example/phish'), false); + assert.equal(isAppLink('javascript:alert(1)'), false); + assert.equal(isAppLink('https://user:pass@rate.vn/smartphones'), false); + + const summary856 = summarizeMerchantDirectory([ + { + id: 'm1', + name: 'Shop 1', + platform: 'constructor', + platform_shop_id: 's1', + is_official: true, + trust_score: 90, + dosafe_risk_score: 10, + } as never, + { + id: 'm2', + name: 'Shop 2', + platform: ' Shopee\u0000 ', + platform_shop_id: 's2', + is_official: false, + trust_score: 70, + dosafe_risk_score: 20, + } as never, + ]); + assert.equal(summary856.platformCounts.shopee, 1); + assert.equal( + Object.hasOwn(summary856.platformCounts, 'constructor'), + false, + 'summarizeMerchantDirectory must not collide with Object.prototype.constructor' + ); + + const filterParams856 = merchantDirectoryFilterParams({ + query: ` apple\u0000 store ${'x'.repeat(200)}`, + platform: 'bad/platform', + officialOnly: true, + riskTier: 'safe', + seedingTier: 'low', + sortBy: 'products_desc', + }); + assert.equal(filterParams856.get('q')?.length, 160); + assert.equal(filterParams856.get('platform'), null); + assert.equal(filterParams856.get('official'), 'true'); + + const heroMerchants856 = toHeroMerchantSuggestions( + Array.from({ length: MAX_HERO_MERCHANTS + 10 }, (_, idx) => ({ + id: `m-${idx}\u0000`, + name: ` Store ${idx}\u0007 `, + platform: 'shopee', + platform_shop_id: `shop-${idx}`, + is_official: true, + trust_score: 150, + dosafe_risk_score: -10, + })) + ); + assert.equal(heroMerchants856.length, MAX_HERO_MERCHANTS); + assert.equal(heroMerchants856[0]?.id, 'm-0'); + assert.equal(heroMerchants856[0]?.name, 'Store 0'); + assert.equal(heroMerchants856[0]?.trust_score, 100); + assert.equal(heroMerchants856[0]?.dosafe_risk_score, 0); + + const heroProducts856 = toHeroProductSuggestions( + Array.from({ length: MAX_HERO_PRODUCTS + 5 }, (_, idx) => ({ + id: `p-${idx}`, + slug: `slug-${idx}`, + name: `Product ${idx}`, + brand: 'Apple', + model: 'iPhone', + model_key: `apple:iphone:${idx}`, + variant_name: '128GB', + key_specs: { + chip: ' A18\u0000 Pro ', + screen: '\u0007 ', + }, + storage_gb: 128, + images: ['img1.jpg', 'img2.jpg'], + thumbnail: 'thumb.jpg', + min_price: 20_000_000, + merchant_count: 3, + user_rating_weighted: 4.8, + average_rating: 4.7, + category_id: 'cat-1', + metadata: { category_slug: ' smartphones\u0000 ', extra: 1 } as never, + specifications: { category_slug: '\u0000 ' } as never, + })) + ); + assert.equal(heroProducts856.length, MAX_HERO_PRODUCTS); + assert.equal(heroProducts856[0]?.key_specs?.chip, 'A18 Pro'); + assert.equal(heroProducts856[0]?.key_specs?.screen, undefined); + assert.deepEqual(heroProducts856[0]?.metadata, { + category_slug: 'smartphones', + }); + assert.deepEqual(heroProducts856[0]?.specifications, {}); + + assert.equal(normalizeRateLocale(' EN-US '), 'en'); + assert.equal(normalizeRateLocale('cs_CZ\u0000'), 'cs'); + assert.equal(normalizeRateLocale('vi-VN'), 'vi'); + assert.equal(normalizeRateLocale('fr-FR'), 'vi'); + console.log( + ' ✅ [97/97] Navigation, MerchantDirectory, Hero Suggestions & Rate i18n Bounds (#856) passed' ); } From c79942a5ff19bbfa30f930b83754d1a2c51bc819 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 20:52:57 +0700 Subject: [PATCH 13/19] fix(web): harden server rate deals radar and cached query key bounds (#858) (#859) Co-authored-by: JOY --- .../__tests__/core-rate-architecture.test.ts | 79 ++++++- apps/web/src/server/rate/deals.ts | 79 +++++-- apps/web/src/server/rate/index.ts | 200 +++++++++++++----- 3 files changed, 285 insertions(+), 73 deletions(-) diff --git a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts index e6ab77ddd..4a483b35c 100644 --- a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts +++ b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts @@ -8757,7 +8757,84 @@ async function runCoreArchitectureSuite() { assert.equal(normalizeRateLocale('vi-VN'), 'vi'); assert.equal(normalizeRateLocale('fr-FR'), 'vi'); console.log( - ' ✅ [97/97] Navigation, MerchantDirectory, Hero Suggestions & Rate i18n Bounds (#856) passed' + ' ✅ [97/98] Navigation, MerchantDirectory, Hero Suggestions & Rate i18n Bounds (#856) passed' + ); + } + + // ========================================================================= + // 98. Server Rate Deals Radar & Cached Query Key Bounds (#858) + // ========================================================================= + { + const { + MAX_DEAL_RADAR_CLUSTERS, + getCachedMasterProductOffers, + getCachedMasterProductReviews, + getCachedMerchantProfileById, + normalizeCachedFacetCategory, + normalizeDealRadarOptions, + sanitizeCachedCompareSlugsCsv, + sanitizeCachedEntityId, + sanitizeCachedInspectQuery, + sanitizeCachedProductSlug, + } = await import('../index'); + + assert.equal(normalizeCachedFacetCategory(undefined), 'smartphones'); + assert.equal(normalizeCachedFacetCategory(' '), 'smartphones'); + assert.equal( + normalizeCachedFacetCategory(' SMARTPHONES\u0000 '), + 'smartphones' + ); + assert.equal(normalizeCachedFacetCategory('tablets'), 'tablets'); + assert.equal(normalizeCachedFacetCategory('arbitrary-category'), 'all'); + + assert.equal(await getCachedMerchantProfileById(' \u0000 '), null); + assert.deepEqual(await getCachedMasterProductOffers(' \u0000 '), []); + assert.deepEqual(await getCachedMasterProductReviews(' \u0000 '), []); + + assert.deepEqual( + normalizeDealRadarOptions({ + categorySlug: ' SMARTPHONES\u0000 ', + platform: ' Shopee\u0007 ', + limit: 999, + }), + { + categoryFilter: 'smartphones', + platformFilter: 'shopee', + limit: 50, + } + ); + assert.deepEqual( + normalizeDealRadarOptions({ + categorySlug: 'arbitrary-invalid-category', + platform: 'bad/platform!@#', + limit: -5, + }), + { + categoryFilter: 'all', + platformFilter: 'all', + limit: 12, + } + ); + assert.equal(MAX_DEAL_RADAR_CLUSTERS, 500); + + assert.equal(sanitizeCachedEntityId(' m-123\u0000\u0007 '), 'm-123'); + assert.equal( + sanitizeCachedProductSlug(' iPhone-16-Pro\u0000 '), + 'iphone-16-pro' + ); + assert.equal(sanitizeCachedProductSlug('bad/slug/../../etc'), ''); + assert.equal( + sanitizeCachedCompareSlugsCsv( + ' iPhone-16\u0000 , bad/slug , GALAXY-S24 , iphone-16 , pixel-9 , xiaomi-15 , extra-6 ' + ), + 'iphone-16,galaxy-s24,pixel-9,xiaomi-15' + ); + assert.equal( + sanitizeCachedInspectQuery(' https://shopee.vn/item\u0000\n123 '), + 'https://shopee.vn/item 123' + ); + console.log( + ' ✅ [98/98] Server Rate Deals Radar & Cached Query Key Bounds (#858) passed' ); } diff --git a/apps/web/src/server/rate/deals.ts b/apps/web/src/server/rate/deals.ts index a25ad98c1..4d5e2b9d7 100644 --- a/apps/web/src/server/rate/deals.ts +++ b/apps/web/src/server/rate/deals.ts @@ -5,6 +5,7 @@ import { getMasterProductCategorySlug, getMasterProductOffersBatch, getMasterProducts, + isCanonicalCategorySlug, normalizeOfferOriginalPrice, resolveProductImageUrl, type CanonicalCategorySlug, @@ -129,6 +130,50 @@ function resolveEffectiveMerchantRisk( * 3. `priceSpreadLeaders`: Products with the largest absolute price spread (`max_price - min_price`) * across 10 retailers so buyers can avoid overpriced listings and buy at the lowest verified retailer. */ +const SAFE_PLATFORM_FILTER_REGEX = /^[a-z0-9_-]{1,40}$/; +export const MAX_DEAL_RADAR_CLUSTERS = 500; +export const MAX_OFFERS_PER_CLUSTER = 200; + +export function normalizeDealRadarOptions(options?: { + categorySlug?: string | null; + platform?: string | null; + limit?: number; +}): { + categoryFilter: CanonicalCategorySlug | 'all'; + platformFilter: string; + limit: number; +} { + const rawCat = + typeof options?.categorySlug === 'string' + ? options.categorySlug + .replace(/\p{Cc}+/gu, '') + .trim() + .toLowerCase() + : ''; + const categoryFilter: CanonicalCategorySlug | 'all' = + rawCat && isCanonicalCategorySlug(rawCat) ? rawCat : 'all'; + + const rawPlat = + typeof options?.platform === 'string' + ? options.platform + .replace(/\p{Cc}+/gu, '') + .trim() + .toLowerCase() + : ''; + const platformFilter = + rawPlat && rawPlat !== 'all' && SAFE_PLATFORM_FILTER_REGEX.test(rawPlat) + ? rawPlat + : 'all'; + + const rawLimit = Number(options?.limit); + const limit = + Number.isFinite(rawLimit) && rawLimit >= 1 + ? Math.min(Math.floor(rawLimit), 50) + : 12; + + return { categoryFilter, platformFilter, limit }; +} + export function computeCrossPlatformDealRadarFromData( clusters: Array<{ master: MasterProduct; @@ -140,14 +185,8 @@ export function computeCrossPlatformDealRadarFromData( limit?: number; } ): CrossPlatformDealRadarResponse { - const categoryFilter = options?.categorySlug || 'all'; - const platformFilter = - (options?.platform || 'all').trim().toLowerCase() || 'all'; - const rawLimit = Number(options?.limit); - const limit = - Number.isFinite(rawLimit) && rawLimit >= 1 - ? Math.min(Math.floor(rawLimit), 50) - : 12; + const { categoryFilter, platformFilter, limit } = + normalizeDealRadarOptions(options); const genuineDeals: CrossPlatformDealRadarItem[] = []; const fakeSaleTraps: CrossPlatformDealRadarItem[] = []; @@ -155,16 +194,24 @@ export function computeCrossPlatformDealRadarFromData( let totalClustersAnalyzed = 0; let totalOffersAnalyzed = 0; - for (const { master, offers } of clusters) { + const boundedClusters = Array.isArray(clusters) + ? clusters.slice(0, MAX_DEAL_RADAR_CLUSTERS) + : []; + + for (const { master, offers } of boundedClusters) { const categorySlug = getMasterProductCategorySlug(master); if (categoryFilter !== 'all' && categorySlug !== categoryFilter) { continue; } + const boundedOffers = Array.isArray(offers) + ? offers.slice(0, MAX_OFFERS_PER_CLUSTER) + : []; + // Bait-price listings (far below every other store) set no market floor, // spread, deal or trap (DOS/Rate#797). const validOffers = plausibleOffers( - offers.filter( + boundedOffers.filter( (o) => Number.isFinite(o.price_current) && o.price_current > 0 && @@ -472,13 +519,7 @@ export function getCrossPlatformDealRadar(options?: { platform?: string; limit?: number; }): Promise { - const categorySlug = options?.categorySlug || 'all'; - const platformFilter = - (options?.platform || 'all').trim().toLowerCase() || 'all'; - const rawLimit = Number(options?.limit); - const limit = - Number.isFinite(rawLimit) && rawLimit >= 1 - ? Math.min(Math.floor(rawLimit), 50) - : 12; - return _getCrossPlatformDealRadarByKey(categorySlug, platformFilter, limit); + const { categoryFilter, platformFilter, limit } = + normalizeDealRadarOptions(options); + return _getCrossPlatformDealRadarByKey(categoryFilter, platformFilter, limit); } diff --git a/apps/web/src/server/rate/index.ts b/apps/web/src/server/rate/index.ts index 021f5811f..2aaaf6153 100644 --- a/apps/web/src/server/rate/index.ts +++ b/apps/web/src/server/rate/index.ts @@ -138,30 +138,101 @@ const _getCachedMasterProductsByKey = cache( )(serializedParams) ); +const SAFE_PRODUCT_SLUG_REGEX = /^[a-z0-9._-]{1,120}$/; + +export function sanitizeCachedEntityId(value: unknown, maxLen = 128): string { + if (typeof value !== 'string') return ''; + return value + .replace(/\p{Cc}+/gu, '') + .trim() + .slice(0, maxLen); +} + +export function sanitizeCachedProductSlug(value: unknown): string { + if (typeof value !== 'string') return ''; + const cleaned = value + .replace(/\p{Cc}+/gu, '') + .trim() + .toLowerCase() + .slice(0, 120); + return SAFE_PRODUCT_SLUG_REGEX.test(cleaned) ? cleaned : ''; +} + +export function sanitizeCachedCompareSlugsCsv(slugsCsv: unknown): string { + if (typeof slugsCsv !== 'string') return ''; + return Array.from( + new Set( + slugsCsv + .split(',') + .map((s) => sanitizeCachedProductSlug(s)) + .filter(Boolean) + ) + ) + .slice(0, 4) + .join(','); +} + +export function sanitizeCachedInspectQuery(rawQuery: unknown): string { + if (typeof rawQuery !== 'string') return ''; + return rawQuery + .replace(/\p{Cc}+/gu, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, 2048); +} + +export function normalizeCachedFacetCategory( + categorySlug?: string | null +): import('../../lib/rate-api').CanonicalCategorySlug | 'all' { + if (categorySlug === undefined || categorySlug === null) { + return 'smartphones'; + } + const cleaned = + typeof categorySlug === 'string' + ? categorySlug + .replace(/\p{Cc}+/gu, '') + .trim() + .toLowerCase() + : ''; + if (!cleaned) return 'smartphones'; + return isCanonicalCategorySlug(cleaned) ? cleaned : 'all'; +} + export function getCachedMasterProducts(params?: MasterProductFilterParams) { - const cleanCat = params?.categorySlug?.trim().toLowerCase() || 'all'; + const rawCat = + typeof params?.categorySlug === 'string' + ? params.categorySlug + .replace(/\p{Cc}+/gu, '') + .trim() + .toLowerCase() + : ''; + const cleanCat = rawCat && isCanonicalCategorySlug(rawCat) ? rawCat : 'all'; return _getCachedMasterProductsByKey( canonicalizeCacheParams(params ?? {}), cleanCat ); } -export const getCachedMasterProductBySlug = cache((slug: string) => { - const cleanSlug = slug.trim().toLowerCase(); - return withRateDataCache( +const _getCachedMasterProductBySlugKey = cache((cleanSlug: string) => + withRateDataCache( _getMasterProductBySlug, ['rate-master-product-by-slug', cleanSlug], { revalidate: 300, tags: [RATE_CACHE_TAGS.PRODUCTS, productCacheTag(cleanSlug)], } - )(cleanSlug); -}); + )(cleanSlug) +); -export const getCachedCategoryDirectoryFacets = cache( - (categorySlug?: string) => { - const cleanCat = categorySlug?.trim().toLowerCase() || 'all'; - return withRateDataCache( +export function getCachedMasterProductBySlug(slug: string) { + const cleanSlug = sanitizeCachedProductSlug(slug); + if (!cleanSlug) return Promise.resolve(null); + return _getCachedMasterProductBySlugKey(cleanSlug); +} + +const _getCachedCategoryDirectoryFacetsByKey = cache( + (cleanCat: import('../../lib/rate-api').CanonicalCategorySlug | 'all') => + withRateDataCache( _getCategoryDirectoryFacets, ['rate-category-facets', cleanCat], { @@ -172,47 +243,61 @@ export const getCachedCategoryDirectoryFacets = cache( ...(cleanCat !== 'all' ? [categoryCacheTag(cleanCat)] : []), ], } - )(categorySlug); - } + )(cleanCat) ); -export const getCachedMasterProductOffers = cache((masterProductId: string) => { - const cleanId = masterProductId.trim(); - return withRateDataCache( +export function getCachedCategoryDirectoryFacets(categorySlug?: string) { + return _getCachedCategoryDirectoryFacetsByKey( + normalizeCachedFacetCategory(categorySlug) + ); +} + +const _getCachedMasterProductOffersByKey = cache((cleanId: string) => + withRateDataCache( _getMasterProductOffers, ['rate-master-product-offers', cleanId], { revalidate: 180, tags: [RATE_CACHE_TAGS.OFFERS, RATE_CACHE_TAGS.DEALS], } - )(cleanId); -}); + )(cleanId) +); -export const getCachedMasterProductReviews = cache( - (masterProductId: string) => { - const cleanId = masterProductId.trim(); - return withRateDataCache( - _getMasterProductReviews, - ['rate-master-product-reviews', cleanId], - { - revalidate: 300, - tags: [RATE_CACHE_TAGS.REVIEWS], - } - )(cleanId); - } +export function getCachedMasterProductOffers(masterProductId: string) { + const cleanId = sanitizeCachedEntityId(masterProductId); + if (!cleanId) return Promise.resolve([]); + return _getCachedMasterProductOffersByKey(cleanId); +} + +const _getCachedMasterProductReviewsByKey = cache((cleanId: string) => + withRateDataCache( + _getMasterProductReviews, + ['rate-master-product-reviews', cleanId], + { + revalidate: 300, + tags: [RATE_CACHE_TAGS.REVIEWS], + } + )(cleanId) ); +export function getCachedMasterProductReviews(masterProductId: string) { + const cleanId = sanitizeCachedEntityId(masterProductId); + if (!cleanId) return Promise.resolve([]); + return _getCachedMasterProductReviewsByKey(cleanId); +} + export const getCachedMasterProductReviewSummary = cache( ( masterProductId: string, existingReviews?: import('../../lib/rate-api').ReviewItem[] ) => { - const cleanId = masterProductId.trim(); + const cleanId = sanitizeCachedEntityId(masterProductId); const reviewsKey = existingReviews ? `reviews:${existingReviews.length}:${existingReviews .slice(0, 20) .map( - (r) => `${r.id}:${r.verdict ?? ''}:${r.include_in_rating ? 1 : 0}` + (r) => + `${sanitizeCachedEntityId(r.id)}:${sanitizeCachedEntityId(r.verdict ?? '', 32)}:${r.include_in_rating ? 1 : 0}` ) .join('|')}` : 'reviews:auto'; @@ -236,11 +321,14 @@ export const getCachedMasterProductPriceHistory = cache( masterProductId: string, existingOffers?: import('../../lib/rate-api').MerchantOffer[] ) => { - const cleanId = masterProductId.trim(); + const cleanId = sanitizeCachedEntityId(masterProductId); const offersKey = existingOffers ? `offers:${existingOffers.length}:${existingOffers .slice(0, 20) - .map((o) => `${o.id}:${o.price_current}:${o.price_original ?? ''}`) + .map( + (o) => + `${sanitizeCachedEntityId(o.id)}:${o.price_current}:${o.price_original ?? ''}` + ) .join('|')}` : 'offers:auto'; return withRateDataCache( @@ -277,21 +365,13 @@ const _getCachedCompareMasterProductsByKey = cache( ); export function getCachedCompareMasterProducts(slugsCsv: string) { - const canonicalSlugsCsv = Array.from( - new Set( - (slugsCsv || '') - .split(',') - .map((s) => s.trim().toLowerCase()) - .filter(Boolean) - ) - ) - .slice(0, 4) - .join(','); + const canonicalSlugsCsv = sanitizeCachedCompareSlugsCsv(slugsCsv); return _getCachedCompareMasterProductsByKey(canonicalSlugsCsv); } async function inspectProductOrShopUrlWithRadar(rawQuery: string) { - const res = await _inspectProductOrShopUrl(rawQuery); + const safeQuery = sanitizeCachedInspectQuery(rawQuery); + const res = await _inspectProductOrShopUrl(safeQuery); return { ...res, radarSignal: res.masterProduct @@ -351,17 +431,22 @@ export async function getCachedMerchantsList( return all; } -export const getCachedMerchantProfileById = cache((merchantId: string) => { - const cleanId = merchantId.trim(); - return withRateDataCache( +const _getCachedMerchantProfileByIdKey = cache((cleanId: string) => + withRateDataCache( _getMerchantProfileById, ['rate-merchant-profile', cleanId], { revalidate: 600, tags: [RATE_CACHE_TAGS.MERCHANTS, merchantCacheTag(cleanId)], } - )(cleanId); -}); + )(cleanId) +); + +export function getCachedMerchantProfileById(merchantId: string) { + const cleanId = sanitizeCachedEntityId(merchantId); + if (!cleanId) return Promise.resolve(null); + return _getCachedMerchantProfileByIdKey(cleanId); +} export const getCachedRatePlatformStats = cache(() => withRateDataCache(_getRatePlatformStats, ['rate-platform-stats'], { @@ -380,10 +465,16 @@ export const getCachedInspectProductOrShopUrl = cache( // JSON, so the category map travels as entries. const SITEMAP_CACHE_WARN_BYTES = 1_500_000; -const categorySlugOnly = (value: Record | null | undefined) => - typeof value?.category_slug === 'string' - ? { category_slug: value.category_slug } - : null; +const categorySlugOnly = ( + value: Record | null | undefined +) => { + if (typeof value?.category_slug !== 'string') return null; + const slug = value.category_slug + .replace(/\p{Cc}+/gu, '') + .trim() + .slice(0, 64); + return slug ? { category_slug: slug } : null; +}; const _getCachedSitemapRowsData = withRateDataCache( async () => { @@ -466,9 +557,12 @@ export { } from './openapi'; export { + MAX_DEAL_RADAR_CLUSTERS, + MAX_OFFERS_PER_CLUSTER, computeCrossPlatformDealRadarFromData, computeProductRadarSignal, getCrossPlatformDealRadar, + normalizeDealRadarOptions, type CrossPlatformDealRadarItem, type CrossPlatformDealRadarResponse, type CrossPlatformPriceSpreadItem, From 3aaf2dd91067060e6efda9129ce76297f6e4b166 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:09:40 +0700 Subject: [PATCH 14/19] fix(web): harden review vote eligibility, community review mapper, and openapi origin bounds (#860) (#861) Co-authored-by: JOY --- .../src/app/api/reviews/[id]/vote/route.ts | 54 ++++++-- apps/web/src/app/api/v1/rate/reviews/route.ts | 115 +++++++++++++----- .../__tests__/core-rate-architecture.test.ts | 87 ++++++++++++- apps/web/src/server/rate/openapi.ts | 13 +- 4 files changed, 224 insertions(+), 45 deletions(-) diff --git a/apps/web/src/app/api/reviews/[id]/vote/route.ts b/apps/web/src/app/api/reviews/[id]/vote/route.ts index f3606ad67..527cf1d5e 100644 --- a/apps/web/src/app/api/reviews/[id]/vote/route.ts +++ b/apps/web/src/app/api/reviews/[id]/vote/route.ts @@ -31,6 +31,14 @@ export async function OPTIONS() { const NOT_ELIGIBLE_AGE_REASON = 'Account must be at least 7 days old to vote or report reviews.'; +function isValidConfirmedTimestamp(raw: unknown): boolean { + if (typeof raw !== 'string') return false; + const trimmed = raw.trim(); + if (!trimmed || trimmed.length > 64) return false; + const ms = Date.parse(trimmed); + return Number.isFinite(ms) && ms > 0; +} + /** * Review Trust Model Section 9: Voter Eligibility Gate. The database applies * the same rules in rate.review_votes_guard (DOS/Rate#613, #771): @@ -40,42 +48,66 @@ const NOT_ELIGIBLE_AGE_REASON = * - Account must be at least 7 days old; a missing or unparseable * `created_at` fails closed. */ -export function evaluateVoterEligibility(user: { - created_at?: string | null; - email_confirmed_at?: string | null; - phone_confirmed_at?: string | null; - is_anonymous?: boolean | null; -}): { eligible: boolean; reason?: string } { +export function evaluateVoterEligibility( + user: + | { + created_at?: string | null; + email_confirmed_at?: string | null; + phone_confirmed_at?: string | null; + is_anonymous?: boolean | null; + } + | null + | undefined +): { eligible: boolean; reason?: string } { + if (!user || typeof user !== 'object') { + return { eligible: false, reason: NOT_ELIGIBLE_AGE_REASON }; + } if (user.is_anonymous === true) { return { eligible: false, reason: 'Anonymous accounts cannot vote on reviews.', }; } - if (!user.email_confirmed_at && !user.phone_confirmed_at) { + if ( + !isValidConfirmedTimestamp(user.email_confirmed_at) && + !isValidConfirmedTimestamp(user.phone_confirmed_at) + ) { return { eligible: false, reason: 'Verified email or phone number is required to vote on reviews.', }; } const createdMs = - typeof user.created_at === 'string' ? Date.parse(user.created_at) : NaN; - if (!Number.isFinite(createdMs)) { + typeof user.created_at === 'string' && user.created_at.trim().length <= 64 + ? Date.parse(user.created_at.trim()) + : NaN; + if (!Number.isFinite(createdMs) || createdMs <= 0) { return { eligible: false, reason: NOT_ELIGIBLE_AGE_REASON }; } const ageDays = (Date.now() - createdMs) / 86_400_000; - if (ageDays < 7) { + if (!Number.isFinite(ageDays) || ageDays < 7) { return { eligible: false, reason: NOT_ELIGIBLE_AGE_REASON }; } return { eligible: true }; } function toSafeVoteCount(value: unknown): number { + if (typeof value === 'boolean' || value === null || value === undefined) { + return 0; + } const n = Number(value); - return Number.isFinite(n) && n > 0 ? Math.floor(n) : 0; + return Number.isFinite(n) && n > 0 ? Math.min(100_000_000, Math.floor(n)) : 0; } function toSafeHelpfulnessWeight(value: unknown, fallback = 1): number { + if ( + typeof value === 'boolean' || + value === null || + value === undefined || + (typeof value === 'string' && !value.trim()) + ) { + return fallback; + } const n = Number(value); return Number.isFinite(n) ? Math.max(0, Math.min(10, n)) : fallback; } diff --git a/apps/web/src/app/api/v1/rate/reviews/route.ts b/apps/web/src/app/api/v1/rate/reviews/route.ts index 5c9f82297..4eef71c53 100644 --- a/apps/web/src/app/api/v1/rate/reviews/route.ts +++ b/apps/web/src/app/api/v1/rate/reviews/route.ts @@ -125,7 +125,13 @@ const SubmitReviewSchema = z.object({ .trim() .uuid('Mã sản phẩm chuẩn hóa (masterProductId) không hợp lệ.'), authorName: z.string().optional(), - ratingStar: z.coerce.number().optional(), + ratingStar: z.preprocess( + (val) => + val === null || (typeof val === 'string' && !val.trim()) + ? undefined + : val, + z.coerce.number().finite().optional() + ), comment: z .string() .trim() @@ -180,8 +186,9 @@ const RPC_REJECTIONS: Record = { function sanitizeCriteria(raw: Record | undefined) { const out: Record = {}; + if (!raw || typeof raw !== 'object') return out; for (const key of CRITERIA_KEYS) { - const val = Number(raw?.[key]); + const val = asNumber(raw[key], NaN); if (Number.isFinite(val) && val >= 1 && val <= 5) { out[key] = Math.round(val * 10) / 10; } @@ -190,10 +197,23 @@ function sanitizeCriteria(raw: Record | undefined) { } function asNumber(value: unknown, fallback: number): number { - const n = Number(value); + if ( + value === null || + value === undefined || + typeof value === 'boolean' || + (typeof value === 'string' && !value.trim()) + ) { + return fallback; + } + const n = typeof value === 'number' ? value : Number(String(value).trim()); return Number.isFinite(n) ? n : fallback; } +function toSafeCount(value: unknown): number { + const n = asNumber(value, 0); + return n > 0 ? Math.min(100_000_000, Math.floor(n)) : 0; +} + /** * Builds the response item from what rate.submit_community_review stored, so the * verdict and weight shown to the user are the ones the database applied. @@ -202,41 +222,75 @@ export function toReviewItem( record: Record, criteria: Record ): ReviewItem { - const verdict = record.verdict; - const flag = record.flag_category as ReviewFlagCategory; - const rawStar = asNumber(record.rating_star, 5); + const safeRecord = + record && typeof record === 'object' && !Array.isArray(record) + ? record + : {}; + const safeFallbackCriteria = + criteria && typeof criteria === 'object' && !Array.isArray(criteria) + ? sanitizeCriteria(criteria) + : {}; + const verdict = safeRecord.verdict; + const flag = safeRecord.flag_category as ReviewFlagCategory; + const rawStar = asNumber(safeRecord.rating_star, 5); const clampedStar = rawStar > 0 ? Math.max(1, Math.min(5, Math.round(rawStar))) : 0; const clampedWeight = Math.max( 0, - Math.min(10, asNumber(record.helpfulness_weight, 0)) + Math.min(10, asNumber(safeRecord.helpfulness_weight, 0)) ); const sanitizedRecordCriteria = - record.criteria_ratings && - typeof record.criteria_ratings === 'object' && - !Array.isArray(record.criteria_ratings) - ? sanitizeCriteria(record.criteria_ratings as Record) + safeRecord.criteria_ratings && + typeof safeRecord.criteria_ratings === 'object' && + !Array.isArray(safeRecord.criteria_ratings) + ? sanitizeCriteria(safeRecord.criteria_ratings as Record) : {}; const effectiveCriteria = Object.keys(sanitizedRecordCriteria).length > 0 ? sanitizedRecordCriteria - : criteria; + : safeFallbackCriteria; + const rawCreatedAt = + typeof safeRecord.created_at === 'string' + ? safeRecord.created_at.trim() + : ''; + const createdMs = + rawCreatedAt && rawCreatedAt.length <= 64 ? Date.parse(rawCreatedAt) : NaN; + const createdAt = + Number.isFinite(createdMs) && createdMs > 0 + ? rawCreatedAt + : new Date().toISOString(); + const cleanComment = + typeof safeRecord.comment === 'string' + ? safeRecord.comment.trim().slice(0, 3000) + : ''; + const cleanVariation = + typeof safeRecord.variation === 'string' + ? safeRecord.variation.trim().slice(0, 120) || null + : null; + const cleanReason = + typeof safeRecord.reason === 'string' + ? safeRecord.reason.trim().slice(0, 500) || null + : null; + const rawFakeProb = asNumber(safeRecord.fake_probability, NaN); return { - id: String(record.id), - author_name: String(record.author_name ?? ''), + id: + typeof safeRecord.id === 'string' || typeof safeRecord.id === 'number' + ? String(safeRecord.id).trim().slice(0, 120) + : '', + author_name: + typeof safeRecord.author_name === 'string' + ? safeRecord.author_name.trim().slice(0, 80) + : '', rating_star: clampedStar, - comment: String(record.comment ?? ''), - variation: typeof record.variation === 'string' ? record.variation : null, - created_at: - typeof record.created_at === 'string' - ? record.created_at - : new Date().toISOString(), - reviewer_type: record.reviewer_type === 'expert' ? 'expert' : 'user', - upvotes: 0, - downvotes: 0, - report_count: 0, + comment: cleanComment, + variation: cleanVariation, + created_at: createdAt, + reviewer_type: safeRecord.reviewer_type === 'expert' ? 'expert' : 'user', + upvotes: toSafeCount(safeRecord.upvotes), + downvotes: toSafeCount(safeRecord.downvotes), + report_count: toSafeCount(safeRecord.report_count), helpfulness_weight: clampedWeight, - include_in_rating: record.include_in_rating === true, + include_in_rating: safeRecord.include_in_rating === true, criteria_ratings: effectiveCriteria, verdict: verdict === 'authentic' || @@ -245,16 +299,15 @@ export function toReviewItem( verdict === 'unanalyzed' ? verdict : null, - fake_probability: - record.fake_probability == null - ? null - : Math.max(0, Math.min(1, asNumber(record.fake_probability, 0))), + fake_probability: Number.isFinite(rawFakeProb) + ? Math.max(0, Math.min(1, rawFakeProb)) + : null, flag_category: FLAG_CATEGORIES.has(flag) ? flag : 'authentic', - reason: typeof record.reason === 'string' ? record.reason : null, + reason: cleanReason, signals: { source: 'rate_community_jreviews', criteria_count: Object.keys(effectiveCriteria).length, - comment_length: String(record.comment ?? '').length, + comment_length: cleanComment.length, }, }; } diff --git a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts index 4a483b35c..d63a64582 100644 --- a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts +++ b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts @@ -8834,7 +8834,92 @@ async function runCoreArchitectureSuite() { 'https://shopee.vn/item 123' ); console.log( - ' ✅ [98/98] Server Rate Deals Radar & Cached Query Key Bounds (#858) passed' + ' ✅ [98/99] Server Rate Deals Radar & Cached Query Key Bounds (#858) passed' + ); + } + + // ========================================================================= + // 99. Review Vote Eligibility, Community Review Mapper & OpenAPI Origin Bounds (#860) + // ========================================================================= + { + const { evaluateVoterEligibility } = + await import('../../../app/api/reviews/[id]/vote/route'); + const { toReviewItem } = + await import('../../../app/api/v1/rate/reviews/route'); + const { normalizeOpenApiServerOrigin } = await import('../openapi'); + + assert.equal(evaluateVoterEligibility(null).eligible, false); + assert.equal(evaluateVoterEligibility(undefined).eligible, false); + assert.equal( + evaluateVoterEligibility({ + created_at: '2026-01-01T00:00:00Z', + email_confirmed_at: ' ', + phone_confirmed_at: 'not-a-timestamp', + }).eligible, + false + ); + assert.equal( + evaluateVoterEligibility({ + created_at: '1970-01-01T00:00:00Z', + email_confirmed_at: '2026-01-01T00:00:00Z', + }).eligible, + false + ); + assert.equal( + evaluateVoterEligibility({ + created_at: new Date(Date.now() - 8 * 86_400_000).toISOString(), + email_confirmed_at: '2026-01-01T00:00:00Z', + }).eligible, + true + ); + + const mappedNullStar = toReviewItem( + { + id: ' rev-1 ', + author_name: ' Nguyen Van A ', + rating_star: null, + comment: ' Đánh giá chi tiết sản phẩm ', + variation: ' ', + created_at: 'invalid-date', + upvotes: '12', + downvotes: -3, + report_count: '2', + helpfulness_weight: null, + fake_probability: '', + reason: ' ', + }, + { screen: 4.56, battery: 99 } + ); + assert.equal(mappedNullStar.id, 'rev-1'); + assert.equal(mappedNullStar.author_name, 'Nguyen Van A'); + assert.equal( + mappedNullStar.rating_star, + 5, + 'null rating_star must fall back to default 5 instead of coercing to 0' + ); + assert.equal(mappedNullStar.variation, null); + assert.ok(Number.isFinite(Date.parse(mappedNullStar.created_at))); + assert.equal(mappedNullStar.upvotes, 12); + assert.equal(mappedNullStar.downvotes, 0); + assert.equal(mappedNullStar.report_count, 2); + assert.equal(mappedNullStar.fake_probability, null); + assert.equal(mappedNullStar.reason, null); + assert.deepEqual(mappedNullStar.criteria_ratings, { screen: 4.6 }); + + assert.equal( + normalizeOpenApiServerOrigin(123 as unknown as string), + 'https://rate.vn' + ); + assert.equal( + normalizeOpenApiServerOrigin('https://user:pass@rate.vn/api'), + 'https://rate.vn' + ); + assert.equal( + normalizeOpenApiServerOrigin('https://staging.rate.vn/path'), + 'https://staging.rate.vn' + ); + console.log( + ' ✅ [99/99] Review Vote Eligibility, Community Review Mapper & OpenAPI Origin Bounds (#860) passed' ); } diff --git a/apps/web/src/server/rate/openapi.ts b/apps/web/src/server/rate/openapi.ts index 6b4ddfa2f..2dfd687d9 100644 --- a/apps/web/src/server/rate/openapi.ts +++ b/apps/web/src/server/rate/openapi.ts @@ -76,13 +76,22 @@ export interface RateOpenApiDocument { export function normalizeOpenApiServerOrigin( rawBaseUrl?: string | null ): string { - const trimmed = (rawBaseUrl ?? '').trim(); - if (!trimmed || trimmed.length > 256) return 'https://rate.vn'; + if (typeof rawBaseUrl !== 'string') return 'https://rate.vn'; + const trimmed = rawBaseUrl.trim(); + if (!trimmed || trimmed.length > 256 || /[\s\p{Cc}]/u.test(trimmed)) { + return 'https://rate.vn'; + } try { const parsed = new URL(trimmed); if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { return 'https://rate.vn'; } + if (parsed.username || parsed.password) { + return 'https://rate.vn'; + } + if (!parsed.origin || parsed.origin === 'null') { + return 'https://rate.vn'; + } return parsed.origin; } catch { return 'https://rate.vn'; From f4785afd00cc3a81956166e8da7afc1b4cbb82a5 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:27:23 +0700 Subject: [PATCH 15/19] fix(importer): harden peer floor, merchant risk, and arena/entity bounds (#862) (#863) Co-authored-by: JOY --- .../__tests__/backfill-review-risk.test.ts | 102 +++++++++++++++++- apps/importer/src/shopee/arena-spec-syncer.ts | 32 ++++-- .../importer/src/shopee/cluster-peer-floor.ts | 44 ++++++-- apps/importer/src/shopee/entity-matcher.ts | 3 + .../src/shopee/merchant-review-risk.ts | 95 ++++++++++++---- 5 files changed, 239 insertions(+), 37 deletions(-) diff --git a/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts b/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts index 3b0636043..13d9418f6 100644 --- a/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts +++ b/apps/importer/src/shopee/__tests__/backfill-review-risk.test.ts @@ -2,8 +2,23 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { AiReviewAnalyzer } from '../ai-review-analyzer.js'; +import { + extractGsmArenaDataSpecs, + MAX_ARENA_CATALOG_ITEMS, + parseGsmArenaBrandCatalogHtml, + parseGsmArenaHtmlToSpecs, + parsePhoneArenaCatalogHtml, +} from '../arena-spec-syncer.js'; +import { MAX_CLUSTER_PEER_ROWS, resolveClusterPeerFloorPrice } from '../cluster-peer-floor.js'; +import { MAX_ENTITY_IMAGES, sanitizeHttpUrl, sanitizeHttpUrlList } from '../entity-matcher.js'; import { computeRetryDelayMs, redactSensitiveString, sanitizeLogMeta } from '../logger.js'; -import { loadAndScoreMerchant, rescoreMerchantReviewRisk } from '../merchant-review-risk.js'; +import { + loadAndScoreMerchant, + rescoreMerchantReviewRisk, + toValidPrice, + uniqueIds, + writeMerchantReviewRisk, +} from '../merchant-review-risk.js'; import { mergeKeySpecs } from '../jobs/backfill-specs.js'; import { mapStoredReview, @@ -534,6 +549,91 @@ async function run() { assert.equal(Object.hasOwn(keyBoundedSpecs, 'K'.repeat(120)), false); assert.deepEqual(keyBoundedSpecs.nested, { 'refresh rate': '120Hz' }); + // DOS/Rate#862: resolveClusterPeerFloorPrice, merchant-review-risk, and Arena/Entity URL bounds + assert.equal( + resolveClusterPeerFloorPrice(null as unknown as never[], 'p-1', 25_000_000), + 25_000_000, + 'Non-array peerRows falls back to fallbackMasterMinPrice instead of throwing' + ); + assert.equal( + resolveClusterPeerFloorPrice( + [ + null as unknown as never, + { id: ' p-self ', price_current: 15_000_000, merchants: { is_official: true } }, + { id: 'p-bool', price_current: true, merchants: { is_official: true } }, + { id: 'p-peer', price_current: 28_990_000, merchants: { is_official: false } }, + ], + 'p-self', + true as unknown as number + ), + 28_990_000, + 'Self-exclusion trims IDs and boolean prices/fallbacks are rejected' + ); + assert.equal(MAX_CLUSTER_PEER_ROWS, 1000); + + assert.equal(toValidPrice(true), 0, 'toValidPrice rejects boolean true'); + assert.equal(mapStoredReview(null as unknown as ReviewRow), null); + assert.equal(mapStoredReview({ ...review(' ', 'p', null) }), null); + assert.equal( + mapStoredReview({ + ...review(' r-trim ', 'p', null), + rating_star: true as unknown as number, + })?.raw.rating_star, + 5, + 'boolean rating_star falls back to 5 instead of coercing true -> 1' + ); + + const boundedScoreMerchants = scoreMerchants( + new AiReviewAnalyzer(), + [ + { + id: 'm-prev-bad', + platform: 'shopee', + is_official: false, + follower_count: true as unknown as number, + review_risk_score: 999, + }, + ], + [{ id: 'p-prev', merchant_id: 'm-prev-bad', cluster_id: 'c1', price_current: 10_000_000 }], + [review('r-prev', 'p-prev', { verdict: 'authentic', flag_category: 'authentic' })], + new Map() + ); + assert.equal( + boundedScoreMerchants[0]?.previousScore, + null, + 'Out-of-range previousScore > 100 is normalized to null' + ); + + assert.deepEqual(uniqueIds([' m-2 ', '', ' ', 'm-1\u0000bad', 'm-1', 'm-2']), ['m-1', 'm-2']); + assert.equal(await loadAndScoreMerchant(scoped.client, new AiReviewAnalyzer(), ' '), null); + await assert.rejects( + () => + writeMerchantReviewRisk( + scoped.client, + ' ', + boundedScoreMerchants[0]!.risk, + '2026-10-24T00:00:00.000Z' + ), + /Invalid merchantId/ + ); + + assert.equal(sanitizeHttpUrl('https://user:pass@example.com/a.jpg'), null); + const cappedImgs = sanitizeHttpUrlList( + Array.from({ length: 70 }, (_, i) => `https://cdn.example.com/img-${i}.jpg`) + ); + assert.equal(cappedImgs.length, MAX_ENTITY_IMAGES); + + const parsedArenaCreds = parseGsmArenaHtmlToSpecs('

Test Phone

', { + gsmarenaUrl: 'https://user:pass@www.gsmarena.com/test-1.php', + phonearenaUrl: 'https://admin:secret@www.phonearena.com/phones/Test_id1', + }); + assert.equal(parsedArenaCreds.gsmarenaUrl, undefined); + assert.equal(parsedArenaCreds.phonearenaUrl, undefined); + assert.deepEqual(extractGsmArenaDataSpecs(null as unknown as string), {}); + assert.deepEqual(parseGsmArenaBrandCatalogHtml(null as unknown as string), []); + assert.deepEqual(parsePhoneArenaCatalogHtml(null as unknown as string), []); + assert.equal(MAX_ARENA_CATALOG_ITEMS, 500); + console.log('✅ Review risk backfill suite passed'); } diff --git a/apps/importer/src/shopee/arena-spec-syncer.ts b/apps/importer/src/shopee/arena-spec-syncer.ts index 89c8cec95..96183c22a 100644 --- a/apps/importer/src/shopee/arena-spec-syncer.ts +++ b/apps/importer/src/shopee/arena-spec-syncer.ts @@ -83,12 +83,15 @@ function toBoundedNumber( return val; } +export const MAX_ARENA_HTML_LENGTH = 2_000_000; +export const MAX_ARENA_CATALOG_ITEMS = 500; + function normalizeArenaUrl( rawUrl: string | undefined, hostPattern: RegExp, baseOrigin?: string ): string | undefined { - const trimmed = rawUrl?.trim(); + const trimmed = typeof rawUrl === 'string' ? rawUrl.trim() : ''; if (!trimmed || trimmed.length > 2048 || /\s/.test(trimmed)) { return undefined; } @@ -100,6 +103,7 @@ function normalizeArenaUrl( const parsed = baseOrigin ? new URL(trimmed, baseOrigin) : new URL(trimmed); if ( (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') || + Boolean(parsed.username || parsed.password) || !hostPattern.test(parsed.hostname) ) { return undefined; @@ -120,6 +124,9 @@ function stripHtmlTags(raw: string): string { * Extracts all `[data-spec="key"]` values from a GSMArena specification page HTML. */ export function extractGsmArenaDataSpecs(html: string): Record { + if (typeof html !== 'string' || !html) return {}; + const safeHtml = + html.length > MAX_ARENA_HTML_LENGTH ? html.slice(0, MAX_ARENA_HTML_LENGTH) : html; const specs: Record = {}; // Scan opening tags, then search for the matching close from there. A single // regex with `[^>]*\bdata-spec=` and a lazy body backtracks polynomially on @@ -130,7 +137,7 @@ export function extractGsmArenaDataSpecs(html: string): Record { // per-tag cursor finds the first close after each opening in amortized // constant time and the whole scan stays linear. const closesByTag = new Map(); - for (const close of html.matchAll(/<\/([a-z0-9]+)>/gi)) { + for (const close of safeHtml.matchAll(/<\/([a-z0-9]+)>/gi)) { const tag = close[1].toLowerCase(); const list = closesByTag.get(tag) ?? []; const at = close.index ?? 0; @@ -139,7 +146,7 @@ export function extractGsmArenaDataSpecs(html: string): Record { } const cursors = new Map(); let match: RegExpExecArray | null; - while ((match = openTag.exec(html)) !== null) { + while ((match = openTag.exec(safeHtml)) !== null) { const keyMatch = /\bdata-spec=["']([^"']+)["']/i.exec(match[2] || ''); if (!keyMatch) continue; const tag = match[1].toLowerCase(); @@ -151,7 +158,7 @@ export function extractGsmArenaDataSpecs(html: string): Record { const close = list[cursor]; if (!close) continue; const key = keyMatch[1]?.trim(); - const val = stripHtmlTags(html.slice(openTag.lastIndex, close.at)); + const val = stripHtmlTags(safeHtml.slice(openTag.lastIndex, close.at)); if (key && val && !specs[key]) { specs[key] = val; } @@ -416,13 +423,16 @@ export function parseGsmArenaHtmlToSpecs( * and merges/overrides split camera & hardware fields. */ export function extractPhoneArenaTableSpecs(html: string): Record { + if (typeof html !== 'string' || !html) return {}; + const safeHtml = + html.length > MAX_ARENA_HTML_LENGTH ? html.slice(0, MAX_ARENA_HTML_LENGTH) : html; const out: Record = {}; // Cell bodies cannot contain another tr/th/td tag, so each attempt stops at // the next cell instead of rescanning the rest of the page (polynomial-redos). const rowRegex = /]*>\s*<(?:th|td)[^<>]*>((?:(?!<\/?t[rhd]\b)[\s\S])*)<\/(?:th|td)>\s*]*>((?:(?!<\/?t[rhd]\b)[\s\S])*)<\/td>\s*<\/tr>/gi; let match: RegExpExecArray | null; - while ((match = rowRegex.exec(html)) !== null) { + while ((match = rowRegex.exec(safeHtml)) !== null) { const label = stripHtmlTags(match[1] || '') .replace(/:$/, '') .toLowerCase(); @@ -449,8 +459,11 @@ export function parseGsmArenaBrandCatalogHtml( html: string, brandHint?: string ): DiscoveredArenaCatalogItem[] { + if (typeof html !== 'string' || !html) return []; + const safeHtml = + html.length > MAX_ARENA_HTML_LENGTH ? html.slice(0, MAX_ARENA_HTML_LENGTH) : html; const results: DiscoveredArenaCatalogItem[] = []; - const trimmedBrand = brandHint?.trim() || ''; + const trimmedBrand = typeof brandHint === 'string' ? brandHint.trim() : ''; const brandPrefixRe = trimmedBrand ? new RegExp(`^${escapeRegExp(trimmedBrand)}(?:\\s|$)`, 'i') : null; @@ -459,7 +472,7 @@ export function parseGsmArenaBrandCatalogHtml( const itemRegex = /]*>\s*]+\.php)["'][^<>]*>((?:(?!<\/?(?:a|li)\b)[\s\S])*)<\/a>\s*<\/li>/gi; let match: RegExpExecArray | null; - while ((match = itemRegex.exec(html)) !== null) { + while (results.length < MAX_ARENA_CATALOG_ITEMS && (match = itemRegex.exec(safeHtml)) !== null) { const href = match[1]?.trim() || ''; const inner = match[2] || ''; const nameMatch = inner.match(/]*>((?:(?!<\/?span\b)[\s\S])*)<\/span>/i); @@ -496,6 +509,9 @@ export function parseGsmArenaBrandCatalogHtml( * Parses a PhoneArena device finder HTML block to discover newly announced devices. */ export function parsePhoneArenaCatalogHtml(html: string): DiscoveredArenaCatalogItem[] { + if (typeof html !== 'string' || !html) return []; + const safeHtml = + html.length > MAX_ARENA_HTML_LENGTH ? html.slice(0, MAX_ARENA_HTML_LENGTH) : html; const results: DiscoveredArenaCatalogItem[] = []; // Match the whole first (attributes cannot span tags, the body cannot // contain another ), then read href from the attributes. One regex with @@ -505,7 +521,7 @@ export function parsePhoneArenaCatalogHtml(html: string): DiscoveredArenaCatalog /(?:^|\s)href=["'](https?:\/\/www\.phonearena\.com\/phones\/[^"'#?]+|\/phones\/[^"'#?]+)["']/i; const seen = new Set(); let match: RegExpExecArray | null; - while ((match = linkRegex.exec(html)) !== null) { + while (results.length < MAX_ARENA_CATALOG_ITEMS && (match = linkRegex.exec(safeHtml)) !== null) { const rawHref = hrefRegex.exec(match[1] || '')?.[1]?.trim() || ''; const text = stripHtmlTags(match[2] || ''); if (!rawHref || !text || text.length < 4) continue; diff --git a/apps/importer/src/shopee/cluster-peer-floor.ts b/apps/importer/src/shopee/cluster-peer-floor.ts index 14ebab714..e0704b046 100644 --- a/apps/importer/src/shopee/cluster-peer-floor.ts +++ b/apps/importer/src/shopee/cluster-peer-floor.ts @@ -1,5 +1,7 @@ import { MAX_VALID_VND_PRICE } from '@repo/shared-data'; +export const MAX_CLUSTER_PEER_ROWS = 1000; + export interface ClusterPeerPriceCandidate { id?: unknown; price_current?: unknown; @@ -19,17 +21,32 @@ export function resolveClusterPeerFloorPrice( ): number | undefined { let officialMin = Number.POSITIVE_INFINITY; let peerMin = Number.POSITIVE_INFINITY; + const safeCurrentId = typeof currentProductId === 'string' ? currentProductId.trim() : ''; - for (const row of peerRows) { - if (row.id && String(row.id) === currentProductId) continue; - const price = Number(row.price_current || 0); - if (!Number.isFinite(price) || price <= 0 || price > MAX_VALID_VND_PRICE) continue; - if (price < peerMin) { - peerMin = price; - } - const mObj = Array.isArray(row.merchants) ? row.merchants[0] : row.merchants; - if (mObj?.is_official && price < officialMin) { - officialMin = price; + if (Array.isArray(peerRows)) { + const limit = Math.min(peerRows.length, MAX_CLUSTER_PEER_ROWS); + for (let i = 0; i < limit; i++) { + const row = peerRows[i]; + if (!row || typeof row !== 'object') continue; + const rowId = + typeof row.id === 'string' || typeof row.id === 'number' ? String(row.id).trim() : ''; + if (safeCurrentId && rowId === safeCurrentId) continue; + if ( + typeof row.price_current === 'boolean' || + row.price_current === null || + row.price_current === undefined + ) { + continue; + } + const price = Number(row.price_current); + if (!Number.isFinite(price) || price <= 0 || price > MAX_VALID_VND_PRICE) continue; + if (price < peerMin) { + peerMin = price; + } + const mObj = Array.isArray(row.merchants) ? row.merchants[0] : row.merchants; + if (mObj?.is_official && price < officialMin) { + officialMin = price; + } } } @@ -39,6 +56,13 @@ export function resolveClusterPeerFloorPrice( if (Number.isFinite(peerMin)) { return peerMin; } + if ( + typeof fallbackMasterMinPrice === 'boolean' || + fallbackMasterMinPrice === null || + fallbackMasterMinPrice === undefined + ) { + return undefined; + } const fb = Number(fallbackMasterMinPrice); return Number.isFinite(fb) && fb > 0 && fb <= MAX_VALID_VND_PRICE ? fb : undefined; } diff --git a/apps/importer/src/shopee/entity-matcher.ts b/apps/importer/src/shopee/entity-matcher.ts index 1103734a7..a6abcb815 100644 --- a/apps/importer/src/shopee/entity-matcher.ts +++ b/apps/importer/src/shopee/entity-matcher.ts @@ -27,6 +27,7 @@ const MAX_RAM_GB = 2048; const MAX_STORAGE_GB = 65536; const MAX_SCREEN_SIZE_INCH = 120; const MAX_BATTERY_MAH = 500_000; +export const MAX_ENTITY_IMAGES = 50; function hasControlOrSpaceChar(val: string): boolean { for (let i = 0; i < val.length; i++) { @@ -43,6 +44,7 @@ export function sanitizeHttpUrl(raw: unknown): string | null { try { const parsed = new URL(trimmed); if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null; + if (parsed.username || parsed.password) return null; if (!parsed.hostname) return null; return trimmed; } catch { @@ -55,6 +57,7 @@ export function sanitizeHttpUrlList(urls: unknown): string[] { const seen = new Set(); const out: string[] = []; for (const item of urls) { + if (out.length >= MAX_ENTITY_IMAGES) break; const valid = sanitizeHttpUrl(item); if (valid && !seen.has(valid)) { seen.add(valid); diff --git a/apps/importer/src/shopee/merchant-review-risk.ts b/apps/importer/src/shopee/merchant-review-risk.ts index 0bbe68ee5..88eed4266 100644 --- a/apps/importer/src/shopee/merchant-review-risk.ts +++ b/apps/importer/src/shopee/merchant-review-risk.ts @@ -78,6 +78,9 @@ export interface ReviewRow { export function mapStoredReview( row: ReviewRow ): { result: AnalyzedReviewResult; raw: ReviewInputItem } | null { + if (!row || typeof row !== 'object') return null; + const reviewId = typeof row.id === 'string' ? row.id.trim() : ''; + if (!reviewId) return null; const vf = Array.isArray(row.review_flags) ? row.review_flags[0] : row.review_flags; const rawVerdict = typeof vf?.verdict === 'string' ? vf.verdict.trim().toLowerCase() : null; if (rawVerdict === 'unanalyzed') return null; @@ -104,12 +107,17 @@ export function mapStoredReview( const includeInRating = Boolean( vf?.include_in_rating ?? (category === 'authentic' && verdict !== 'fake') ); - const parsedStar = Number(row.rating_star); + const parsedStar = + typeof row.rating_star === 'boolean' || + row.rating_star === null || + row.rating_star === undefined + ? Number.NaN + : Number(row.rating_star); const ratingStar = Number.isFinite(parsedStar) && parsedStar >= 1 && parsedStar <= 5 ? Math.round(parsedStar) : 5; return { result: { - review_id: row.id, + review_id: reviewId, verdict, flag_category: category, fake_probability: 0, @@ -119,7 +127,7 @@ export function mapStoredReview( signals: {} as AnalyzedReviewResult['signals'], }, raw: { - id: row.id, + id: reviewId, rating_star: ratingStar, comment: typeof row.comment === 'string' ? row.comment.slice(0, 4000) : '', }, @@ -128,6 +136,7 @@ export function mapStoredReview( /** A usable VND price, or 0. */ export const toValidPrice = (raw: unknown): number => { + if (typeof raw === 'boolean' || raw === null || raw === undefined) return 0; const n = Number(raw); return Number.isFinite(n) && n > 0 && n <= MAX_VALID_VND_PRICE ? n : 0; }; @@ -225,7 +234,12 @@ export function scoreMerchants( officialByMerchant, masterMinByCluster ); - const rawFollowers = Number(merchant.follower_count); + const rawFollowers = + typeof merchant.follower_count === 'boolean' || + merchant.follower_count === null || + merchant.follower_count === undefined + ? Number.NaN + : Number(merchant.follower_count); const safeFollowerCount = Number.isFinite(rawFollowers) && rawFollowers >= 0 && @@ -242,15 +256,34 @@ export function scoreMerchants( reviewResults: bucket.results, rawReviews: bucket.raws, }); + const rawPrev = + typeof merchant.review_risk_score === 'boolean' || + merchant.review_risk_score === null || + merchant.review_risk_score === undefined + ? Number.NaN + : Number(merchant.review_risk_score); + const previousScore = + Number.isFinite(rawPrev) && rawPrev >= 0 && rawPrev <= 100 ? rawPrev : null; scores.push({ merchantId, - previousScore: merchant.review_risk_score ?? null, + previousScore, risk, }); } return scores.sort((a, b) => a.merchantId.localeCompare(b.merchantId)); } +function normalizeSafeId(raw: unknown): string | null { + if (typeof raw !== 'string') return null; + const trimmed = raw.trim(); + if (!trimmed || trimmed.length > 128) return null; + for (let i = 0; i < trimmed.length; i++) { + const code = trimmed.charCodeAt(i); + if (code <= 0x1f || code === 0x7f) return null; + } + return trimmed; +} + /** Writes only the review risk columns (and updated_at) on rate.merchants. */ export async function writeMerchantReviewRisk( supabase: RateSupabaseClient, @@ -258,17 +291,32 @@ export async function writeMerchantReviewRisk( risk: MerchantRiskEvaluation, nowIso = new Date().toISOString() ): Promise { + const safeMerchantId = normalizeSafeId(merchantId); + if (!safeMerchantId) { + throw new Error('Invalid merchantId for writeMerchantReviewRisk'); + } + const rawScore = + typeof (risk as { risk_score?: unknown })?.risk_score === 'boolean' + ? Number.NaN + : Number(risk?.risk_score); + const safeScore = Number.isFinite(rawScore) + ? Math.min(100, Math.max(0, Math.round(rawScore))) + : 0; + const safeNowIso = + typeof nowIso === 'string' && nowIso.trim() && Number.isFinite(Date.parse(nowIso.trim())) + ? new Date(nowIso.trim()).toISOString() + : new Date().toISOString(); const { error } = await supabase .from('merchants') .update({ - review_risk_score: risk.risk_score, + review_risk_score: safeScore, review_risk_breakdown: merchantRiskBreakdown(risk), - review_risk_updated_at: nowIso, - updated_at: nowIso, + review_risk_updated_at: safeNowIso, + updated_at: safeNowIso, }) - .eq('id', merchantId); + .eq('id', safeMerchantId); if (error) { - throw new Error(`Failed to write review risk for ${merchantId}: ${error.message}`); + throw new Error(`Failed to write review risk for ${safeMerchantId}: ${error.message}`); } } @@ -305,8 +353,15 @@ async function fetchAllByIds( return rows; } -const uniqueIds = (values: ReadonlyArray): string[] => - [...new Set(values.filter((v): v is string => Boolean(v)))].sort(); +export const uniqueIds = (values: ReadonlyArray): string[] => { + if (!Array.isArray(values)) return []; + const seen = new Set(); + for (const v of values) { + const safe = normalizeSafeId(v); + if (safe) seen.add(safe); + } + return [...seen].sort(); +}; /** * Scores one seller from the same inputs the backfill reads for it: its @@ -319,10 +374,12 @@ export async function loadAndScoreMerchant( analyzer: AiReviewAnalyzer, merchantId: string ): Promise { + const safeMerchantId = normalizeSafeId(merchantId); + if (!safeMerchantId) return null; const { data: merchant, error: merchantErr } = await supabase .from('merchants') .select(MERCHANT_RISK_SELECT) - .eq('id', merchantId) + .eq('id', safeMerchantId) .maybeSingle(); if (merchantErr) throw new Error(`merchants: ${merchantErr.message}`); if (!merchant) return null; @@ -332,7 +389,7 @@ export async function loadAndScoreMerchant( 'products', PRODUCT_RISK_SELECT, 'merchant_id', - [merchantId] + [safeMerchantId] ); const clusterIds = uniqueIds(listings.map(p => p.cluster_id)); // The peers' official flag comes embedded, so no per-merchant read. @@ -351,7 +408,7 @@ export async function loadAndScoreMerchant( const rows: PeerRow[] = [...listings, ...peers]; for (const { merchants: embedded, ...product } of rows) { productById.set(product.id, product); - if (!product.merchant_id || product.merchant_id === merchantId) continue; + if (!product.merchant_id || product.merchant_id === safeMerchantId) continue; const official = Array.isArray(embedded) ? embedded[0]?.is_official : embedded?.is_official; peerMerchants.set(product.merchant_id, { id: product.merchant_id, @@ -387,7 +444,7 @@ export async function loadAndScoreMerchant( reviews, masterMinByCluster ); - return scores.find(s => s.merchantId === merchantId) ?? null; + return scores.find(s => s.merchantId === safeMerchantId) ?? null; } /** @@ -399,7 +456,9 @@ export async function rescoreMerchantReviewRisk( analyzer: AiReviewAnalyzer, merchantId: string ): Promise { - const score = await loadAndScoreMerchant(supabase, analyzer, merchantId); - if (score) await writeMerchantReviewRisk(supabase, merchantId, score.risk); + const safeMerchantId = normalizeSafeId(merchantId); + if (!safeMerchantId) return null; + const score = await loadAndScoreMerchant(supabase, analyzer, safeMerchantId); + if (score) await writeMerchantReviewRisk(supabase, safeMerchantId, score.risk); return score; } From 82b10156848c39bd9a5107728f0674f0c182bd32 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:47:04 +0700 Subject: [PATCH 16/19] fix(web): harden API handler price/IP bounds, watchlist offers, and crowd ingest URLs (#864) (#866) Co-authored-by: JOY --- .../app/api/v1/rate/extension/ingest/route.ts | 5 +- .../src/app/api/v1/rate/watchlist/route.ts | 89 +------- .../__tests__/core-rate-architecture.test.ts | 118 ++++++++++ apps/web/src/server/rate/api-handler.ts | 212 ++++++++++++++++-- 4 files changed, 326 insertions(+), 98 deletions(-) diff --git a/apps/web/src/app/api/v1/rate/extension/ingest/route.ts b/apps/web/src/app/api/v1/rate/extension/ingest/route.ts index dbdc0f66b..ac4b6873b 100644 --- a/apps/web/src/app/api/v1/rate/extension/ingest/route.ts +++ b/apps/web/src/app/api/v1/rate/extension/ingest/route.ts @@ -7,6 +7,7 @@ import { import { BoundedSlidingWindowRateLimiter, extractClientRateLimitKey, + isSafeCrowdIngestHttpUrl, jsonApiError, } from '@/server/rate/api-handler'; import { parseEcommerceUrlOrQuery } from '@/lib/rate-api'; @@ -43,8 +44,8 @@ const CrowdIngestSchema = z.object({ .url() .max(1024) .refine( - (u) => /^https?:\/\//i.test(u), - 'Product URL must use http:// or https://' + (u) => isSafeCrowdIngestHttpUrl(u), + 'Product URL must use http:// or https:// without embedded credentials' ), title: z.string().trim().min(3).max(320), selectedVariant: z.string().trim().max(160).nullish(), diff --git a/apps/web/src/app/api/v1/rate/watchlist/route.ts b/apps/web/src/app/api/v1/rate/watchlist/route.ts index 4191b655b..9ef597ab7 100644 --- a/apps/web/src/app/api/v1/rate/watchlist/route.ts +++ b/apps/web/src/app/api/v1/rate/watchlist/route.ts @@ -1,10 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; -import { plausibleMinPrice, plausibleOffers } from '@/lib/offer-plausibility'; import { z } from 'zod'; -import { - MAX_VALID_VND_PRICE, - normalizeOfferOriginalPrice, -} from '@/lib/rate-api'; +import { MAX_VALID_VND_PRICE } from '@/lib/rate-api'; import { computeProductRadarSignal, getCachedMasterProductBySlug, @@ -17,6 +13,8 @@ import { jsonApiError, parseWatchlistSlugsParam, resolveWatchlistMasterPriceBounds, + selectBestOfferFromListings, + withPlausibleMinPrice, } from '@/server/rate/api-handler'; import { authenticateRateRequest, @@ -64,83 +62,6 @@ const RPC_REJECTIONS: Record = { }, }; -export interface WatchlistBestOffer { - id: string; - title: string; - price: number; - original_price: number; - url: string; - platform: string; - merchant_name: string; - is_official_mall: boolean; -} - -// The stored min_price can be a bait-price listing; target checks use the -// lowest plausible offer instead (DOS/Rate#797). -function withPlausibleMinPrice( - master: T, - offers: readonly { price_current: number }[] -): T { - return { - ...master, - min_price: plausibleMinPrice(Number(master.min_price || 0), offers), - }; -} - -function selectBestOfferFromListings( - offers: Awaited> -): { bestOffer: WatchlistBestOffer | null; offerPlatforms: string[] } { - // A bait-price listing is never the best offer (DOS/Rate#797). - const valid = plausibleOffers(offers).filter((o) => { - const price = Number(o.price_current); - return ( - Number(o.stock ?? 1) > 0 && - Number.isFinite(price) && - price > 0 && - price <= MAX_VALID_VND_PRICE - ); - }); - const platforms = Array.from( - new Set( - valid - .map((o) => - String(o.merchant?.platform || o.platform || 'shopee').toLowerCase() - ) - .filter(Boolean) - ) - ); - if (valid.length === 0) { - return { bestOffer: null, offerPlatforms: platforms }; - } - - const sorted = [...valid].sort((a, b) => { - const diff = Number(a.price_current) - Number(b.price_current); - if (diff !== 0) return diff; - const mallA = a.merchant?.is_official ? 1 : 0; - const mallB = b.merchant?.is_official ? 1 : 0; - return mallB - mallA; - }); - - const best = sorted[0]!; - const bestPrice = Number(best.price_current); - const validOrig = normalizeOfferOriginalPrice(bestPrice, best.price_original); - return { - bestOffer: { - id: best.id, - title: best.title, - price: bestPrice, - original_price: validOrig ?? bestPrice, - url: best.url, - platform: String( - best.merchant?.platform || best.platform || 'shopee' - ).toLowerCase(), - merchant_name: String(best.merchant?.name || 'Official Store'), - is_official_mall: Boolean(best.merchant?.is_official), - }, - offerPlatforms: platforms, - }; -} - /** * GET /api/v1/rate/watchlist?slugs=apple-iphone-16-128gb,apple-iphone-16-pro-max-256gb * Evaluates live deal status across all 10 platforms for a list of watched product slugs, @@ -343,12 +264,14 @@ export async function POST(request: NextRequest) { } if (!rpcError && rpcData) { + const safeRpcData = + typeof rpcData === 'object' && !Array.isArray(rpcData) ? rpcData : {}; return NextResponse.json( { ok: true, persistedInCloud: true, data: { - ...(typeof rpcData === 'object' ? rpcData : {}), + ...safeRpcData, best_offer: bestOffer, offer_platforms: offerPlatforms, deal_radar: radarExportFields(radarSignal), diff --git a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts index d63a64582..c89728516 100644 --- a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts +++ b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts @@ -8923,6 +8923,124 @@ async function runCoreArchitectureSuite() { ); } + // 100. Issue #864: API Handler Price/IP Bounds, Watchlist Offer Selection, and Crowd Ingest URL Checks + { + const { + extractClientRateLimitKey, + isSafeCrowdIngestHttpUrl, + parseOptionalVndPriceParam, + resolveWatchlistMasterPriceBounds, + selectBestOfferFromListings, + withPlausibleMinPrice, + } = await import('../api-handler'); + + assert.equal( + parseOptionalVndPriceParam('15000000', Number.NaN), + 15_000_000, + 'parseOptionalVndPriceParam must fall back to 500M cap when maxVnd is NaN' + ); + assert.deepEqual( + resolveWatchlistMasterPriceBounds(null, 24_990_000, Number.NaN), + { + currentMinPrice: 24_990_000, + currentMaxPrice: 24_990_000, + }, + 'null master must fall back to bestOfferPrice without throwing' + ); + assert.deepEqual( + resolveWatchlistMasterPriceBounds( + { min_price: true, max_price: true }, + 24_990_000 + ), + { + currentMinPrice: 24_990_000, + currentMaxPrice: 24_990_000, + }, + 'boolean min_price/max_price must not coerce to 1 VND' + ); + + assert.equal(extractClientRateLimitKey(null), 'anon'); + const spoofedHeaders = { + get(name: string) { + return name.toLowerCase() === 'x-forwarded-for' + ? ' 203.0.113.10\r\n\u0000 , 10.0.0.1 ' + : null; + }, + } as unknown as Headers; + assert.equal(extractClientRateLimitKey(spoofedHeaders), '203.0.113.10'); + const wsHeaders = new Headers({ + 'x-forwarded-for': ' 203.0.113.10\t99 , 10.0.0.1 ', + }); + assert.equal(extractClientRateLimitKey(wsHeaders), '203.0.113.1099'); + + const plausibleBoolMaster = withPlausibleMinPrice( + { min_price: true as unknown as number }, + [{ price_current: 28_990_000 }] + ); + assert.equal( + plausibleBoolMaster.min_price, + 28_990_000, + 'withPlausibleMinPrice must ignore boolean min_price and use plausible offer price' + ); + + const bestOfferRes = selectBestOfferFromListings([ + { + id: 'off-bool', + title: 'Boolean Price Offer', + price_current: true as unknown as number, + price_original: 0, + stock: 10, + url: 'https://shopee.vn/product/1/2', + platform: 'shopee', + merchant: { name: 'Bad Shop', platform: 'shopee', is_official: true }, + }, + { + id: ' off-valid ', + title: ' Apple\u0000 iPhone 16 128GB ', + price_current: 21_990_000, + price_original: 22_990_000, + stock: 5, + url: ' https://shopee.vn/product/88201679/27110898667 ', + platform: 'shopee', + merchant: { + name: ' Apple\u0007 Flagship Store ', + platform: 'Shopee\u0000', + is_official: true, + }, + }, + ] as unknown as Parameters[0]); + assert.equal(bestOfferRes.bestOffer?.id, 'off-valid'); + assert.equal(bestOfferRes.bestOffer?.title, 'Apple iPhone 16 128GB'); + assert.equal( + bestOfferRes.bestOffer?.merchant_name, + 'Apple Flagship Store' + ); + assert.equal(bestOfferRes.bestOffer?.platform, 'shopee'); + assert.deepEqual(bestOfferRes.offerPlatforms, ['shopee']); + + assert.equal( + isSafeCrowdIngestHttpUrl( + 'https://shopee.vn/product-i.88201679.27110898667' + ), + true + ); + assert.equal( + isSafeCrowdIngestHttpUrl( + 'https://user:pass@shopee.vn/product-i.88201679.27110898667' + ), + false, + 'isSafeCrowdIngestHttpUrl must reject embedded URL credentials' + ); + assert.equal( + isSafeCrowdIngestHttpUrl('ftp://shopee.vn/product-i.1.2'), + false + ); + + console.log( + ' ✅ [100/100] API Handler Price/IP Bounds, Watchlist Offer Selection & Crowd Ingest URL Checks (#864) passed' + ); + } + console.log('🎉 All Rate.vn Core Architecture tests passed!'); } diff --git a/apps/web/src/server/rate/api-handler.ts b/apps/web/src/server/rate/api-handler.ts index 0aac9cb89..62dac7aef 100644 --- a/apps/web/src/server/rate/api-handler.ts +++ b/apps/web/src/server/rate/api-handler.ts @@ -1,6 +1,11 @@ import 'server-only'; import { NextResponse } from 'next/server'; import { z } from 'zod'; +import { plausibleMinPrice, plausibleOffers } from '@/lib/offer-plausibility'; +import { + MAX_VALID_VND_PRICE, + normalizeOfferOriginalPrice, +} from '@/lib/rate-api'; export const PUBLIC_API_CORS_HEADERS: Record = { 'Access-Control-Allow-Origin': '*', @@ -110,12 +115,16 @@ export function parseOptionalVndPriceParam( raw: string | null | undefined, maxVnd = 500_000_000 ): number | null { - if (raw == null) return null; + if (raw == null || typeof raw !== 'string') return null; const trimmed = raw.trim(); if (!trimmed) return null; const n = Number(trimmed); if (!Number.isFinite(n) || n <= 0) return null; - return Math.min(Math.round(n), maxVnd); + const safeMaxVnd = + typeof maxVnd === 'number' && Number.isFinite(maxVnd) && maxVnd > 0 + ? Math.round(maxVnd) + : 500_000_000; + return Math.min(Math.round(n), safeMaxVnd); } /** @@ -152,33 +161,202 @@ export function parseBoundedPositiveIntArrayParam( * when `master.min_price` is missing, non-positive, or exceeds `maxVnd` (1B VND). */ export function resolveWatchlistMasterPriceBounds( - master: { min_price?: unknown; max_price?: unknown }, + master: { min_price?: unknown; max_price?: unknown } | null | undefined, bestOfferPrice?: number | null, maxVnd = 1_000_000_000 ): { currentMinPrice: number; currentMaxPrice: number } { - const rawMin = Number(master.min_price); + const safeMaxVnd = + typeof maxVnd === 'number' && Number.isFinite(maxVnd) && maxVnd > 0 + ? Math.round(maxVnd) + : 1_000_000_000; + const rawMin = + !master || + typeof master !== 'object' || + typeof master.min_price === 'boolean' || + master.min_price === null || + master.min_price === undefined + ? Number.NaN + : Number(master.min_price); const safeBestOffer = typeof bestOfferPrice === 'number' && Number.isFinite(bestOfferPrice) && bestOfferPrice > 0 && - bestOfferPrice <= maxVnd + bestOfferPrice <= safeMaxVnd ? bestOfferPrice : 0; const currentMinPrice = - Number.isFinite(rawMin) && rawMin > 0 && rawMin <= maxVnd + Number.isFinite(rawMin) && rawMin > 0 && rawMin <= safeMaxVnd ? rawMin : safeBestOffer; - const rawMax = Number(master.max_price); + const rawMax = + !master || + typeof master !== 'object' || + typeof master.max_price === 'boolean' || + master.max_price === null || + master.max_price === undefined + ? Number.NaN + : Number(master.max_price); const currentMaxPrice = currentMinPrice > 0 && Number.isFinite(rawMax) && rawMax >= currentMinPrice && - rawMax <= maxVnd + rawMax <= safeMaxVnd ? rawMax : currentMinPrice; return { currentMinPrice, currentMaxPrice }; } +export interface WatchlistBestOffer { + id: string; + title: string; + price: number; + original_price: number; + url: string; + platform: string; + merchant_name: string; + is_official_mall: boolean; +} + +export interface WatchlistCandidateOffer { + id?: unknown; + title?: unknown; + price_current: number; + price_original?: unknown; + stock?: unknown; + url?: unknown; + platform?: unknown; + merchant?: { + name?: unknown; + platform?: unknown; + is_official?: unknown; + } | null; +} + +// The stored min_price can be a bait-price listing; target checks use the +// lowest plausible offer instead (DOS/Rate#797). +export function withPlausibleMinPrice( + master: T, + offers: readonly { price_current: number }[] +): T { + const rawMin = + !master || + typeof master !== 'object' || + typeof master.min_price === 'boolean' || + master.min_price === null || + master.min_price === undefined + ? 0 + : Number(master.min_price || 0); + return { + ...master, + min_price: plausibleMinPrice( + Number.isFinite(rawMin) ? rawMin : 0, + Array.isArray(offers) ? offers : [] + ), + }; +} + +export function selectBestOfferFromListings( + offers: readonly WatchlistCandidateOffer[] +): { bestOffer: WatchlistBestOffer | null; offerPlatforms: string[] } { + const safeOffers = Array.isArray(offers) ? offers : []; + // A bait-price listing is never the best offer (DOS/Rate#797). + const valid = plausibleOffers(safeOffers).filter((o) => { + if (!o || typeof o !== 'object') return false; + if (typeof o.price_current === 'boolean' || typeof o.stock === 'boolean') { + return false; + } + const price = Number(o.price_current); + const stock = o.stock == null ? 1 : Number(o.stock); + return ( + Number.isFinite(stock) && + stock > 0 && + Number.isFinite(price) && + price > 0 && + price <= MAX_VALID_VND_PRICE + ); + }); + const platforms = Array.from( + new Set( + valid + .map((o) => + String(o.merchant?.platform || o.platform || 'shopee') + .replace(/\p{Cc}+/gu, '') + .trim() + .toLowerCase() + ) + .filter(Boolean) + ) + ); + if (valid.length === 0) { + return { bestOffer: null, offerPlatforms: platforms }; + } + + const sorted = [...valid].sort((a, b) => { + const diff = Number(a.price_current) - Number(b.price_current); + if (diff !== 0) return diff; + const mallA = a.merchant?.is_official ? 1 : 0; + const mallB = b.merchant?.is_official ? 1 : 0; + return mallB - mallA; + }); + + const best = sorted[0]!; + const bestPrice = Number(best.price_current); + const validOrig = normalizeOfferOriginalPrice( + bestPrice, + typeof best.price_original === 'boolean' + ? 0 + : Number(best.price_original ?? 0) + ); + const cleanTitle = String(best.title || '') + .replace(/\p{Cc}+/gu, ' ') + .trim() + .slice(0, 320); + const cleanMerchantName = String(best.merchant?.name || 'Official Store') + .replace(/\p{Cc}+/gu, ' ') + .trim() + .slice(0, 120); + return { + bestOffer: { + id: String(best.id || '').trim(), + title: cleanTitle, + price: bestPrice, + original_price: validOrig ?? bestPrice, + url: typeof best.url === 'string' ? best.url.trim().slice(0, 2048) : '', + platform: String(best.merchant?.platform || best.platform || 'shopee') + .replace(/\p{Cc}+/gu, '') + .trim() + .toLowerCase(), + merchant_name: cleanMerchantName || 'Official Store', + is_official_mall: Boolean(best.merchant?.is_official), + }, + offerPlatforms: platforms, + }; +} + +export function isSafeCrowdIngestHttpUrl(rawUrl: string): boolean { + if (typeof rawUrl !== 'string') return false; + const trimmed = rawUrl.trim(); + if ( + !trimmed || + trimmed.length > 1024 || + /\s/.test(trimmed) || + /\p{Cc}/u.test(trimmed) + ) { + return false; + } + try { + const parsed = new URL(trimmed); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + return false; + } + if (parsed.username || parsed.password) return false; + if (!parsed.hostname) return false; + return true; + } catch { + return false; + } +} + const CANONICAL_PRODUCT_SLUG_REGEX = /^[a-zA-Z0-9._-]+$/; /** @@ -313,15 +491,23 @@ export class BoundedSlidingWindowRateLimiter { } export function extractClientRateLimitKey( - headers: Pick + headers: Pick | null | undefined ): string { + if (!headers || typeof headers.get !== 'function') return 'anon'; + const sanitizeIpCandidate = (raw: string | null | undefined): string => { + if (typeof raw !== 'string') return ''; + return raw + .replace(/\p{Cc}+/gu, '') + .replace(/\s+/g, '') + .slice(0, 128); + }; const forwardedFor = headers.get('x-forwarded-for'); if (forwardedFor) { - const firstIp = forwardedFor.split(',')[0]?.trim(); - if (firstIp) return firstIp.slice(0, 128); + const firstIp = sanitizeIpCandidate(forwardedFor.split(',')[0]); + if (firstIp) return firstIp; } - const realIp = headers.get('x-real-ip')?.trim(); - if (realIp) return realIp.slice(0, 128); + const realIp = sanitizeIpCandidate(headers.get('x-real-ip')); + if (realIp) return realIp; return 'anon'; } From 5eaeb5fbf3f512f41723b007e59f62bf7f59e954 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:58:59 +0700 Subject: [PATCH 17/19] fix(web): deal radar counts every offer of a cluster (#867) (#869) #859 sliced each cluster to its first 200 offers before computing the market min, median, max and spread. Offers arrive sorted by price, so a product with more than 200 offers lost its priciest listings and its spread (the catalog E2E "reads page past the 1000-row cap" saw 0 and has failed on dev since). Drop the per-cluster slice; keep the cluster cap and option normalisation from #859. Guard: a 250-offer cluster with the priciest offer last still reports it. Co-authored-by: JOY Co-authored-by: Claude Opus 5.5 --- .../__tests__/core-rate-architecture.test.ts | 43 +++++++++++++++++++ apps/web/src/server/rate/deals.ts | 9 ++-- apps/web/src/server/rate/index.ts | 1 - 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts index c89728516..4eaee9e9c 100644 --- a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts +++ b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts @@ -8817,6 +8817,49 @@ async function runCoreArchitectureSuite() { ); assert.equal(MAX_DEAL_RADAR_CLUSTERS, 500); + // Every offer of a cluster counts: with the priciest of 250 offers last, + // the spread still reaches it (DOS/Rate#867; a 200-offer slice made it 0). + { + const { computeCrossPlatformDealRadarFromData: radar867 } = + await import('../index'); + const offers867 = Array.from({ length: 250 }, (_, i) => ({ + id: 'o867-' + i, + platform: i % 2 ? 'shopee' : 'lazada', + url: 'https://example.test/867/' + i, + price_current: i < 249 ? 10_000_000 : 12_000_000, + price_original: null, + merchant: { + id: 'm867-' + i, + name: 'Shop ' + i, + risk_score: 5, + is_official: true, + }, + })); + const feed867 = radar867( + [ + { + master: { + id: 'id-867', + slug: 'phone-867', + name: 'Phone 867', + brand: 'Brand', + category: 'smartphones', + min_price: 10_000_000, + max_price: 12_000_000, + }, + offers: offers867, + }, + ] as never, + { categorySlug: 'all', limit: 10 } + ); + assert.equal(feed867.totalOffersAnalyzed, 250); + const spread867 = feed867.priceSpreadLeaders.find( + (s) => s.productSlug === 'phone-867' + ); + assert.ok(spread867, 'a 20% spread across 250 offers is reported'); + assert.equal(spread867.maxPrice, 12_000_000); + } + assert.equal(sanitizeCachedEntityId(' m-123\u0000\u0007 '), 'm-123'); assert.equal( sanitizeCachedProductSlug(' iPhone-16-Pro\u0000 '), diff --git a/apps/web/src/server/rate/deals.ts b/apps/web/src/server/rate/deals.ts index 4d5e2b9d7..162ed438b 100644 --- a/apps/web/src/server/rate/deals.ts +++ b/apps/web/src/server/rate/deals.ts @@ -132,7 +132,6 @@ function resolveEffectiveMerchantRisk( */ const SAFE_PLATFORM_FILTER_REGEX = /^[a-z0-9_-]{1,40}$/; export const MAX_DEAL_RADAR_CLUSTERS = 500; -export const MAX_OFFERS_PER_CLUSTER = 200; export function normalizeDealRadarOptions(options?: { categorySlug?: string | null; @@ -204,14 +203,14 @@ export function computeCrossPlatformDealRadarFromData( continue; } - const boundedOffers = Array.isArray(offers) - ? offers.slice(0, MAX_OFFERS_PER_CLUSTER) - : []; + // Every offer counts: offers arrive sorted by price, so a slice would keep + // only the cheapest and understate the spread (DOS/Rate#867, #333). + const clusterOffers = Array.isArray(offers) ? offers : []; // Bait-price listings (far below every other store) set no market floor, // spread, deal or trap (DOS/Rate#797). const validOffers = plausibleOffers( - boundedOffers.filter( + clusterOffers.filter( (o) => Number.isFinite(o.price_current) && o.price_current > 0 && diff --git a/apps/web/src/server/rate/index.ts b/apps/web/src/server/rate/index.ts index 2aaaf6153..65e31807a 100644 --- a/apps/web/src/server/rate/index.ts +++ b/apps/web/src/server/rate/index.ts @@ -558,7 +558,6 @@ export { export { MAX_DEAL_RADAR_CLUSTERS, - MAX_OFFERS_PER_CLUSTER, computeCrossPlatformDealRadarFromData, computeProductRadarSignal, getCrossPlatformDealRadar, From aaa730f0a47a13704a04384deec22eea134147f3 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:05:21 +0700 Subject: [PATCH 18/19] fix(web): harden inspect observed prices, health TTL clock guard, and revalidate tokens (#868) (#872) Co-authored-by: JOY --- apps/web/src/app/api/v1/rate/deals/route.ts | 12 ++- apps/web/src/app/api/v1/rate/inspect/route.ts | 36 ++----- .../src/app/api/v1/rate/leaderboard/route.ts | 7 +- .../src/app/api/v1/rate/revalidate/route.ts | 12 ++- apps/web/src/server/health.ts | 14 ++- .../__tests__/core-rate-architecture.test.ts | 93 +++++++++++++++++++ apps/web/src/server/rate/api-handler.ts | 45 +++++++++ 7 files changed, 181 insertions(+), 38 deletions(-) diff --git a/apps/web/src/app/api/v1/rate/deals/route.ts b/apps/web/src/app/api/v1/rate/deals/route.ts index da1d7d3ee..8046d21e4 100644 --- a/apps/web/src/app/api/v1/rate/deals/route.ts +++ b/apps/web/src/app/api/v1/rate/deals/route.ts @@ -5,6 +5,7 @@ import { PUBLIC_API_CORS_HEADERS, jsonApiError, parseBoundedIntParam, + parseBoundedStringParam, } from '@/server/rate/api-handler'; const VALID_CATEGORIES = new Set([ @@ -39,9 +40,10 @@ export async function GET(request: Request) { try { const { searchParams } = new URL(request.url); const rawCat = ( - searchParams.get('category') || - searchParams.get('cat') || - 'all' + parseBoundedStringParam( + searchParams.get('category') || searchParams.get('cat'), + 32 + ) || 'all' ) .trim() .toLowerCase(); @@ -51,7 +53,9 @@ export async function GET(request: Request) { ? (rawCat as CanonicalCategorySlug) : 'all'; - const rawPlatform = (searchParams.get('platform') || 'all') + const rawPlatform = ( + parseBoundedStringParam(searchParams.get('platform'), 32) || 'all' + ) .trim() .toLowerCase(); const platform = VALID_PLATFORMS.has(rawPlatform) ? rawPlatform : 'all'; diff --git a/apps/web/src/app/api/v1/rate/inspect/route.ts b/apps/web/src/app/api/v1/rate/inspect/route.ts index cf1181ad9..26443b058 100644 --- a/apps/web/src/app/api/v1/rate/inspect/route.ts +++ b/apps/web/src/app/api/v1/rate/inspect/route.ts @@ -12,6 +12,7 @@ import { jsonApiError, parseBoundedStringParam, parseOptionalVndPriceParam, + resolveInspectObservedPrices, } from '@/server/rate/api-handler'; export async function OPTIONS() { @@ -51,32 +52,15 @@ export async function GET(request: NextRequest) { const inspection = await getCachedInspectProductOrShopUrl(rawQuery); const canonicalUrl = canonicalizeProductUrl(rawQuery); - const rawObservedPrice = Number( - queryPrice ?? - inspection.matchedOffer?.price_current ?? - inspection.masterProduct?.min_price ?? - 0 - ); - const observedPrice = - Number.isFinite(rawObservedPrice) && - rawObservedPrice > 0 && - rawObservedPrice <= MAX_VALID_VND_PRICE - ? Math.round(rawObservedPrice) - : 0; - const rawObservedOriginalPrice = Number( - queryOriginalPrice ?? - (queryPrice === null - ? inspection.matchedOffer?.price_original - : null) ?? - 0 - ); - const observedOriginalPrice = - Number.isFinite(rawObservedOriginalPrice) && - rawObservedOriginalPrice > 0 && - rawObservedOriginalPrice >= observedPrice && - rawObservedOriginalPrice <= MAX_VALID_VND_PRICE - ? Math.round(rawObservedOriginalPrice) - : 0; + const { observedPrice, observedOriginalPrice } = + resolveInspectObservedPrices({ + queryPrice, + queryOriginalPrice, + matchedOfferPriceCurrent: inspection.matchedOffer?.price_current, + matchedOfferPriceOriginal: inspection.matchedOffer?.price_original, + masterMinPrice: inspection.masterProduct?.min_price, + maxVnd: MAX_VALID_VND_PRICE, + }); const gatekeeper = detectAccessoryOrJunk({ title: diff --git a/apps/web/src/app/api/v1/rate/leaderboard/route.ts b/apps/web/src/app/api/v1/rate/leaderboard/route.ts index dbc379c1c..f167ba98d 100644 --- a/apps/web/src/app/api/v1/rate/leaderboard/route.ts +++ b/apps/web/src/app/api/v1/rate/leaderboard/route.ts @@ -31,9 +31,10 @@ export async function GET(req: NextRequest) { try { const { searchParams } = new URL(req.url); const rawCat = ( - searchParams.get('category') || - searchParams.get('cat') || - 'smartphones' + parseBoundedStringParam( + searchParams.get('category') || searchParams.get('cat'), + 32 + ) || 'smartphones' ) .toLowerCase() .trim(); diff --git a/apps/web/src/app/api/v1/rate/revalidate/route.ts b/apps/web/src/app/api/v1/rate/revalidate/route.ts index 0780dd710..e969e9546 100644 --- a/apps/web/src/app/api/v1/rate/revalidate/route.ts +++ b/apps/web/src/app/api/v1/rate/revalidate/route.ts @@ -62,7 +62,10 @@ export function verifyRevalidateSecret( !cleanExpected || cleanProvided.length > MAX_REVALIDATE_TOKEN_LENGTH || cleanExpected.length > MAX_REVALIDATE_TOKEN_LENGTH || - /\s/.test(cleanProvided) + /\s/.test(cleanProvided) || + /\s/.test(cleanExpected) || + /\p{Cc}/u.test(cleanProvided) || + /\p{Cc}/u.test(cleanExpected) ) { return false; } @@ -75,7 +78,9 @@ export function extractProvidedToken(request: NextRequest): string | null { const headerToken = request.headers.get('x-rate-revalidate-token'); if (headerToken && headerToken.trim()) { const trimmed = headerToken.trim(); - return trimmed.length <= MAX_REVALIDATE_TOKEN_LENGTH && !/\s/.test(trimmed) + return trimmed.length <= MAX_REVALIDATE_TOKEN_LENGTH && + !/\s/.test(trimmed) && + !/\p{Cc}/u.test(trimmed) ? trimmed : null; } @@ -84,7 +89,8 @@ export function extractProvidedToken(request: NextRequest): string | null { const trimmed = authHeader.replace(/^Bearer\s+/i, '').trim(); return trimmed.length > 0 && trimmed.length <= MAX_REVALIDATE_TOKEN_LENGTH && - !/\s/.test(trimmed) + !/\s/.test(trimmed) && + !/\p{Cc}/u.test(trimmed) ? trimmed : null; } diff --git a/apps/web/src/server/health.ts b/apps/web/src/server/health.ts index 921727966..665c772ac 100644 --- a/apps/web/src/server/health.ts +++ b/apps/web/src/server/health.ts @@ -53,11 +53,21 @@ async function probeDatabase(): Promise { * callers share one probe, and a result is reused for DATABASE_CHECK_TTL_MS. */ export async function checkDatabase(now = Date.now()): Promise { - if (cached && now - cached.at < DATABASE_CHECK_TTL_MS) return cached.status; + const safeNow = + typeof now === 'number' && Number.isFinite(now) && now > 0 + ? now + : Date.now(); + if ( + cached && + safeNow >= cached.at && + safeNow - cached.at < DATABASE_CHECK_TTL_MS + ) { + return cached.status; + } inFlight ??= probeDatabase() .catch((): CheckStatus => 'error') .then((status) => { - cached = { status, at: Date.now() }; + cached = { status, at: safeNow }; return status; }) .finally(() => { diff --git a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts index 4eaee9e9c..4196573c9 100644 --- a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts +++ b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts @@ -9084,6 +9084,99 @@ async function runCoreArchitectureSuite() { ); } + // 101. Issue #868: Inspect Observed Price Bounds, Health TTL Clock Guard & Revalidate Token Checks + { + const { NextRequest } = await import('next/server'); + const { resolveInspectObservedPrices } = await import('../api-handler'); + const { extractProvidedToken, verifyRevalidateSecret } = + await import('../../../app/api/v1/rate/revalidate/route'); + + assert.deepEqual( + resolveInspectObservedPrices({ + queryPrice: null, + queryOriginalPrice: null, + matchedOfferPriceCurrent: true, + matchedOfferPriceOriginal: true, + masterMinPrice: 18_990_000, + }), + { + observedPrice: 18_990_000, + observedOriginalPrice: 0, + }, + 'resolveInspectObservedPrices must reject boolean price_current/price_original and fall back to masterMinPrice' + ); + assert.deepEqual( + resolveInspectObservedPrices({ + queryPrice: null, + queryOriginalPrice: null, + matchedOfferPriceCurrent: 22_990_000.4, + matchedOfferPriceOriginal: 25_990_000.4, + masterMinPrice: 20_000_000, + }), + { + observedPrice: 22_990_000, + observedOriginalPrice: 25_990_000, + } + ); + assert.deepEqual( + resolveInspectObservedPrices({ + queryPrice: 24_990_000, + queryOriginalPrice: null, + matchedOfferPriceCurrent: 22_990_000, + matchedOfferPriceOriginal: 25_990_000, + masterMinPrice: 20_000_000, + }), + { + observedPrice: 24_990_000, + observedOriginalPrice: 0, + }, + 'when queryPrice is provided without queryOriginalPrice, matchedOfferPriceOriginal must not be attached' + ); + + assert.equal( + verifyRevalidateSecret('secret\u0000123', 'secret\u0000123'), + false + ); + assert.equal(verifyRevalidateSecret('secret 123', 'secret 123'), false); + assert.equal( + verifyRevalidateSecret('aaaa-aaaa-aaaa', 'aaaa-aaaa-aaaa'), + true + ); + assert.equal( + extractProvidedToken( + new NextRequest('https://rate.vn/api/v1/rate/revalidate', { + headers: { 'x-rate-revalidate-token': 'tok\u0007en' }, + }) + ), + null + ); + + let probes868 = 0; + await withHealthDatabase( + async () => { + probes868++; + return new Response(JSON.stringify([{ id: 'cat-1' }]), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }, + async () => { + assert.equal(await healthMod.checkDatabase(100_000), 'ok'); + assert.equal(probes868, 1); + // Same window: cached + assert.equal(await healthMod.checkDatabase(105_000), 'ok'); + assert.equal(probes868, 1); + // Backward clock jump (now < cached.at): must not blindly trust a future-stamped cache + assert.equal(await healthMod.checkDatabase(50_000), 'ok'); + assert.equal(probes868, 2); + } + ); + + console.log( + ' ✅ [101/101] Inspect Observed Price Bounds, Health TTL Clock Guard & Revalidate Token Checks (#868) passed' + ); + } + console.log('🎉 All Rate.vn Core Architecture tests passed!'); } diff --git a/apps/web/src/server/rate/api-handler.ts b/apps/web/src/server/rate/api-handler.ts index 62dac7aef..16b4ef417 100644 --- a/apps/web/src/server/rate/api-handler.ts +++ b/apps/web/src/server/rate/api-handler.ts @@ -357,6 +357,51 @@ export function isSafeCrowdIngestHttpUrl(rawUrl: string): boolean { } } +export function resolveInspectObservedPrices(input: { + queryPrice?: number | null; + queryOriginalPrice?: number | null; + matchedOfferPriceCurrent?: unknown; + matchedOfferPriceOriginal?: unknown; + masterMinPrice?: unknown; + maxVnd?: number; +}): { observedPrice: number; observedOriginalPrice: number } { + const safeMaxVnd = + typeof input.maxVnd === 'number' && + Number.isFinite(input.maxVnd) && + input.maxVnd > 0 + ? Math.round(input.maxVnd) + : MAX_VALID_VND_PRICE; + const toPositiveBoundedPrice = (val: unknown): number => { + if ( + val === null || + val === undefined || + typeof val === 'boolean' || + (typeof val === 'string' && !val.trim()) + ) { + return 0; + } + const n = Number(val); + return Number.isFinite(n) && n > 0 && n <= safeMaxVnd ? Math.round(n) : 0; + }; + + const observedPrice = + toPositiveBoundedPrice(input.queryPrice) || + toPositiveBoundedPrice(input.matchedOfferPriceCurrent) || + toPositiveBoundedPrice(input.masterMinPrice) || + 0; + + const rawOrigCandidate = + input.queryOriginalPrice ?? + (input.queryPrice === null || input.queryPrice === undefined + ? input.matchedOfferPriceOriginal + : null); + const candidateOrig = toPositiveBoundedPrice(rawOrigCandidate); + const observedOriginalPrice = + observedPrice > 0 && candidateOrig >= observedPrice ? candidateOrig : 0; + + return { observedPrice, observedOriginalPrice }; +} + const CANONICAL_PRODUCT_SLUG_REGEX = /^[a-zA-Z0-9._-]+$/; /** From afbe1f56ca1d47d97fdf9cc70cd197f634bd21de Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:09:26 +0700 Subject: [PATCH 19/19] feat(ops): commit agent release helpers and report the live commit (#865) (#870) * feat(ops): commit agent release helpers and report the live commit (#865) Agents merged, promoted, updated Project #9 and verified releases with copy-paste commands or helpers kept only in a session scratchpad. On 2026-10-24 a production redeploy took a deployment older than main and briefly rolled back a release (#814). - scripts/ops/merge-when-green.mjs: merge once the head's checks pass, pinned to the head SHA; PRs into main always use a merge commit. - scripts/ops/board.mjs: set Project #9 status and agent plus the status:*/agent:* labels in one command. - scripts/ops/promote.mjs: open or reuse the dev to main PR with the released PRs listed, then merge it with a merge commit when green. - scripts/ops/verify-live.mjs: check the live commit equals main, the database is readable and the main routes answer 200; on a mismatch, print the production deployment built from main as the only redeploy source. - /api/health reports `commit` (VERCEL_GIT_COMMIT_SHA, null off Vercel). - Unit tests run in CI; AGENTS.md and RELEASE.md point to the scripts. Co-Authored-By: Claude Opus 5.5 * fix(ops): address review of the release helpers (#865) - verify-live re-reads main every poll and suggests a fix only when the live commit is behind main (compare live...main); it never suggests rolling a newer or diverged deployment back. The fix it prints is `vercel promote` of main's production deployment, not a redeploy. - merge-when-green also gates on commit statuses (Vercel), reads the required checks from the branch rules, paginates check runs, encodes the branch ref, validates the merge method before waiting, tolerates transient API errors and takes --ignore for a known-broken check. - promote never reuses a fork's dev PR, refreshes a reused PR's body and counts unnumbered commits correctly. - board writes labels first, adds the assignee and replaces another agent's claim label on a hand-over. Co-Authored-By: Claude Opus 5.5 * fix(ops): never take another agent's issue silently; harden waits (#865) Round-2 review of #870: - board refuses an issue claimed by another agent unless --take-over is passed (AGENTS.md section 1), and prints the claim it replaces. - merge-when-green rejects an --ignore of a required check, re-derives the merge method each poll and retries a brief merge refusal after a fresh read. - promote edits a reused PR's body only when it changed (an edit re-runs the PR-base check). - verify-live survives a transient error re-reading main. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: JOY Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci.yml | 3 + AGENTS.md | 7 + Docs/Resources/RELEASE.md | 3 +- apps/web/src/app/api/health/route.ts | 3 + .../__tests__/core-rate-architecture.test.ts | 3 + apps/web/src/server/rate/openapi.ts | 2 +- scripts/ops/board.mjs | 210 ++++++++++++ scripts/ops/board.test.mjs | 80 +++++ scripts/ops/lib.mjs | 53 +++ scripts/ops/merge-when-green.mjs | 312 ++++++++++++++++++ scripts/ops/merge-when-green.test.mjs | 212 ++++++++++++ scripts/ops/promote.mjs | 142 ++++++++ scripts/ops/promote.test.mjs | 60 ++++ scripts/ops/verify-live.mjs | 231 +++++++++++++ scripts/ops/verify-live.test.mjs | 122 +++++++ 15 files changed, 1441 insertions(+), 2 deletions(-) create mode 100644 scripts/ops/board.mjs create mode 100644 scripts/ops/board.test.mjs create mode 100644 scripts/ops/lib.mjs create mode 100644 scripts/ops/merge-when-green.mjs create mode 100644 scripts/ops/merge-when-green.test.mjs create mode 100644 scripts/ops/promote.mjs create mode 100644 scripts/ops/promote.test.mjs create mode 100644 scripts/ops/verify-live.mjs create mode 100644 scripts/ops/verify-live.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75791ba2d..0bcaeb6b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,9 @@ jobs: - name: Test the E2E stale-stack pruning run: node --test scripts/e2e/stale-stacks.test.mjs + - name: Test the agent release scripts + run: node --test scripts/ops/merge-when-green.test.mjs scripts/ops/board.test.mjs scripts/ops/promote.test.mjs scripts/ops/verify-live.test.mjs + - name: Replay rate migrations on PostgreSQL 17 (Docker) run: node scripts/test-migration-replay.mjs diff --git a/AGENTS.md b/AGENTS.md index da45f22a8..894ffb537 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,11 @@ Tài liệu này tổng hợp đầy đủ **User-Level Hard Rules (`C:\Users\JO - **TUYỆT ĐỐI KHÔNG** chạy `vercel deploy --prod`, `vercel --prod`, `--prebuilt --prod`, `--skip-domain` rồi `promote`, hay promote một bản Preview của `dev` lên production, kể cả khi JOY đã cho phép "deploy". Lời cho phép deploy nghĩa là merge PR promote `dev` -> `main`, không phải deploy từ máy local. - Chốt kỹ thuật: `scripts/vercel-production-guard.mjs` (chạy trong `buildCommand` của `vercel.json`) làm build production thất bại nếu không phải GitHub build từ `DOS/Rate@main`; workflow `Validate PR Base` chặn PR vào `main` không từ `dev`; workflow `Branch Divergence` báo lỗi khi `main` có commit mà `dev` không có. Không được sửa hay vô hiệu hóa các chốt này để deploy cho nhanh. - Ngoại lệ duy nhất: **rollback** về một bản production cũ đã build từ Git (`vercel rollback`), vì không build lại. Sau đó sửa tiếp qua PR hotfix. + - **Script vận hành có sẵn trong repo (`scripts/ops/`, dùng thay cho lệnh gõ tay):** + - `node scripts/ops/merge-when-green.mjs `: chờ commit head xanh rồi merge, ghim theo SHA head. Xanh nghĩa là mọi check run và mọi commit status (Vercel) đều đạt, và các check mà ruleset bắt buộc đã báo kết quả. PR vào `main` luôn dùng merge commit, PR khác mặc định squash. `--ignore ""` chỉ dùng khi một check không bắt buộc đang hỏng có lý do đã ghi trong PR. + - `node scripts/ops/promote.mjs`: mở hoặc dùng lại PR promote `dev` -> `main`, liệt kê các PR được phát hành, merge bằng merge commit khi CI xanh. + - `node scripts/ops/verify-live.mjs --wait 900`: kiểm tra rate.vn chạy đúng commit của `main` (`/api/health` trả `commit`), database đọc được và các route chính trả 200. Script đọc lại `main` mỗi vòng chờ. Chỉ khi bản live chậm hơn `main`, nó mới in lệnh `vercel promote --scope metados`; nếu bản live đi trước hoặc lệch nhánh so với `main`, nó dặn không làm gì. Không bao giờ redeploy hay promote một deployment build từ commit khác (`promote` không build lại, và bật lại tự gán domain sau một lần `vercel rollback`). + - `node scripts/ops/board.mjs --status in-progress|in-review|done --agent `: cập nhật Project #9 và nhãn (xem mục 1). Script từ chối issue đang mang nhãn `agent:*` của agent khác; `--take-over` chỉ dùng khi agent đó đã bàn giao hoặc JOY chỉ định. 7. **Chủ động thực thi liên tục & đề xuất việc tiếp theo (Proactive Next-Task):** - **Khi đang ở chế độ `/goal` hoặc được giao làm hết backlog:** Tự động tạo/nhận và làm liên tục từng task đến khi sạch toàn bộ backlog mà **KHÔNG dừng lại hỏi `Bắt đầu chứ?`**. @@ -75,6 +80,8 @@ gh issue list -R DOS/Rate-Services --state open ``` - **Chuyển trạng thái sang `In Progress` & Ghi danh Agent ngay khi bắt đầu (BẮT BUỘC đồng bộ cả Issue lẫn Board Project #9, comment bằng Tiếng Anh):** + Cách nhanh, một lệnh làm cả nhãn lẫn board (tự thêm issue vào Project #9 nếu chưa có, chỉ giữ một nhãn `status:*`): `node scripts/ops/board.mjs --status in-progress --agent $agentNick`. Các trạng thái khác: `ready`, `in-review`, `done`, `backlog`. Các lệnh `gh` dưới đây là cách làm tương đương bằng tay. + ```powershell $agentNick = "Crawl" # Nickname của Agent hiện tại $agentLabel = "agent:$($agentNick.ToLower())" diff --git a/Docs/Resources/RELEASE.md b/Docs/Resources/RELEASE.md index 525ad4eee..ebad64b04 100644 --- a/Docs/Resources/RELEASE.md +++ b/Docs/Resources/RELEASE.md @@ -17,8 +17,9 @@ This is the release workflow for `rate.vn`. It follows the same model as DOS/DOS `gh pr create --repo DOS/Rate --base main --head dev --title "chore(release): promote dev to main"` List the PRs being released in the body. 4. Wait for the required checks on the promotion PR: `Typecheck & Build Monorepo`, `PR into main comes from dev`, `main must not lead dev`. + `node scripts/ops/promote.mjs` does steps 3 to 5 in one command: it opens the promotion PR from `dev` (or reuses the open one) with the released PRs listed, then merges it with a merge commit once its checks pass. `--no-merge` leaves it open. 5. Promotions are batched and need no per-release question: the agent promotes at the first checkpoint reached: the backlog runs out, 10 or more PRs are waiting on `dev`, the oldest unpromoted change is more than 24 hours old, or the agent is ending a long working session. Promote only when `dev` head CI is green and no PR is mid-way through fixing a production bug. The agent merges it with a merge commit: `gh pr merge --repo DOS/Rate --merge`. JOY does not merge PRs; never ask JOY to click Merge. **Never squash or rebase a promotion.** Squashing creates a commit on `main` that can never exist on `dev`, and the next promotion then diffs from a stale merge base. -6. Vercel builds `main` and assigns `rate.vn`. Verify the live revision (the deployment's Git SHA equals `main`), `/api/health`, the main routes and a browser check before calling the release done. To check a build before it takes traffic, create it from Git with `autoAssignCustomDomains: false` (Vercel API `POST /v13/deployments` with `gitSource` ref `main` and `target: "production"`), verify it with `vercel curl --deployment `, then `vercel promote `. Promoting an existing production deployment does not rebuild. +6. Vercel builds `main` and assigns `rate.vn`. Verify the live revision (the deployment's Git SHA equals `main`), `/api/health`, the main routes and a browser check before calling the release done. `node scripts/ops/verify-live.mjs --wait 900` checks the first three: `/api/health` reports the deployment's `commit`, which must equal `main`, and `checks.database` must be `ok`. It re-reads `main` on every poll. Only when the live commit is behind `main` does it print `vercel promote --scope metados`; when live is ahead of or diverged from `main` it says to do nothing. Never redeploy or promote a deployment built from another commit: that rolls the release back. `promote` does not rebuild, and it turns production domain auto-assignment back on after a `vercel rollback`. To check a build before it takes traffic, create it from Git with `autoAssignCustomDomains: false` (Vercel API `POST /v13/deployments` with `gitSource` ref `main` and `target: "production"`), verify it with `vercel curl --deployment `, then `vercel promote `. Promoting an existing production deployment does not rebuild. ## Hotfix diff --git a/apps/web/src/app/api/health/route.ts b/apps/web/src/app/api/health/route.ts index c5295376a..666a2bf40 100644 --- a/apps/web/src/app/api/health/route.ts +++ b/apps/web/src/app/api/health/route.ts @@ -14,6 +14,9 @@ export async function GET() { service: 'rate-web', version: '2.0.0', timestamp: new Date().toISOString(), + // The Git commit this deployment was built from, so a release check can + // compare it with main (scripts/ops/verify-live.mjs). Null off Vercel. + commit: process.env.VERCEL_GIT_COMMIT_SHA || null, checks: { database }, }, { diff --git a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts index 4196573c9..022b5c41d 100644 --- a/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts +++ b/apps/web/src/server/rate/__tests__/core-rate-architecture.test.ts @@ -3510,9 +3510,12 @@ async function runCoreArchitectureSuite() { const healthBody60 = (await healthRes60.json()) as { status: string; service: string; + commit: string | null; }; assert.equal(healthBody60.status, 'ok'); assert.equal(healthBody60.service, 'rate-web'); + // The deployed commit, for scripts/ops/verify-live.mjs (DOS/Rate#865). + assert.equal(healthBody60.commit, process.env.VERCEL_GIT_COMMIT_SHA || null); // 60.2 /api/v1/rate/openapi & /api/v1/rate/openapi.json const openApiOpt = await openApiRoute.OPTIONS(); diff --git a/apps/web/src/server/rate/openapi.ts b/apps/web/src/server/rate/openapi.ts index 2dfd687d9..8bfbb172b 100644 --- a/apps/web/src/server/rate/openapi.ts +++ b/apps/web/src/server/rate/openapi.ts @@ -1479,7 +1479,7 @@ export function buildRateOpenApiSpec( operationId: 'getHealthStatus', summary: 'Liveness & readiness health check probe', description: - 'Returns service metadata (`status`, `service: "rate-web"`, `version`, ISO-8601 `timestamp`) and `checks.database`, the result of a small read with the app\'s own database client (3 s timeout, reused for up to 15 s per instance), with `Cache-Control: no-store, max-age=0` for load balancers and uptime monitors.', + 'Returns service metadata (`status`, `service: "rate-web"`, `version`, ISO-8601 `timestamp`, `commit`: the Git SHA the deployment was built from, or null outside Vercel) and `checks.database`, the result of a small read with the app\'s own database client (3 s timeout, reused for up to 15 s per instance), with `Cache-Control: no-store, max-age=0` for load balancers and uptime monitors.', tags: ['Platform Telemetry'], responses: { '200': { diff --git a/scripts/ops/board.mjs b/scripts/ops/board.mjs new file mode 100644 index 000000000..65c12fd9d --- /dev/null +++ b/scripts/ops/board.mjs @@ -0,0 +1,210 @@ +#!/usr/bin/env node +// Sets an issue's status and agent on GitHub Project #9 and in its labels, as +// AGENTS.md section 1 requires, in one command: +// +// node scripts/ops/board.mjs ... --status in-progress --agent Guard +// [--repo DOS/Rate] +// +// The status label replaces any other status:* label, the agent label +// (agent:) is created when missing, and an issue not yet on the board +// is added to it. An issue another agent has claimed is refused unless +// --take-over is passed (a hand-over or JOY's assignment). + +import { parseArgs } from 'node:util'; +import { DEFAULT_REPO, fail, gh, ghJson, isMain } from './lib.mjs'; + +export const PROJECT = { + owner: 'DOS', + number: 9, + id: 'PVT_kwDOA388Vc4Bm5Ml', + statusField: 'PVTSSF_lADOA388Vc4Bm5MlzhlgSt8', + agentField: 'PVTF_lADOA388Vc4Bm5MlzhlqBH8', +}; + +export const STATUSES = { + backlog: { option: 'f75ad846', label: null }, + ready: { option: 'e18bf179', label: 'status:ready' }, + 'in-progress': { option: '47fc9ee4', label: 'status:in-progress' }, + 'in-review': { option: 'aba860b9', label: 'status:in-review' }, + done: { option: '98236657', label: 'status:done' }, +}; + +export const agentLabel = (nick) => `agent:${nick.trim().toLowerCase()}`; + +/** + * Label edits that leave exactly one status label and one agent label. + * An issue claimed by another agent is never taken silently (AGENTS.md + * section 1): that needs `takeOver`, and its claim label is then removed. + */ +export function labelChanges(current, status, nick, { takeOver = false } = {}) { + const spec = STATUSES[status]; + if (!spec) throw new Error(`unknown status ${status}`); + const mine = agentLabel(nick); + const others = current.filter((l) => l.startsWith('agent:') && l !== mine); + if (others.length > 0 && !takeOver) { + throw new Error( + `claimed by ${others.join(', ')}: pass --take-over only when that agent handed it over or JOY assigned it` + ); + } + const want = [spec.label, mine].filter(Boolean); + return { + add: want.filter((l) => !current.includes(l)), + remove: current.filter( + (l) => + (l.startsWith('status:') && l !== spec.label) || + (l.startsWith('agent:') && l !== mine) + ), + }; +} + +const ISSUE_QUERY = `query($owner: String!, $name: String!, $n: Int!) { + repository(owner: $owner, name: $name) { + issue(number: $n) { + url + labels(first: 50) { nodes { name } } + projectItems(first: 20) { nodes { id project { number } } } + } + } +}`; + +function readIssue(repo, number) { + const [owner, name] = repo.split('/'); + const data = ghJson([ + 'api', + 'graphql', + '-f', + `query=${ISSUE_QUERY}`, + '-F', + `owner=${owner}`, + '-F', + `name=${name}`, + '-F', + `n=${number}`, + ]); + const issue = data.data.repository.issue; + if (!issue) throw new Error(`${repo}#${number} not found`); + return { + url: issue.url, + labels: issue.labels.nodes.map((l) => l.name), + itemId: + issue.projectItems.nodes.find((i) => i.project.number === PROJECT.number) + ?.id ?? null, + }; +} + +export function setBoard({ + repo = DEFAULT_REPO, + number, + status, + nick, + takeOver = false, +}) { + const spec = STATUSES[status]; + if (!spec) throw new Error(`unknown status ${status}`); + const issue = readIssue(repo, number); + + // Labels first: the required task-claim check reads them, so a failure + // later on the board leaves the claim intact. + const { add, remove } = labelChanges(issue.labels, status, nick, { + takeOver, + }); + const taken = remove.filter((l) => l.startsWith('agent:')); + if (taken.length > 0) { + console.log(`${repo}#${number}: taking over from ${taken.join(', ')}`); + } + if (add.includes(agentLabel(nick))) { + gh( + [ + 'label', + 'create', + agentLabel(nick), + '-R', + repo, + '--color', + 'D93F0B', + '--description', + `Claimed by Agent: ${nick}`, + ], + { allowFailure: true } // already exists; a real failure resurfaces below + ); + } + gh([ + 'issue', + 'edit', + String(number), + '-R', + repo, + '--add-assignee', + '@me', + ...add.flatMap((l) => ['--add-label', l]), + ...remove.flatMap((l) => ['--remove-label', l]), + ]); + + let itemId = issue.itemId; + if (!itemId) { + itemId = ghJson([ + 'project', + 'item-add', + String(PROJECT.number), + '--owner', + PROJECT.owner, + '--url', + issue.url, + '--format', + 'json', + ]).id; + } + const edit = (field, value) => + gh([ + 'project', + 'item-edit', + '--project-id', + PROJECT.id, + '--id', + itemId, + '--field-id', + field, + ...value, + ]); + edit(PROJECT.statusField, ['--single-select-option-id', spec.option]); + edit(PROJECT.agentField, ['--text', nick]); + return `${repo}#${number} -> ${status} (${nick})`; +} + +if (isMain(import.meta.url)) { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + repo: { type: 'string', default: DEFAULT_REPO }, + status: { type: 'string' }, + agent: { type: 'string' }, + 'take-over': { type: 'boolean', default: false }, + }, + }); + const numbers = positionals.map(Number); + if ( + numbers.length === 0 || + numbers.some((n) => !Number.isInteger(n) || n < 1) || + !STATUSES[values.status] || + !values.agent?.trim() + ) { + fail( + `usage: board.mjs ... --status ${Object.keys(STATUSES).join('|')} --agent [--take-over] [--repo owner/name]` + ); + } + for (const number of numbers) { + try { + console.log( + setBoard({ + repo: values.repo, + number, + status: values.status, + nick: values.agent.trim(), + takeOver: values['take-over'], + }) + ); + } catch (err) { + fail(`${values.repo}#${number}: ${err.message}`); + } + } +} diff --git a/scripts/ops/board.test.mjs b/scripts/ops/board.test.mjs new file mode 100644 index 000000000..6b6aac46a --- /dev/null +++ b/scripts/ops/board.test.mjs @@ -0,0 +1,80 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { STATUSES, agentLabel, labelChanges } from './board.mjs'; + +test('agent labels are lowercase agent:', () => { + assert.equal(agentLabel(' Guard '), 'agent:guard'); +}); + +test('moves an issue to a new status and claims it', () => { + assert.deepEqual( + labelChanges(['bug', 'status:ready'], 'in-progress', 'Guard'), + { + add: ['status:in-progress', 'agent:guard'], + remove: ['status:ready'], + } + ); +}); + +test('leaves exactly one status label', () => { + assert.deepEqual( + labelChanges( + ['status:in-progress', 'status:in-review', 'agent:guard'], + 'done', + 'Guard' + ), + { add: ['status:done'], remove: ['status:in-progress', 'status:in-review'] } + ); +}); + +test('never takes an issue another agent claimed without --take-over', () => { + assert.throws( + () => + labelChanges( + ['status:in-progress', 'agent:crawl'], + 'in-progress', + 'Guard' + ), + /claimed by agent:crawl: pass --take-over/ + ); +}); + +test('a deliberate hand-over replaces the previous agent label', () => { + assert.deepEqual( + labelChanges(['status:in-review', 'agent:crawl'], 'in-progress', 'Guard', { + takeOver: true, + }), + { + add: ['status:in-progress', 'agent:guard'], + remove: ['status:in-review', 'agent:crawl'], + } + ); +}); + +test('changes nothing when the labels already match', () => { + assert.deepEqual( + labelChanges(['status:in-review', 'agent:guard'], 'in-review', 'guard'), + { + add: [], + remove: [], + } + ); +}); + +test('backlog has no status label and removes any', () => { + assert.equal(STATUSES.backlog.label, null); + assert.deepEqual(labelChanges(['status:ready'], 'backlog', 'Guard'), { + add: ['agent:guard'], + remove: ['status:ready'], + }); +}); + +test('rejects an unknown status', () => { + assert.throws(() => labelChanges([], 'blocked', 'Guard'), /unknown status/); +}); + +test('every status has a Project #9 option id', () => { + for (const spec of Object.values(STATUSES)) + assert.match(spec.option, /^[0-9a-f]{8}$/); +}); diff --git a/scripts/ops/lib.mjs b/scripts/ops/lib.mjs new file mode 100644 index 000000000..297cd4b39 --- /dev/null +++ b/scripts/ops/lib.mjs @@ -0,0 +1,53 @@ +// Shared helpers for the agent release scripts in scripts/ops (DOS/Rate#865). +// They drive the GitHub CLI with argument arrays, never a shell string, so no +// value is ever interpreted by a shell. + +import { spawnSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; + +export const DEFAULT_REPO = 'DOS/Rate'; + +export function gh(args, { allowFailure = false } = {}) { + const r = spawnSync('gh', args, { + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + }); + if (r.error) throw r.error; + if (r.status !== 0 && !allowFailure) { + throw new Error( + `gh ${args[0]} ${args[1] ?? ''} failed: ${r.stderr.trim()}` + ); + } + return r; +} + +export function ghJson(args) { + const out = gh(args).stdout.trim(); + return out ? JSON.parse(out) : null; +} + +/** + * Every item of a paginated REST list: `jq` selects the items of one page, + * each printed as one JSON line. + */ +export function ghPaginated(path, jq) { + return gh(['api', '--paginate', path, '--jq', jq]) + .stdout.split('\n') + .filter((line) => line.trim()) + .map((line) => JSON.parse(line)); +} + +/** A path segment per branch-name part: `fix#12` must not become `fix`. */ +export const encodeRef = (ref) => + ref.split('/').map(encodeURIComponent).join('/'); + +export const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const isMain = (url) => + url === pathToFileURL(process.argv[1] ?? '').href; + +/** Prints the message and exits non-zero: the scripts' one failure path. */ +export function fail(message) { + console.error(message); + process.exit(1); +} diff --git a/scripts/ops/merge-when-green.mjs b/scripts/ops/merge-when-green.mjs new file mode 100644 index 000000000..b6d491432 --- /dev/null +++ b/scripts/ops/merge-when-green.mjs @@ -0,0 +1,312 @@ +#!/usr/bin/env node +// Merges a PR once its head commit is green, pinned to that head so a commit +// pushed meanwhile is never merged unchecked. Green means every check run and +// every commit status (Vercel posts statuses, not check runs) passed, and the +// checks the branch's rulesets require have reported. +// +// node scripts/ops/merge-when-green.mjs [--repo DOS/Rate] +// [--method squash|merge] [--require ""]... [--ignore ""]... +// +// PRs into main always use a merge commit (Docs/Resources/RELEASE.md); other +// PRs default to squash, and a squashed head branch is deleted afterwards. +// --ignore lets a known-broken, non-required check through; name it in the PR. +// Prints one line and exits 0 when merged, 1 when it cannot merge. + +import { parseArgs } from 'node:util'; +import { + DEFAULT_REPO, + encodeRef, + fail, + gh, + ghJson, + ghPaginated, + isMain, + sleep, +} from './lib.mjs'; + +/** Bot reviews that report late or never finish and gate nothing. */ +export const IGNORED_CHECKS = ['DOS Code Review', 'CodeRabbit']; +const PASSING = new Set(['success', 'skipped', 'neutral']); +const PROTECTED_BRANCHES = new Set(['main', 'dev', 'master']); +/** Polls a green but `blocked` PR gets before it is reported as blocked. */ +export const BLOCKED_POLLS = 6; +/** Consecutive GitHub API errors tolerated before giving up. */ +const MAX_API_ERRORS = 3; +/** Merge refusals retried after a fresh read of the head. */ +const MAX_MERGE_REFUSALS = 3; + +/** Required checks an --ignore would hide: that can never turn green. */ +export const ignoredRequired = (required, ignore) => + required.filter((name) => ignore.includes(name)); + +/** A re-run adds a check run with the same name: keep the newest of each. */ +export function latestRuns(checkRuns) { + const byName = new Map(); + for (const run of checkRuns) { + const seen = byName.get(run.name); + if (!seen || run.id > seen.id) byName.set(run.name, run); + } + return [...byName.values()]; +} + +/** + * `checkRuns` from the check-runs API, `statuses` from the combined status + * API (already the latest per context). + */ +export function evaluateChecks( + checkRuns, + statuses = [], + { ignore = IGNORED_CHECKS, required = [] } = {} +) { + const runs = latestRuns(checkRuns).filter((r) => !ignore.includes(r.name)); + const contexts = statuses.filter((s) => !ignore.includes(s.context)); + const failed = [ + ...runs + .filter((r) => r.status === 'completed' && !PASSING.has(r.conclusion)) + .map((r) => r.name), + ...contexts + .filter((s) => s.state === 'failure' || s.state === 'error') + .map((s) => s.context), + ]; + if (failed.length > 0) return { state: 'failed', failed }; + const reported = new Set([ + ...runs.map((r) => r.name), + ...contexts.map((s) => s.context), + ]); + const pending = [ + ...runs.filter((r) => r.status !== 'completed').map((r) => r.name), + ...contexts.filter((s) => s.state === 'pending').map((s) => s.context), + ...required.filter((name) => !reported.has(name)), + ]; + // No check run yet means CI has not started, not that it passed. + if (runs.length === 0 || pending.length > 0) { + return { state: 'pending', pending }; + } + return { state: 'passed' }; +} + +/** Check names the branch's rulesets require, from the rules API. */ +export function requiredChecksFromRules(rules) { + return [ + ...new Set( + (rules ?? []) + .filter((r) => r.type === 'required_status_checks') + .flatMap((r) => r.parameters?.required_status_checks ?? []) + .map((c) => c.context) + ), + ]; +} + +export function mergeMethodFor(baseRef, requested) { + if (baseRef === 'main') { + if (requested && requested !== 'merge') { + throw new Error( + `PRs into main merge with a merge commit, not ${requested} (RELEASE.md)` + ); + } + return 'merge'; + } + if (requested && !['squash', 'merge'].includes(requested)) { + throw new Error(`unknown merge method ${requested}`); + } + return requested ?? 'squash'; +} + +export function shouldDeleteBranch({ method, headRef, headRepo, baseRepo }) { + return ( + method === 'squash' && + headRepo === baseRepo && + !PROTECTED_BRANCHES.has(headRef) + ); +} + +/** + * What to do with the PR now: 'merge', 'wait', or a final 'done' / 'fail' + * with a one-line message. + */ +export function decide(pr, checks, blockedPolls) { + if (pr.merged) + return { action: 'done', message: `PR ${pr.number} already merged` }; + if (pr.state !== 'open') + return { action: 'fail', message: `PR ${pr.number} is ${pr.state}` }; + if (pr.mergeable_state === 'dirty') { + return { action: 'fail', message: `PR ${pr.number} has merge conflicts` }; + } + if (checks.state === 'failed') { + return { + action: 'fail', + message: `PR ${pr.number} failing at ${pr.head.sha.slice(0, 8)}: ${checks.failed.join(', ')}`, + }; + } + if (checks.state === 'pending') return { action: 'wait' }; + // GitHub recomputes mergeability a little after the last check finishes. + if (pr.mergeable_state === 'blocked' || pr.mergeable_state === 'unknown') { + return blockedPolls + 1 >= BLOCKED_POLLS + ? { + action: 'fail', + message: `PR ${pr.number} is blocked at ${pr.head.sha.slice(0, 8)} although its checks passed (a required review or check is missing)`, + } + : { action: 'wait', blocked: true }; + } + return { action: 'merge' }; +} + +function readHead(repo, number) { + const pr = ghJson(['api', `repos/${repo}/pulls/${number}`]); + if (pr.state !== 'open') return { pr, runs: [], statuses: [], required: [] }; + const runs = ghPaginated( + `repos/${repo}/commits/${pr.head.sha}/check-runs?per_page=100`, + '.check_runs[] | {id, name, status, conclusion}' + ); + const statuses = + ghJson(['api', `repos/${repo}/commits/${pr.head.sha}/status`])?.statuses ?? + []; + const rules = ghJson([ + 'api', + `repos/${repo}/rules/branches/${encodeRef(pr.base.ref)}`, + ]); + return { pr, runs, statuses, required: requiredChecksFromRules(rules) }; +} + +export async function mergeWhenGreen({ + repo = DEFAULT_REPO, + number, + method: requested, + required: extraRequired = [], + ignore = [], + intervalMs = 30_000, + timeoutMs = 60 * 60_000, +} = {}) { + const deadline = Date.now() + timeoutMs; + let blockedPolls = 0; + let apiErrors = 0; + let mergeRefusals = 0; + for (;;) { + let head; + try { + head = readHead(repo, number); + apiErrors = 0; + } catch (err) { + if (++apiErrors >= MAX_API_ERRORS) { + return { ok: false, message: `PR ${number}: ${err.message}` }; + } + await sleep(intervalMs); + continue; + } + const { pr } = head; + // Refuse a wrong method or an ignored required check before waiting, not + // after the checks pass; both are re-read each poll (a PR can be + // retargeted). + let method = null; + if (pr.state === 'open') { + try { + method = mergeMethodFor(pr.base.ref, requested); + } catch (err) { + return { ok: false, message: `PR ${number}: ${err.message}` }; + } + const conflict = ignoredRequired(head.required, ignore); + if (conflict.length > 0) { + return { + ok: false, + message: `PR ${number}: cannot --ignore required check(s) ${conflict.join(', ')}`, + }; + } + } + const checks = evaluateChecks(head.runs, head.statuses, { + ignore: [...IGNORED_CHECKS, ...ignore], + required: [...head.required, ...extraRequired], + }); + const next = decide(pr, checks, blockedPolls); + if (next.action === 'done') return { ok: true, message: next.message }; + if (next.action === 'fail') return { ok: false, message: next.message }; + if (next.action === 'merge') { + const merged = gh( + [ + 'api', + '-X', + 'PUT', + `repos/${repo}/pulls/${number}/merge`, + '-f', + `merge_method=${method}`, + '-f', + `sha=${pr.head.sha}`, + '--jq', + '.sha', + ], + { allowFailure: true } + ); + if (merged.status !== 0) { + // A check re-queued after the last read (for example by a PR edit) + // makes GitHub refuse briefly: read the head again before giving up. + if (++mergeRefusals < MAX_MERGE_REFUSALS) { + await sleep(intervalMs); + continue; + } + return { + ok: false, + message: `PR ${number} merge refused: ${merged.stderr.trim()}`, + }; + } + if ( + shouldDeleteBranch({ + method, + headRef: pr.head.ref, + headRepo: pr.head.repo?.full_name, + baseRepo: pr.base.repo.full_name, + }) + ) { + gh( + [ + 'api', + '-X', + 'DELETE', + `repos/${repo}/git/refs/heads/${encodeRef(pr.head.ref)}`, + ], + { allowFailure: true } + ); + } + return { + ok: true, + message: `PR ${number} MERGED ${merged.stdout.trim()}`, + }; + } + blockedPolls = next.blocked ? blockedPolls + 1 : 0; + if (Date.now() > deadline) { + const waiting = checks.pending?.length + ? ` (waiting on ${checks.pending.join(', ')})` + : ''; + return { + ok: false, + message: `PR ${number} still not green after ${timeoutMs / 60_000} min${waiting}`, + }; + } + await sleep(intervalMs); + } +} + +if (isMain(import.meta.url)) { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + repo: { type: 'string', default: DEFAULT_REPO }, + method: { type: 'string' }, + require: { type: 'string', multiple: true }, + ignore: { type: 'string', multiple: true }, + }, + }); + const number = Number(positionals[0]); + if (!Number.isInteger(number) || number < 1) { + fail( + 'usage: merge-when-green.mjs [--repo owner/name] [--method squash|merge] [--require ]... [--ignore ]...' + ); + } + const result = await mergeWhenGreen({ + repo: values.repo, + number, + method: values.method, + required: values.require ?? [], + ignore: values.ignore ?? [], + }); + if (!result.ok) fail(result.message); + console.log(result.message); +} diff --git a/scripts/ops/merge-when-green.test.mjs b/scripts/ops/merge-when-green.test.mjs new file mode 100644 index 000000000..1c8733696 --- /dev/null +++ b/scripts/ops/merge-when-green.test.mjs @@ -0,0 +1,212 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + BLOCKED_POLLS, + decide, + evaluateChecks, + ignoredRequired, + latestRuns, + mergeMethodFor, + requiredChecksFromRules, + shouldDeleteBranch, +} from './merge-when-green.mjs'; +import { encodeRef } from './lib.mjs'; + +const run = (name, status, conclusion = null, id = 1) => ({ + id, + name, + status, + conclusion, +}); +const status = (context, state) => ({ context, state }); +const pr = (overrides = {}) => ({ + number: 12, + merged: false, + state: 'open', + mergeable_state: 'clean', + head: { sha: 'abcdef1234567890' }, + ...overrides, +}); +const green = [run('CI', 'completed', 'success')]; + +test('keeps only the newest run of a re-run check', () => { + const runs = latestRuns([ + run('CI', 'completed', 'failure', 1), + run('CI', 'completed', 'success', 2), + ]); + assert.deepEqual( + runs.map((r) => r.conclusion), + ['success'] + ); +}); + +test('passes when every check succeeded or was skipped', () => { + assert.deepEqual( + evaluateChecks([ + run('CI', 'completed', 'success'), + run('Lint', 'completed', 'skipped'), + ]), + { state: 'passed' } + ); +}); + +test('fails on a failed or cancelled check, even while others run', () => { + const r = evaluateChecks([ + run('CI', 'completed', 'cancelled'), + run('E2E', 'in_progress'), + ]); + assert.equal(r.state, 'failed'); + assert.deepEqual(r.failed, ['CI']); +}); + +test('waits while a check runs, before any check exists, or while a required one is missing', () => { + assert.equal(evaluateChecks([run('CI', 'queued')]).state, 'pending'); + assert.equal(evaluateChecks([]).state, 'pending'); + const r = evaluateChecks([run('Lint', 'completed', 'success')], [], { + required: ['Typecheck & Build Monorepo'], + }); + assert.equal(r.state, 'pending'); + assert.deepEqual(r.pending, ['Typecheck & Build Monorepo']); +}); + +test('a commit status counts like a check: Vercel failing fails, pending waits', () => { + const failing = evaluateChecks(green, [status('Vercel', 'failure')]); + assert.equal(failing.state, 'failed'); + assert.deepEqual(failing.failed, ['Vercel']); + assert.equal( + evaluateChecks(green, [status('Vercel', 'error')]).state, + 'failed' + ); + assert.equal( + evaluateChecks(green, [status('Vercel', 'pending')]).state, + 'pending' + ); + assert.equal( + evaluateChecks(green, [status('Vercel', 'success')]).state, + 'passed' + ); +}); + +test('a required check may report as a commit status', () => { + assert.equal( + evaluateChecks(green, [status('Vercel', 'success')], { + required: ['Vercel'], + }).state, + 'passed' + ); +}); + +test('ignores bot reviews that never finish, as checks or statuses', () => { + assert.equal( + evaluateChecks( + [...green, run('DOS Code Review', 'in_progress')], + [status('CodeRabbit', 'pending')] + ).state, + 'passed' + ); +}); + +test('--ignore lets a named broken check through, nothing else', () => { + const runs = [...green, run('E2E (Playwright)', 'completed', 'failure')]; + assert.equal(evaluateChecks(runs).state, 'failed'); + assert.equal( + evaluateChecks(runs, [], { ignore: ['E2E (Playwright)'] }).state, + 'passed' + ); +}); + +test('an --ignore that hides a required check is refused', () => { + assert.deepEqual( + ignoredRequired( + ['Typecheck & Build Monorepo', 'task-claim'], + ['E2E (Playwright)', 'task-claim'] + ), + ['task-claim'] + ); + assert.deepEqual(ignoredRequired(['task-claim'], ['E2E (Playwright)']), []); +}); + +test('reads the required check names from the branch rules', () => { + assert.deepEqual( + requiredChecksFromRules([ + { type: 'deletion' }, + { + type: 'required_status_checks', + parameters: { + required_status_checks: [ + { context: 'Typecheck & Build Monorepo' }, + { context: 'task-claim' }, + ], + }, + }, + { + type: 'required_status_checks', + parameters: { required_status_checks: [{ context: 'task-claim' }] }, + }, + ]), + ['Typecheck & Build Monorepo', 'task-claim'] + ); + assert.deepEqual(requiredChecksFromRules(null), []); +}); + +test('PRs into main always merge with a merge commit', () => { + assert.equal(mergeMethodFor('main'), 'merge'); + assert.equal(mergeMethodFor('main', 'merge'), 'merge'); + assert.throws(() => mergeMethodFor('main', 'squash'), /merge commit/); + assert.equal(mergeMethodFor('dev'), 'squash'); + assert.equal(mergeMethodFor('dev', 'merge'), 'merge'); + assert.throws(() => mergeMethodFor('dev', 'rebase'), /unknown merge method/); +}); + +test('deletes only a squashed branch of the same repository that is not dev or main', () => { + const base = { + method: 'squash', + headRef: 'fix/x', + headRepo: 'DOS/Rate', + baseRepo: 'DOS/Rate', + }; + assert.equal(shouldDeleteBranch(base), true); + assert.equal(shouldDeleteBranch({ ...base, method: 'merge' }), false); + assert.equal(shouldDeleteBranch({ ...base, headRef: 'dev' }), false); + assert.equal( + shouldDeleteBranch({ ...base, headRepo: 'someone/Rate' }), + false + ); + assert.equal(shouldDeleteBranch({ ...base, headRepo: undefined }), false); +}); + +test('branch names are encoded per path segment', () => { + assert.equal(encodeRef('fix#12'), 'fix%2312'); + assert.equal(encodeRef('feat/a b'), 'feat/a%20b'); +}); + +test('decides from the PR state and its checks', () => { + const passed = { state: 'passed' }; + assert.equal(decide(pr({ merged: true }), passed, 0).action, 'done'); + assert.equal(decide(pr({ state: 'closed' }), passed, 0).action, 'fail'); + assert.match( + decide(pr({ mergeable_state: 'dirty' }), passed, 0).message, + /conflicts/ + ); + assert.match( + decide(pr(), { state: 'failed', failed: ['CI'] }, 0).message, + /failing at abcdef12: CI/ + ); + assert.equal( + decide(pr(), { state: 'pending', pending: ['CI'] }, 0).action, + 'wait' + ); + assert.equal(decide(pr(), passed, 0).action, 'merge'); +}); + +test('gives a green but blocked PR a few polls before reporting it', () => { + const blocked = pr({ mergeable_state: 'blocked' }); + assert.deepEqual(decide(blocked, { state: 'passed' }, 0), { + action: 'wait', + blocked: true, + }); + const last = decide(blocked, { state: 'passed' }, BLOCKED_POLLS - 1); + assert.equal(last.action, 'fail'); + assert.match(last.message, /required review or check/); +}); diff --git a/scripts/ops/promote.mjs b/scripts/ops/promote.mjs new file mode 100644 index 000000000..1200d5b5b --- /dev/null +++ b/scripts/ops/promote.mjs @@ -0,0 +1,142 @@ +#!/usr/bin/env node +// Promotes dev to main as Docs/Resources/RELEASE.md describes: opens the +// promotion PR from dev itself (or reuses the open one), lists the PRs being +// released, and merges it with a merge commit once its checks pass. +// +// node scripts/ops/promote.mjs [--repo DOS/Rate] [--no-merge] +// +// Then check the release: node scripts/ops/verify-live.mjs --wait 900 + +import { parseArgs } from 'node:util'; +import { DEFAULT_REPO, fail, gh, ghJson, isMain } from './lib.mjs'; +import { mergeWhenGreen } from './merge-when-green.mjs'; + +export const PROMOTION_TITLE = 'chore(release): promote dev to main'; +/** The compare API lists at most this many commits. */ +const COMPARE_COMMIT_LIMIT = 250; + +/** + * The squashed PRs on dev ("title (#123)"), oldest first and once each, and + * how many single-parent commits carry no PR number. Merge commits are + * back-merges, earlier promotions or merge-commit PRs, not separate work. + */ +export function releasedPrs(commits) { + const seen = new Set(); + const prs = []; + let unnumbered = 0; + for (const c of commits) { + if (c.parents.length > 1) continue; + const subject = c.commit.message.split('\n')[0]; + const match = subject.match(/\(#(\d+)\)\s*$/); + if (!match) { + unnumbered += 1; + continue; + } + if (seen.has(match[1])) continue; + seen.add(match[1]); + prs.push({ + number: Number(match[1]), + title: subject.replace(/\s*\(#\d+\)\s*$/, ''), + }); + } + return { prs, unnumbered }; +} + +export function promotionBody({ prs, unnumbered }, { truncated = false } = {}) { + const lines = ['Promote dev to main.', '']; + lines.push(...prs.map((p) => `- #${p.number} ${p.title}`)); + if (unnumbered > 0) { + lines.push(`- ${unnumbered} other commit(s) without a PR number`); + } + if (truncated) { + lines.push( + `- more than ${COMPARE_COMMIT_LIMIT} commits: the list above is incomplete` + ); + } + lines.push( + '', + 'Merge with a merge commit, never squash (Docs/Resources/RELEASE.md).' + ); + return lines.join('\n'); +} + +/** The open promotion PR: head dev of this repository, never a fork's dev. */ +export const ownPromotionPr = (open) => + (open ?? []).find((p) => !p.isCrossRepository) ?? null; + +export async function promote({ repo = DEFAULT_REPO, merge = true } = {}) { + const compare = ghJson(['api', `repos/${repo}/compare/main...dev`]); + if (compare.ahead_by === 0) { + return { + ok: true, + message: 'dev has nothing that main lacks: no promotion needed', + }; + } + const body = promotionBody(releasedPrs(compare.commits), { + truncated: compare.ahead_by > compare.commits.length, + }); + + const existing = ownPromotionPr( + ghJson([ + 'pr', + 'list', + '--repo', + repo, + '--base', + 'main', + '--head', + 'dev', + '--state', + 'open', + '--json', + 'number,url,isCrossRepository,body', + ]) + ); + let number = existing?.number; + if (number) { + // dev may have moved since the PR was opened: list what it releases now. + // Only on a real change, since an edit re-runs the PR-base check. + if ((existing.body ?? '').trim() !== body.trim()) { + gh(['pr', 'edit', String(number), '--repo', repo, '--body', body]); + } + console.log(`reusing open promotion PR #${number}`); + } else { + const url = gh([ + 'pr', + 'create', + '--repo', + repo, + '--base', + 'main', + '--head', + 'dev', + '--title', + PROMOTION_TITLE, + '--body', + body, + ]).stdout.trim(); + number = Number(url.split('/').pop()); + console.log(`opened ${url} (${compare.ahead_by} commits)`); + } + if (!merge) + return { + ok: true, + message: `promotion PR #${number} left open (--no-merge)`, + }; + return mergeWhenGreen({ repo, number, method: 'merge' }); +} + +if (isMain(import.meta.url)) { + const { values } = parseArgs({ + options: { + repo: { type: 'string', default: DEFAULT_REPO }, + 'no-merge': { type: 'boolean', default: false }, + }, + }); + const result = await promote({ + repo: values.repo, + merge: !values['no-merge'], + }); + if (!result.ok) fail(result.message); + console.log(result.message); +} diff --git a/scripts/ops/promote.test.mjs b/scripts/ops/promote.test.mjs new file mode 100644 index 000000000..0d8918e6e --- /dev/null +++ b/scripts/ops/promote.test.mjs @@ -0,0 +1,60 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { ownPromotionPr, promotionBody, releasedPrs } from './promote.mjs'; + +const commit = (message, parents = 1) => ({ + commit: { message }, + parents: Array.from({ length: parents }, (_, i) => ({ sha: String(i) })), +}); + +test('lists the squashed PRs on dev, oldest first and once each', () => { + const released = releasedPrs([ + commit('fix(web): bait prices (#800)\n\nbody'), + commit('Merge pull request #806 from DOS/dev', 2), + commit('feat(web): health (#830)'), + commit('fix(web): bait prices (#800)'), + commit('chore: tidy without a number'), + ]); + assert.deepEqual(released, { + prs: [ + { number: 800, title: 'fix(web): bait prices' }, + { number: 830, title: 'feat(web): health' }, + ], + unnumbered: 1, + }); +}); + +test('the PR number is the trailing one; an issue number stays in the title', () => { + assert.deepEqual(releasedPrs([commit('fix(x): y (#834) (#835)')]).prs, [ + { number: 835, title: 'fix(x): y (#834)' }, + ]); + assert.deepEqual(releasedPrs([commit('docs: mention (#12) in the text')]), { + prs: [], + unnumbered: 1, + }); +}); + +test('the body lists the PRs, the rest and the merge rule', () => { + const body = promotionBody( + { prs: [{ number: 830, title: 'feat(web): health' }], unnumbered: 2 }, + { truncated: true } + ); + assert.match(body, /^Promote dev to main\./); + assert.match(body, /- #830 feat\(web\): health/); + assert.match(body, /2 other commit\(s\) without a PR number/); + assert.match(body, /more than 250 commits/); + assert.match(body, /never squash/); +}); + +test('a fork PR from its own dev is never taken as the promotion', () => { + assert.equal(ownPromotionPr([{ number: 9, isCrossRepository: true }]), null); + assert.deepEqual( + ownPromotionPr([ + { number: 9, isCrossRepository: true }, + { number: 10, isCrossRepository: false }, + ]), + { number: 10, isCrossRepository: false } + ); + assert.equal(ownPromotionPr([]), null); +}); diff --git a/scripts/ops/verify-live.mjs b/scripts/ops/verify-live.mjs new file mode 100644 index 000000000..ba3141a5a --- /dev/null +++ b/scripts/ops/verify-live.mjs @@ -0,0 +1,231 @@ +#!/usr/bin/env node +// Checks that rate.vn serves main: the live deployment's commit (reported by +// /api/health) equals main, the database is readable, and the main routes +// answer 200. Docs/Resources/RELEASE.md step 6. +// +// node scripts/ops/verify-live.mjs [--wait ] [--base-url https://rate.vn] +// [--repo DOS/Rate] [--route /path]... +// +// --wait keeps polling while the only problem is the commit (the build of main +// is still deploying); main is re-read on every poll, so a newer promotion is +// followed, never undone. When the live commit is behind main and main's +// production deployment is built, it prints that deployment as the one to +// promote: on 2026-10-24 a redeploy of an older production deployment rolled +// back a release (DOS/Rate#814). It never suggests a deployment of any other +// commit. + +import { parseArgs } from 'node:util'; +import { DEFAULT_REPO, fail, ghJson, isMain, sleep } from './lib.mjs'; + +export const DEFAULT_ROUTES = [ + '/', + '/smartphones', + '/earbuds', + '/deals', + '/api/v1/rate/stats', +]; +const REQUEST_TIMEOUT_MS = 30_000; +const POLL_MS = 20_000; + +/** Problems with the live site; an empty list means it serves main. */ +export function evaluateLive({ mainSha, health, routes }) { + const problems = []; + if (health.status !== 200) { + problems.push({ + kind: 'health', + message: `/api/health returned ${health.status}`, + }); + } + if (health.body?.checks?.database !== 'ok') { + problems.push({ + kind: 'database', + message: `database check is ${health.body?.checks?.database ?? 'missing'}`, + }); + } + const commit = health.body?.commit; + if (!commit) { + problems.push({ + kind: 'commit', + message: + '/api/health reports no commit (a deployment from before DOS/Rate#865, or the site is down)', + }); + } else if (commit !== mainSha) { + problems.push({ + kind: 'commit', + message: `live commit ${commit.slice(0, 8)} is not main ${mainSha.slice(0, 8)}`, + }); + } + for (const r of routes) { + if (r.status !== 200) + problems.push({ + kind: 'route', + message: `${r.path} returned ${r.status}`, + }); + } + return problems; +} + +/** Still worth waiting: only the commit differs, so a build may be deploying. */ +export const onlyCommitMismatch = (problems) => + problems.length > 0 && problems.every((p) => p.kind === 'commit'); + +/** The newest successful Production deployment status of main's commit. */ +export function redeploySource(statuses) { + const ok = statuses.find((s) => s.state === 'success' && s.environment_url); + return ok?.environment_url ?? null; +} + +/** + * What to do about a live commit that is not main. `relation` is the GitHub + * compare status of live...main: 'ahead' means main has commits live lacks, + * i.e. live is behind main. Only then is main's own deployment the fix. + */ +export function restoreAdvice({ relation, source }) { + if (relation === 'behind' || relation === 'diverged') { + return 'Live is not behind main (main moved, or live has commits main lacks). Do not redeploy or promote anything: re-run verify-live.'; + } + if (!source) { + return 'No successful Production deployment of main yet: wait for the Vercel build of main. Never redeploy or promote an older deployment.'; + } + const how = `vercel promote ${source} --scope metados`; + return relation === 'ahead' + ? `Live is behind main. Once main's build is Ready, put it on the domains with: ${how} (a promote does not rebuild and re-enables auto-assignment after a rollback). Never promote or redeploy a deployment built from another commit.` + : `Live commit unknown. If rate.vn does not serve main's build, the only deployment to promote is main's: ${how}`; +} + +// One retry after a network error: a single dropped connection is not an +// outage, while an HTTP error status is reported as it is. +async function request(url, attempt = 1) { + try { + const res = await fetch(url, { + headers: { 'cache-control': 'no-cache' }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + let body = null; + if ((res.headers.get('content-type') ?? '').includes('application/json')) { + body = await res.json().catch(() => null); + } else { + await res.arrayBuffer(); + } + return { status: res.status, body }; + } catch (err) { + if (attempt < 2) { + await sleep(2_000); + return request(url, attempt + 1); + } + const cause = + err instanceof Error ? (err.cause?.code ?? err.name) : 'unknown'; + return { status: `error (${cause})`, body: null }; + } +} + +async function readLive(baseUrl, paths) { + const health = await request(`${baseUrl}/api/health`); + const routes = []; + for (const path of paths) { + routes.push({ path, status: (await request(`${baseUrl}${path}`)).status }); + } + return { health, routes }; +} + +function findRedeploySource(repo, sha) { + const deployments = ghJson([ + 'api', + `repos/${repo}/deployments?sha=${sha}&environment=Production&per_page=5`, + ]); + for (const d of deployments ?? []) { + const url = redeploySource( + ghJson([ + 'api', + `repos/${repo}/deployments/${d.id}/statuses?per_page=10`, + ]) ?? [] + ); + if (url) return url; + } + return null; +} + +function adviceFor(repo, liveSha, mainSha) { + try { + const relation = liveSha + ? ghJson(['api', `repos/${repo}/compare/${liveSha}...${mainSha}`])?.status + : null; + return restoreAdvice({ + relation, + source: findRedeploySource(repo, mainSha), + }); + } catch (err) { + return `Could not look up main's deployment (${err.message}). Do not redeploy or promote anything until it is known.`; + } +} + +export async function verifyLive({ + repo = DEFAULT_REPO, + baseUrl = 'https://rate.vn', + routes = DEFAULT_ROUTES, + waitMs = 0, +} = {}) { + const deadline = Date.now() + waitMs; + let apiErrors = 0; + for (;;) { + // Re-read every poll: a promotion during the wait moves main forward. + let mainSha; + try { + mainSha = ghJson(['api', `repos/${repo}/commits/main`]).sha; + apiErrors = 0; + } catch (err) { + if (++apiErrors >= 3 || Date.now() >= deadline) { + return { + ok: false, + message: `Could not read main (${err.message}). Do not redeploy or promote anything until it is known.`, + }; + } + await sleep(POLL_MS); + continue; + } + const live = await readLive(baseUrl, routes); + const problems = evaluateLive({ mainSha, ...live }); + if (problems.length === 0) { + return { + ok: true, + message: `${baseUrl} serves main ${mainSha.slice(0, 8)}: database ok, ${routes.length} routes 200`, + }; + } + if (!onlyCommitMismatch(problems) || Date.now() >= deadline) { + const lines = problems.map((p) => `- ${p.message}`); + if (problems.some((p) => p.kind === 'commit')) { + lines.push(adviceFor(repo, live.health.body?.commit, mainSha)); + } + return { + ok: false, + message: [ + `${baseUrl} does not serve main ${mainSha.slice(0, 8)}:`, + ...lines, + ].join('\n'), + }; + } + await sleep(POLL_MS); + } +} + +if (isMain(import.meta.url)) { + const { values } = parseArgs({ + options: { + repo: { type: 'string', default: DEFAULT_REPO }, + 'base-url': { type: 'string', default: 'https://rate.vn' }, + route: { type: 'string', multiple: true }, + wait: { type: 'string', default: '0' }, + }, + }); + const waitSeconds = Number(values.wait); + if (!Number.isFinite(waitSeconds) || waitSeconds < 0) + fail('--wait takes seconds'); + const result = await verifyLive({ + repo: values.repo, + baseUrl: values['base-url'].replace(/\/$/, ''), + routes: values.route ?? DEFAULT_ROUTES, + waitMs: waitSeconds * 1000, + }); + if (!result.ok) fail(result.message); + console.log(result.message); +} diff --git a/scripts/ops/verify-live.test.mjs b/scripts/ops/verify-live.test.mjs new file mode 100644 index 000000000..d34935d12 --- /dev/null +++ b/scripts/ops/verify-live.test.mjs @@ -0,0 +1,122 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + evaluateLive, + onlyCommitMismatch, + redeploySource, + restoreAdvice, +} from './verify-live.mjs'; + +const MAIN = 'c20a9ef1b196e26b9a0d5498344ce6dc59f5cb31'; +const SOURCE = 'https://rate-main-metados.vercel.app'; +const healthy = (commit = MAIN) => ({ + status: 200, + body: { status: 'ok', commit, checks: { database: 'ok' } }, +}); +const ok = [{ path: '/', status: 200 }]; + +test('a site that serves main with a readable database passes', () => { + assert.deepEqual( + evaluateLive({ mainSha: MAIN, health: healthy(), routes: ok }), + [] + ); +}); + +test('an older deployment is a commit mismatch worth waiting for', () => { + const problems = evaluateLive({ + mainSha: MAIN, + health: healthy('d24f3bf5a6bb'), + routes: ok, + }); + assert.equal(problems.length, 1); + assert.match( + problems[0].message, + /live commit d24f3bf5 is not main c20a9ef1/ + ); + assert.equal(onlyCommitMismatch(problems), true); +}); + +test('a deployment that reports no commit fails and says why', () => { + const problems = evaluateLive({ + mainSha: MAIN, + health: healthy(null), + routes: ok, + }); + assert.match(problems[0].message, /reports no commit .*#865/); +}); + +test('a database outage or a failing route is not worth waiting for', () => { + const outage = evaluateLive({ + mainSha: MAIN, + health: { + status: 503, + body: { status: 'degraded', commit: MAIN, checks: { database: 'error' } }, + }, + routes: [{ path: '/laptops', status: 500 }], + }); + assert.deepEqual( + outage.map((p) => p.kind), + ['health', 'database', 'route'] + ); + assert.equal(onlyCommitMismatch(outage), false); +}); + +test('an unreachable health endpoint fails every health check', () => { + const problems = evaluateLive({ + mainSha: MAIN, + health: { status: 'error (ETIMEDOUT)', body: null }, + routes: ok, + }); + assert.deepEqual( + problems.map((p) => p.kind), + ['health', 'database', 'commit'] + ); +}); + +test('the redeploy source is the newest successful deployment of main', () => { + assert.equal( + redeploySource([ + { state: 'success', environment_url: SOURCE }, + { state: 'pending', environment_url: '' }, + ]), + SOURCE + ); + assert.equal( + redeploySource([{ state: 'failure', environment_url: 'https://x' }]), + null + ); + assert.equal(redeploySource([]), null); +}); + +test('only a live commit behind main gets main promoted', () => { + const behind = restoreAdvice({ relation: 'ahead', source: SOURCE }); + assert.match(behind, /Live is behind main/); + assert.match( + behind, + /vercel promote https:\/\/rate-main-metados\.vercel\.app --scope metados/ + ); + assert.doesNotMatch(behind, /vercel redeploy/); +}); + +test('a live commit ahead of or diverged from main is never rolled back', () => { + for (const relation of ['behind', 'diverged']) { + const advice = restoreAdvice({ relation, source: SOURCE }); + assert.match(advice, /Do not redeploy or promote anything/); + assert.doesNotMatch(advice, /vercel promote/); + } +}); + +test('without a built deployment of main, it says to wait', () => { + assert.match( + restoreAdvice({ relation: 'ahead', source: null }), + /wait for the Vercel build of main/ + ); +}); + +test('an unknown live commit names only main deployment', () => { + assert.match( + restoreAdvice({ relation: null, source: SOURCE }), + /only deployment to promote is main's: vercel promote/ + ); +});