diff --git a/apps/importer/src/shopee/__tests__/arena-audit.test.ts b/apps/importer/src/shopee/__tests__/arena-audit.test.ts index 017fe7e0b..a383bb04b 100644 --- a/apps/importer/src/shopee/__tests__/arena-audit.test.ts +++ b/apps/importer/src/shopee/__tests__/arena-audit.test.ts @@ -8,6 +8,9 @@ import assert from 'node:assert/strict'; import { ArenaSpecSyncer, UPSTREAM_2026_ARENA_REFERENCE_URLS, + parseGsmArenaHtmlToSpecs, + parseGsmArenaBrandCatalogHtml, + parsePhoneArenaCatalogHtml, type ArenaAuditMasterRow, } from '../arena-spec-syncer.js'; @@ -118,6 +121,54 @@ async function run() { assert.equal(empty.missingFromDatabase.length, UPSTREAM_2026_ARENA_REFERENCE_URLS.length); assert.deepEqual(empty.incompleteSpecMasters, []); + // parseGsmArenaHtmlToSpecs enforces integer RAM/storage bounds, clean TB formatting, and meta URL checks (#831) + const parsedSpec = parseGsmArenaHtmlToSpecs( + `

Samsung Galaxy S26 Ultra

+ 0.5GB 0.5GB RAM, 1200GB 12.5GB RAM, 1200GB 16GB RAM`, + { + gsmarenaUrl: 'https://www.gsmarena.com/samsung_galaxy_s26_ultra-14015.php', + phonearenaUrl: 'https://evil.example/phones/Samsung', + } + ); + assert.equal(parsedSpec.ramGb, 16); + assert.equal(parsedSpec.storageGb, 1200); + assert.equal(parsedSpec.keySpecs.storage, '1200GB'); + assert.equal( + parsedSpec.gsmarenaUrl, + 'https://www.gsmarena.com/samsung_galaxy_s26_ultra-14015.php' + ); + assert.equal(parsedSpec.phonearenaUrl, undefined); + assert.equal(parsedSpec.source, 'gsmarena'); + + const parsedTbSpec = parseGsmArenaHtmlToSpecs( + `

Apple iPad Pro M5

+ 1.5TB 16GB RAM` + ); + assert.equal(parsedTbSpec.storageGb, 1536); + assert.equal(parsedTbSpec.keySpecs.storage, '1.5TB'); + + // parseGsmArenaBrandCatalogHtml normalizes *.gsmarena.com image URLs and drops non-gsmarena image sources (#831) + const gsmCatalog = parseGsmArenaBrandCatalogHtml( + ``, + 'Xiaomi' + ); + assert.equal(gsmCatalog.length, 2); + assert.equal(gsmCatalog[0]?.imageUrl, 'https://fdn2.gsmarena.com/vv/bigpic/xiaomi-18.jpg'); + assert.equal(gsmCatalog[1]?.imageUrl, undefined); + + // parsePhoneArenaCatalogHtml canonicalizes URLs and rejects path traversal outside /phones/ (#831) + const paCatalog = parsePhoneArenaCatalogHtml( + `
+ Apple iPhone 18 Pro + Leaked News +
` + ); + assert.equal(paCatalog.length, 1); + assert.equal(paCatalog[0]?.url, 'https://www.phonearena.com/phones/Apple-iPhone-18-Pro_id12539'); + console.log('✅ ArenaSpecSyncer catalog audit suite passed'); } diff --git a/apps/importer/src/shopee/__tests__/entity-matcher.test.ts b/apps/importer/src/shopee/__tests__/entity-matcher.test.ts index 381ae9cd2..e131c5c93 100644 --- a/apps/importer/src/shopee/__tests__/entity-matcher.test.ts +++ b/apps/importer/src/shopee/__tests__/entity-matcher.test.ts @@ -11,6 +11,8 @@ import { EntityMatcher, ENTITY_MATCH_ACCEPT_THRESHOLD, ENTITY_MATCH_CANDIDATE_FLOOR, + sanitizeHttpUrl, + sanitizeHttpUrlList, } from '../entity-matcher.js'; import { ProductNormalizer, @@ -515,6 +517,118 @@ async function runEntityMatcherSuite() { ' ✅ [7/7] Zod 4 JSON Schema, URL/phone prompt-injection filter, MasterProductSchema defaults & structured logger passed' ); + // 8. DOS/Rate#821: Tier 2 malformed fuzzy row filtering, Tier 3 image/spec bounds, and enrichMasterProductById zero/out-of-bounds fallback + assert.equal(sanitizeHttpUrl('https://rate.vn/img/a.png'), 'https://rate.vn/img/a.png'); + assert.equal(sanitizeHttpUrl('javascript:alert(1)'), null); + assert.equal(sanitizeHttpUrl('[object Object]'), null); + assert.deepEqual( + sanitizeHttpUrlList([ + 'https://rate.vn/img/1.png', + '[object Object]', + 'javascript:alert(1)', + 'https://rate.vn/img/1.png', + ]), + ['https://rate.vn/img/1.png'] + ); + + const malformedFuzzyDb = createMockSupabase({ + fuzzyMatches: [ + { master_id: '', name: 'Missing ID', similarity: 0.95 }, + { master_id: 'nan-sim', name: 'NaN Sim', similarity: Number.NaN }, + { master_id: 'valid-fuzzy-821', name: 'Samsung Galaxy S24 Ultra 256GB', similarity: 0.78 }, + ], + }); + const malformedFuzzyMatcher = new EntityMatcher(malformedFuzzyDb.client, offlineEnricher); + const malformedFuzzyRes = await malformedFuzzyMatcher.resolveOrCreateMasterProduct(s24New); + assert.deepEqual(malformedFuzzyRes, { masterId: 'valid-fuzzy-821', isNew: false }); + + const tier3BoundsDb = createMockSupabase(); + const tier3BoundsMatcher = new EntityMatcher(tier3BoundsDb.client, offlineEnricher); + await tier3BoundsMatcher.resolveOrCreateMasterProduct({ + ...s24New, + ram_gb: 0, + storage_gb: 999_999, + screen_size_inch: -5, + battery_mah: 9_999_999, + images: ['[object Object]', 'https://rate.vn/img/valid.png', 'javascript:void(0)'], + }); + const seededRow = tier3BoundsDb.capturedUpserts[0]?.payload; + assert.equal(seededRow?.ram_gb, null); + assert.equal(seededRow?.storage_gb, null); + assert.equal(seededRow?.screen_size_inch, null); + assert.equal(seededRow?.battery_mah, null); + assert.deepEqual(seededRow?.images, ['https://rate.vn/img/valid.png']); + assert.equal(seededRow?.thumbnail, 'https://rate.vn/img/valid.png'); + + let updatedRow821: Record | undefined; + let forwardedParams821: Record | undefined; + const enrichDb821 = { + from(table: string) { + assert.equal(table, 'master_products'); + return { + select() { + return { + eq() { + return { + async maybeSingle() { + return { + data: { + id: 'master-821', + brand: 'Samsung', + model: 'Galaxy S24 Ultra', + storage_gb: 0, + ram_gb: 99999, + screen_size_inch: 0, + battery_mah: -100, + has_5g: false, + name: 'Samsung Galaxy S24 Ultra 256GB', + description: '', + colors: ['Titanium Gray'], + specifications: { category_slug: 'smartphones' }, + key_specs: {}, + }, + error: null, + }; + }, + }; + }, + }; + }, + update(payload: Record) { + updatedRow821 = payload; + return { + async eq() { + return { error: null }; + }, + }; + }, + }; + }, + } as unknown as RateSupabaseClient; + const stubEnricher821 = { + async fetchSpecsFromAi(params: Record) { + forwardedParams821 = params; + return { + ram_gb: 12, + storage_gb: 256, + screen_size_inch: 6.8, + battery_mah: 5000, + has_5g: true, + specifications: { display: { size_inch: 6.8 } }, + key_specs: { ram: '12GB' }, + }; + }, + } as unknown as AiSpecEnricher; + const enrichMatcher821 = new EntityMatcher(enrichDb821, stubEnricher821); + assert.equal(await enrichMatcher821.enrichMasterProductById('master-821'), true); + assert.equal(forwardedParams821?.storageGb, null); + assert.equal(forwardedParams821?.ramGb, null); + assert.equal(updatedRow821?.ram_gb, 12); + assert.equal(updatedRow821?.storage_gb, 256); + assert.equal(updatedRow821?.screen_size_inch, 6.8); + assert.equal(updatedRow821?.battery_mah, 5000); + console.log(' ✅ [8/8] DOS/Rate#821 EntityMatcher bounds & HTTP URL sanitization passed'); + console.log('🎉 All EntityMatcher & Open-Core Audit C6/C8/T15/Low tests passed!'); } diff --git a/apps/importer/src/shopee/__tests__/open-core-intelligence.test.ts b/apps/importer/src/shopee/__tests__/open-core-intelligence.test.ts index f5c97aa6f..6030696ea 100644 --- a/apps/importer/src/shopee/__tests__/open-core-intelligence.test.ts +++ b/apps/importer/src/shopee/__tests__/open-core-intelligence.test.ts @@ -153,6 +153,168 @@ async function runSuite() { keyOf('Laptop ASUS Vivo Book 15 X1504 512GB'), keyOf('Laptop ASUS Vivobook 15 X1504 512GB') ); + // Xiaomi's "Mipad" and "Mi Pad" are no Apple iPad, and they are tablets (DOS/Rate#826). + const mipad5 = 'Xiaomi Mipad 5 Pro 12.4 Tablet, 8G/256G Ram, Mi Pad 5 Pro 12.4 - Imported Goods'; + assert.equal(brandOf(mipad5), 'Xiaomi'); + assert.match(keyOf(mipad5), /^xiaomi:/); + assert.equal( + brandOf('Máy tính bảng Xiaomi Mi Pad 5 Mipad 5 Ram 6G /128Gb likenew Hàng Nhập khẩu'), + 'Xiaomi' + ); + assert.equal(brandOf('Mipad 5 Pro 128GB'), 'Xiaomi'); + assert.equal(keyOf('Mi pad 6 8GB 256GB'), 'xiaomi:mi-pad-6:256gb'); + assert.equal(categoryOf('Mi pad 6 8GB 256GB'), 'tablets'); + assert.equal(categoryOf('Mipad 5 Pro 128GB'), 'tablets'); + // An iPad after a space or punctuation is still Apple; "Kami Pad" is no Xiaomi. + assert.equal(keyOf('Apple iPad A16 11 inch 2025 Wifi 128GB'), 'apple:ipad-a16:128gb'); + assert.equal(brandOf('(iPad Air M4) 256GB'), 'Apple'); + assert.equal(brandOf('Ốp lưng cho iPad/Mipad 5'), 'Apple'); + assert.equal(brandOf('Kami Pad 5 128GB'), 'Khác'); + // A size written apart ends the model like one written together (DOS/Rate#815). + assert.equal( + keyOf('Samsung Galaxy S24 Ultra 256 GB Chính hãng'), + keyOf('Samsung Galaxy S24 Ultra 256GB Chính hãng') + ); + assert.equal( + keyOf('Samsung Galaxy S24 Ultra 256 GB Chính hãng'), + 'samsung:galaxy-s24-ultra:256gb' + ); + assert.equal(keyOf('Xiaomi 14 Ultra 1 TB'), 'xiaomi:14-ultra:1024gb'); + // Only the number right before the unit ends the model. + assert.equal(keyOf('Xiaomi 14 Ultra 512 GB'), 'xiaomi:14-ultra:512gb'); + // "ProMax" written together is "Pro Max", for iPhones and the generic fallback. + assert.equal(keyOf('iPhone 16 ProMax 256GB'), 'apple:iphone-16-pro-max:256gb'); + assert.equal(keyOf('iphone 16 promax 256gb'), 'apple:iphone-16-pro-max:256gb'); + assert.equal(keyOf('iPhone 16ProMax 256GB'), 'apple:iphone-16-pro-max:256gb'); + assert.equal(keyOf('iPhone 16Plus 128GB'), 'apple:iphone-16-plus:128gb'); + assert.equal(keyOf('iPhone 16e 128GB'), 'apple:iphone-16e:128gb'); + assert.equal(keyOf('Xiaomi 17 ProMax 512GB'), keyOf('Xiaomi 17 Pro Max 512GB')); + // "Max" after "Pro" stays past the 4-token cap: the Pro Max is not the Pro. + assert.equal(keyOf('Xiaomi Redmi Note 17 Pro Max 512GB'), 'xiaomi:redmi-note-17-pro-max:512gb'); + assert.equal(keyOf('Xiaomi Redmi Note 17 ProMax 512GB'), 'xiaomi:redmi-note-17-pro-max:512gb'); + assert.equal(keyOf('Xiaomi Redmi Note 17 Pro 512GB'), 'xiaomi:redmi-note-17-pro:512gb'); + // Sony's WF/WH-1000XM line with or without the hyphen. + assert.equal(keyOf('Tai nghe TWS Sony WF1000XM5'), keyOf('Tai nghe TWS Sony WF-1000XM5')); + assert.equal(keyOf('Tai nghe TWS Sony WF1000XM5'), 'sony:wf-1000xm5:unknown'); + assert.equal(keyOf('Sony WH1000XM6 Bluetooth'), 'sony:wh-1000xm6:unknown'); + assert.equal(brandOf('Tai nghe WF1000XM5'), 'Sony'); + assert.equal(categoryOf('Sony WF1000XM5'), 'earbuds'); + // Without the hyphen it needs the "XM": other "Wh1000" strings are not Sony. + assert.equal(brandOf('Laptop Lenovo ThinkPad 50Wh1000 16 GB'), 'Lenovo'); + assert.equal(brandOf('Huawei FreeBuds WH1000'), 'Huawei'); + // Apple lines no pattern knew are not iPhones (DOS/Rate#817). + assert.equal( + keyOf('iPad A16 Wifi 128GB 2025 | Chính hãng Apple Việt Nam'), + 'apple:ipad-a16:128gb' + ); + assert.equal(keyOf('Apple iPad (A16) Wi-Fi 128GB'), 'apple:ipad-a16:128gb'); + assert.equal(keyOf('Apple iPad Gen 10 64GB'), keyOf('Apple iPad 10 64GB')); + assert.equal( + keyOf('MacBook Neo 13 inch A18 Pro 2026 6CPU 5GPU 8GB 256GB'), + 'apple:macbook-neo:256gb' + ); + assert.equal(categoryOf('MacBook Neo 13 inch A18 Pro 2026 6CPU 5GPU 8GB 256GB'), 'laptops'); + assert.equal(keyOf('iPhone Air 256GB'), 'apple:iphone-air:256gb'); + assert.equal(keyOf('iPhone Duo 512GB'), 'apple:iphone-duo:512gb'); + // An A16 or later chip after the screen size or generation names the base iPad; an + // earlier chip leaves the generation. + assert.equal(keyOf('Apple iPad 11 inch (A16) 128GB'), 'apple:ipad-a16:128gb'); + assert.equal(keyOf('iPad 11 (A16) WiFi 128GB - Hàng Chính Hãng'), 'apple:ipad-a16:128gb'); + assert.equal(keyOf('Apple iPad Gen 11 A16 128GB'), 'apple:ipad-a16:128gb'); + assert.equal(keyOf('iPad 11-inch (A16) 128GB'), 'apple:ipad-a16:128gb'); + assert.equal(keyOf('iPad Gen 10 (A14 Bionic) 64GB'), keyOf('Apple iPad 10 64GB')); + assert.equal(keyOf('Apple iPad 10.9 inch (A14) 64GB'), keyOf('Apple iPad 10 64GB')); + assert.equal(keyOf('MacBook Neoprene Sleeve 13 inch'), 'apple:macbook:unknown'); + // iPad mini 6 and 7 are two models (DOS/Rate#819): "(A17 Pro)" is the 7th, "8.3 inch" is + // its screen, and a size or a year after "mini" is no generation. + assert.equal(keyOf('Apple iPad mini 6 Wi-Fi 64GB'), 'apple:ipad-mini-6:64gb'); + assert.equal(keyOf('Apple iPad mini 7 64GB'), 'apple:ipad-mini-7:64gb'); + assert.equal(keyOf('Apple iPad mini 8.3 inch (A17 Pro) Wi-Fi 128GB'), 'apple:ipad-mini-7:128gb'); + assert.equal(keyOf('Apple iPad mini 6, Wi-Fi 64GB'), 'apple:ipad-mini-6:64gb'); + assert.equal(keyOf('Apple iPad mini 8,3 inch Wi-Fi 64GB'), 'apple:ipad-mini:64gb'); + assert.equal(keyOf('iPad mini 7 (A17 Pro) Wifi 128GB'), 'apple:ipad-mini-7:128gb'); + assert.equal(keyOf('iPad mini (6th generation) 64GB'), 'apple:ipad-mini-6:64gb'); + assert.equal(keyOf('iPad mini 64GB'), 'apple:ipad-mini:64gb'); + assert.equal(keyOf('iPad mini 2024 128GB'), 'apple:ipad-mini:128gb'); + // A "5G" or "4G" network or a storage after "mini" is no generation: "mini 5G" is not the + // 2019 iPad mini 5, and "mini 64 Wifi" is no "mini 64". + assert.equal(keyOf('Apple iPad mini 5G 64GB'), 'apple:ipad-mini:64gb'); + assert.equal(keyOf('Apple iPad mini 4G 64GB'), 'apple:ipad-mini:64gb'); + assert.equal(keyOf('Apple iPad mini 64 Wifi'), 'apple:ipad-mini:unknown'); + assert.equal(keyOf('Apple iPad mini 64/256GB'), 'apple:ipad-mini:256gb'); + assert.equal(keyOf('Apple iPad mini 6 5G 64GB'), 'apple:ipad-mini-6:64gb'); + assert.equal( + normalizer.normalize(createDummyShopeeProduct('iPad Mini 7 Wifi 128GB')).model, + 'iPad mini 7' + ); + assert.equal( + normalizer.normalize(createDummyShopeeProduct('MACBOOK NEO 13 INCH 256GB')).model, + 'MacBook Neo' + ); + assert.equal( + normalizer.normalize(createDummyShopeeProduct('iPhone Air 256GB')).model, + 'iPhone Air' + ); + assert.equal( + normalizer.normalize(createDummyShopeeProduct('iPhone Duo 512GB')).model, + 'iPhone Duo' + ); + assert.equal( + normalizer.normalize(createDummyShopeeProduct('iPad (a16) 128GB')).model, + 'iPad A16' + ); + // "(2nd generation)" is the generation number, one or two digits, any ordinal. + assert.equal(keyOf('Apple Watch Ultra (2nd generation) 49mm'), keyOf('Apple Watch Ultra 2 49mm')); + assert.equal(keyOf('AirPods Pro (2nd generation)'), keyOf('AirPods Pro 2')); + assert.equal(keyOf('AirPods Pro (2nd gen) USB-C'), keyOf('AirPods Pro 2 USB-C')); + assert.equal(keyOf('AirPods (3rd generation)'), keyOf('AirPods 3')); + assert.equal(keyOf('iPhone SE (3rd generation) 64GB'), keyOf('iPhone SE 3 64GB')); + assert.equal(keyOf('iPad (10th generation) 64GB'), keyOf('Apple iPad 10 64GB')); + assert.equal( + keyOf('Apple Watch Series (10th generation) 46mm'), + keyOf('Apple Watch Series 10 46mm') + ); + // The persisted name keeps one space, and "Series" alone is not the SE. + const ultra2 = normalizer.normalize( + createDummyShopeeProduct('Apple Watch Ultra (2nd generation) 49mm') + ); + assert.equal(ultra2.model, 'Watch Ultra 2'); + assert.ok(!/\s{2}/.test(ultra2.name), ultra2.name); + assert.equal(keyOf('Apple Watch Series'), 'apple:watch:unknown'); + // A case size or storage after "SE" is not its generation: one SE, not "SE 40"; an + // ordinal one is ("SE 2nd Gen"). + assert.equal(keyOf('Apple Watch SE 40mm'), 'apple:watch-se:unknown'); + assert.equal(keyOf('Apple Watch SE 44mm GPS'), 'apple:watch-se:unknown'); + assert.equal(keyOf('Apple Watch SE 44 mm GPS'), 'apple:watch-se:unknown'); + assert.equal(keyOf('Apple Watch SE 2nd Gen GPS 44mm'), 'apple:watch-se-2:unknown'); + assert.equal(keyOf('Apple Watch SE 3rd Gen 44mm'), 'apple:watch-se-3:unknown'); + assert.equal(keyOf('Apple Watch SE 2022 40mm'), 'apple:watch-se-2022:unknown'); + assert.equal(keyOf('Apple Watch SE 32GB'), 'apple:watch-se:32gb'); + assert.equal(keyOf('Apple Watch SE2 40mm'), 'apple:watch-se2:unknown'); + // With no line matched, the family the title names first, else iPhone. + assert.equal(keyOf('MacBook M4'), 'apple:macbook:unknown'); + assert.equal(keyOf('Apple "iPad" 256GB'), 'apple:ipad:256gb'); + assert.equal(keyOf('Đặt trước iPhone mới, ưu đãi MacBook'), 'apple:iphone:unknown'); + assert.equal(keyOf('Túi MacBook, iPhone mới'), 'apple:macbook:unknown'); + assert.equal(keyOf('Apple Watch Band 45mm'), 'apple:watch:unknown'); + assert.equal(keyOf('Apple AirPods Case'), 'apple:airpods:unknown'); + assert.equal(keyOf('Apple Swatch band'), 'apple:iphone:unknown'); + assert.equal(keyOf('Apple watchOS 11 hướng dẫn'), 'apple:iphone:unknown'); + // Long whitespace stays linear in the Apple patterns (CodeQL js/polynomial-redos). + const appleRunStart = Date.now(); + const spaces = ' '.repeat(50_000); + for (const title of [ + `Apple iPad${spaces}x`, + `Apple iPad 11${spaces}x`, + `Apple iPad 11 (${spaces}x`, + `Apple Watch${spaces}(2nd generation)`, + `Apple Watch SE${spaces}2`, + `Apple iPad mini 7${spaces}inch`, + `Apple Watch SE 2${spaces}mm`, + ]) { + keyOf(title); + } + assert.ok(Date.now() - appleRunStart < 1_000, 'a 50k-space Apple title must not hang'); // "Omen" only as a word: Sennheiser "Momentum" is not HP. assert.equal(brandOf('Tai nghe Sennheiser Momentum 4 Wireless'), 'Sennheiser'); assert.equal(brandOf('Laptop HP Omen 16 512GB'), 'HP'); @@ -342,7 +504,7 @@ async function runSuite() { createDummyShopeeProduct('Tai nghe Bluetooth Samsung Galaxy Buds3 Pro Chính hãng') ); assert.equal(galaxyBuds3Pro.category_slug, 'earbuds'); - assert.equal(galaxyBuds3Pro.model_key, 'samsung:galaxy-buds3-pro:1gb'); + assert.equal(galaxyBuds3Pro.model_key, 'samsung:galaxy-buds3-pro:unknown'); assert.equal(galaxyBuds3Pro.is_benchmark_enriched, true); console.log(' ✅ [3/5] Multi-Category Benchmark Enrichment (5 Categories, 12 Flagships) passed'); @@ -857,10 +1019,10 @@ async function runSuite() { ); assert.equal(unbenchmarkedEarbuds.category_slug, 'earbuds'); assert.equal(unbenchmarkedEarbuds.specifications.category_slug, 'earbuds'); - assert.equal(unbenchmarkedEarbuds.storage_gb, 1); + assert.equal(unbenchmarkedEarbuds.storage_gb, null); assert.equal(unbenchmarkedEarbuds.variant_name, 'Standard'); assert.equal(unbenchmarkedEarbuds.needs_review, false); - assert.ok(unbenchmarkedEarbuds.model_key.endsWith(':1gb')); + assert.ok(unbenchmarkedEarbuds.model_key.endsWith(':unknown')); assert.equal(unbenchmarkedEarbuds.specifications.hardware?.os, 'Bluetooth 5.3 Audio DSP'); assert.equal(unbenchmarkedEarbuds.specifications.cellular?.has_5g, false); assert.equal(unbenchmarkedEarbuds.specifications.cellular?.sim_type, 'No SIM'); @@ -1042,7 +1204,104 @@ async function runSuite() { assert.equal(brk521.suspicious_ratio_pct, 50); } - console.log(' ✅ [13/13] AiReviewAnalyzer MerchantRiskBreakdown Schema Alignment (#521) passed'); + console.log(' ✅ [13/14] AiReviewAnalyzer MerchantRiskBreakdown Schema Alignment (#521) passed'); + + // ========================================================================= + // 14. Earbud :unknown Keys, AirPods Pro Years, iPad RAM & Non-Phone Spec Hygiene (#643, #657) + // ========================================================================= + { + // #643: Earbuds have no storage variant and must normalize to :unknown + assert.equal(keyOf('Tai nghe Samsung Galaxy Buds3 Pro'), 'samsung:galaxy-buds3-pro:unknown'); + assert.equal(keyOf('AirPods Pro 2'), 'apple:airpods-pro-2:unknown'); + assert.equal(keyOf('Apple AirPods 4'), 'apple:airpods-4:unknown'); + + // #643: FPT Shop AirPods Pro 2022 / 2023 map to AirPods Pro 2 (:unknown) + assert.equal(keyOf('Tai nghe AirPods Pro 2022'), 'apple:airpods-pro-2:unknown'); + assert.equal(keyOf('Tai nghe AirPods Pro 2023'), 'apple:airpods-pro-2:unknown'); + assert.equal(keyOf('Tai nghe AirPods Pro (2023) USB-C'), 'apple:airpods-pro-2:unknown'); + + // #643: FPT Shop names 256GB iPad Pro M5 by its 12GB RAM + const ipadProM5Ram = normalizer.normalize( + createDummyShopeeProduct('iPad Pro 11 M5 2025 WiFi 12GB') + ); + assert.equal(ipadProM5Ram.model_key, 'apple:ipad-pro-m5:256gb'); + assert.equal(ipadProM5Ram.ram_gb, 12); + assert.equal(ipadProM5Ram.storage_gb, 256); + + const ipadAirRamOnly = normalizer.normalize( + createDummyShopeeProduct('iPad Air 11 M3 2025 WiFi 8GB') + ); + assert.equal(ipadAirRamOnly.model_key, 'apple:ipad-air-m3:unknown'); + assert.equal(ipadAirRamOnly.ram_gb, 8); + assert.equal(ipadAirRamOnly.storage_gb, null); + + // #643: Vietnamese diacritic translocation in slugify & Cloudflare challenge brand safety + assert.equal(brandOf('Just a moment...'), 'Khác'); + assert.equal(brandOf('Please wait a moment'), 'Khác'); + assert.equal(keyOf('Just a moment...'), 'khac:just-a-moment:unknown'); + assert.equal( + keyOf('Bàn phím dành cho Laptop Lenovo ThinkPad X1'), + 'lenovo:ban-phim-danh-cho:unknown' + ); + assert.equal(keyOf('Đồng hồ Samsung Galaxy Watch7 44mm'), 'samsung:galaxy-watch7:unknown'); + + // #657 Case 1: Title-stated RAM beats benchmark RAM + const s26Ultra12Gb = normalizer.normalize( + createDummyShopeeProduct('Samsung Galaxy S26 Ultra - 12GB/256GB') + ); + assert.equal(s26Ultra12Gb.model_key, 'samsung:galaxy-s26-ultra:256gb'); + assert.equal(s26Ultra12Gb.ram_gb, 12); + assert.equal(s26Ultra12Gb.specifications.hardware?.ram_gb, 12); + assert.equal(s26Ultra12Gb.key_specs.ram, '12GB'); + + const s26UltraBenchmarkRam = normalizer.normalize( + createDummyShopeeProduct('Samsung Galaxy S26 Ultra 256GB') + ); + assert.equal(s26UltraBenchmarkRam.ram_gb, 16); + assert.equal(s26UltraBenchmarkRam.specifications.hardware?.ram_gb, 16); + + // #657 Case 2 & 3: Earbuds, watches, tablets, laptops get no phone-shaped storage/RAM/screen/SAR/Face ID + const wf1000xm6 = normalizer.normalize(createDummyShopeeProduct('Tai nghe Sony WF-1000XM6')); + assert.equal(wf1000xm6.model_key, 'sony:wf-1000xm6:unknown'); + assert.equal(wf1000xm6.ram_gb, null); + assert.equal(wf1000xm6.storage_gb, null); + assert.equal(wf1000xm6.screen_size_inch, null); + assert.equal(wf1000xm6.specifications.hardware?.ram_gb, undefined); + assert.equal(wf1000xm6.specifications.hardware?.storage_gb, undefined); + assert.equal(wf1000xm6.specifications.display?.size_inch, undefined); + assert.equal(wf1000xm6.specifications.display?.refresh_rate_hz, undefined); + assert.equal(wf1000xm6.specifications.display?.ppi, undefined); + assert.equal(wf1000xm6.specifications.regulatory?.sar_head, undefined); + assert.equal(wf1000xm6.specifications.regulatory?.sar_body, undefined); + assert.equal(wf1000xm6.specifications.design?.biometrics, undefined); + + const airpodsPro2 = normalizer.normalize(createDummyShopeeProduct('Tai nghe AirPods Pro 2')); + assert.equal(airpodsPro2.model_key, 'apple:airpods-pro-2:unknown'); + assert.equal(airpodsPro2.ram_gb, null); + assert.equal(airpodsPro2.storage_gb, null); + assert.equal(airpodsPro2.screen_size_inch, null); + assert.equal(airpodsPro2.specifications.design?.biometrics, undefined); + assert.equal(airpodsPro2.specifications.regulatory?.sar_head, undefined); + + assert.equal(ipad.specifications.design?.biometrics, undefined); + assert.equal(ipad.specifications.regulatory?.sar_head, undefined); + assert.equal(macbook.specifications.design?.biometrics, undefined); + assert.equal(macbook.specifications.regulatory?.sar_head, undefined); + assert.equal(watchUltra.ram_gb, null); + assert.equal(watchUltra.specifications.design?.biometrics, undefined); + assert.equal(watchUltra.specifications.regulatory?.sar_head, undefined); + + // Smartphones retain their template SAR and biometrics + assert.equal(s24.specifications.regulatory?.sar_head, '1.10 W/kg'); + assert.equal(s24.specifications.regulatory?.sar_body, '1.15 W/kg'); + assert.equal(s24.specifications.design?.biometrics, 'Fingerprint (under display), Face unlock'); + const iphone16 = normalizer.normalize(createDummyShopeeProduct('iPhone 16 Pro Max 256GB')); + assert.equal(iphone16.specifications.design?.biometrics, 'Face ID'); + assert.equal(iphone16.specifications.regulatory?.sar_head, '1.10 W/kg'); + } + console.log( + ' ✅ [14/14] Earbud :unknown Keys, AirPods Pro Years, iPad RAM & Non-Phone Spec Hygiene (#643, #657) passed' + ); console.log('🎉 All @repo/crawlers Open-Core Intelligence tests passed!'); } diff --git a/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts b/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts index 2507ed027..582fc17c5 100644 --- a/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts +++ b/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts @@ -8,7 +8,11 @@ */ import assert from 'node:assert/strict'; import { trimTrailingSlashes } from '../ai-spec-enricher.js'; -import { ShopeeIngestionPipeline, platformReviewIdFor } from '../shopee-to-supabase.js'; +import { + ShopeeIngestionPipeline, + normalizeScrapedProductPrices, + platformReviewIdFor, +} from '../shopee-to-supabase.js'; import type { RateSupabaseClient, ShopeeProduct, ShopeeReview } from '../types.js'; function createBaseProduct(overrides: Partial = {}): ShopeeProduct { @@ -670,6 +674,41 @@ async function runTests() { ); console.log(' ✅ [6/6] Duplicate detection past the 1000-row cap passed'); + // 7. DOS/Rate#821: normalizeScrapedProductPrices HTTP URL sanitization and ingestProduct unpriced rejection + const sanitizedProduct = normalizeScrapedProductPrices({ + ...createBaseProduct(), + images: ['[object Object]', 'https://rate.vn/img/clean.png', 'javascript:alert(1)'], + shop: { + ...createBaseProduct().shop, + url: 'javascript:alert(1)', + }, + reviews: [ + { + reviewId: 'rev-821', + author: 'Tester', + authorAvatar: 'data:image/png;base64,AAAA', + ratingStar: 5, + comment: 'Good', + images: ['[object Object]', 'https://rate.vn/img/rev.png'], + }, + ], + }); + assert.deepEqual(sanitizedProduct.images, ['https://rate.vn/img/clean.png']); + assert.equal(sanitizedProduct.shop.url, undefined); + assert.equal(sanitizedProduct.reviews[0]?.authorAvatar, undefined); + assert.deepEqual(sanitizedProduct.reviews[0]?.images, ['https://rate.vn/img/rev.png']); + + const { stub: unpricedStub } = createPipelineStubSupabase(); + const unpricedPipeline = new ShopeeIngestionPipeline({ + supabaseClient: unpricedStub, + apiKey: '', + }); + await assert.rejects( + () => unpricedPipeline.ingestProduct(createBaseProduct({ priceMin: 0, priceMax: 0 })), + /Cannot ingest unpriced or out-of-bounds product listing/ + ); + console.log(' ✅ [7/7] DOS/Rate#821 HTTP URL sanitization & unpriced rejection passed'); + console.log('\n🎉 ALL SHOPEE INGESTION PIPELINE TESTS PASSED!'); } diff --git a/apps/importer/src/shopee/ai-spec-enricher.ts b/apps/importer/src/shopee/ai-spec-enricher.ts index a7bf5bd0a..35e8445d6 100644 --- a/apps/importer/src/shopee/ai-spec-enricher.ts +++ b/apps/importer/src/shopee/ai-spec-enricher.ts @@ -16,7 +16,7 @@ import { type KeySpecs, } from '@repo/shared-data'; import type { NormalizedProductCandidate } from './product-normalizer.js'; -import { computeRetryDelayMs, createImporterLogger } from './logger.js'; +import { computeRetryDelayMs, createImporterLogger, sanitizeAiGeneratedText } from './logger.js'; export type SupportedAiProvider = 'openai' | 'dos-ai' | 'gemini' | 'deepseek' | 'groq' | 'openrouter' | 'ollama'; @@ -41,10 +41,10 @@ export interface AiEnrichmentResponse { } export const AiSpecEnrichmentPayloadSchema = z.object({ - ram_gb: z.number().positive().nullable().optional(), - storage_gb: z.number().positive().nullable().optional(), - screen_size_inch: z.number().positive().nullable().optional(), - battery_mah: z.number().positive().nullable().optional(), + ram_gb: z.number().int().positive().max(2048).nullable().optional(), + storage_gb: z.number().int().positive().max(65536).nullable().optional(), + screen_size_inch: z.number().positive().max(120).nullable().optional(), + battery_mah: z.number().int().positive().max(500000).nullable().optional(), has_5g: z.boolean().optional(), specifications: ProductSpecificationsSchema, key_specs: KeySpecsSchema.optional(), @@ -121,6 +121,19 @@ function toBoundedNumberOrUndefined( return num; } +function toBoundedIntOrUndefined(val: unknown, max: number, allowZero = false): number | undefined { + const num = toBoundedNumberOrUndefined(val, max, allowZero); + if (num === undefined || !Number.isInteger(num)) return undefined; + return num; +} + +function sanitizeKeySpecText(val: unknown): string | undefined { + const str = toStringOrUndefined(val); + if (!str) return undefined; + const cleaned = sanitizeAiGeneratedText(str).trim().slice(0, 160); + return cleaned.length > 0 ? cleaned : undefined; +} + function toBooleanOrUndefined(val: unknown): boolean | undefined { if (typeof val === 'boolean') return val; if (typeof val === 'string') { @@ -202,8 +215,8 @@ export function sanitizeSpecifications(raw: unknown): ProductSpecifications { size_inch: toBoundedNumberOrUndefined(rawDisplay.size_inch, 120), type: toStringOrUndefined(rawDisplay.type), resolution: toStringOrUndefined(rawDisplay.resolution), - refresh_rate_hz: toBoundedNumberOrUndefined(rawDisplay.refresh_rate_hz, 1000), - ppi: toBoundedNumberOrUndefined(rawDisplay.ppi, 5000), + refresh_rate_hz: toBoundedIntOrUndefined(rawDisplay.refresh_rate_hz, 1000), + ppi: toBoundedIntOrUndefined(rawDisplay.ppi, 5000), protection: toStringOrUndefined(rawDisplay.protection), features: toStringOrUndefined(rawDisplay.features), }) @@ -215,8 +228,8 @@ export function sanitizeSpecifications(raw: unknown): ProductSpecifications { chipset: toStringOrUndefined(rawHardware.chipset), cpu: toStringOrUndefined(rawHardware.cpu), gpu: toStringOrUndefined(rawHardware.gpu), - ram_gb: toBoundedNumberOrUndefined(rawHardware.ram_gb, 2048), - storage_gb: toBoundedNumberOrUndefined(rawHardware.storage_gb, 65536), + ram_gb: toBoundedIntOrUndefined(rawHardware.ram_gb, 2048), + storage_gb: toBoundedIntOrUndefined(rawHardware.storage_gb, 65536), storage_expansion: toStringOrUndefined(rawHardware.storage_expansion), os: toStringOrUndefined(rawHardware.os), }) @@ -225,7 +238,7 @@ export function sanitizeSpecifications(raw: unknown): ProductSpecifications { const rawBattery = asRecord(root.battery); const batteryParsed = rawBattery ? BatterySchema.safeParse({ - capacity_mah: toBoundedNumberOrUndefined(rawBattery.capacity_mah, 500000), + capacity_mah: toBoundedIntOrUndefined(rawBattery.capacity_mah, 500000), type: toStringOrUndefined(rawBattery.type), charging: toStringOrUndefined(rawBattery.charging), fast_charging_w: toBoundedNumberOrUndefined(rawBattery.fast_charging_w, 1000, true), @@ -420,6 +433,21 @@ export function trimTrailingSlashes(rawUrl: string): string { return end === rawUrl.length ? rawUrl : rawUrl.slice(0, end); } +function resolveValidHttpBaseUrl(candidateUrl: string, fallbackUrl: string): string { + const cleaned = trimTrailingSlashes((candidateUrl || '').trim()); + if (cleaned) { + try { + const parsed = new URL(cleaned); + if ((parsed.protocol === 'http:' || parsed.protocol === 'https:') && parsed.hostname) { + return cleaned; + } + } catch { + // fall through to fallbackUrl + } + } + return trimTrailingSlashes(fallbackUrl); +} + export class AiSpecEnricher { private provider: SupportedAiProvider; private apiKey: string; @@ -434,19 +462,25 @@ export class AiSpecEnricher { const defaults = PROVIDER_DEFAULTS[this.provider]; this.apiKey = config.apiKey ?? this.resolveApiKey(this.provider); - this.baseUrl = trimTrailingSlashes( + this.baseUrl = resolveValidHttpBaseUrl( config.baseUrl || process.env.AI_BASE_URL || process.env.DOS_AI_BASE_URL || process.env.OPENAI_BASE_URL || - defaults.baseUrl + defaults.baseUrl, + defaults.baseUrl ); this.model = config.model || process.env.AI_MODEL || process.env.OPENAI_MODEL || defaults.defaultModel; - this.timeoutMs = config.timeoutMs ?? Number(process.env.AI_TIMEOUT_MS || 25000); - this.temperature = config.temperature ?? 0.1; + const rawTimeout = config.timeoutMs ?? Number(process.env.AI_TIMEOUT_MS || 25000); + this.timeoutMs = + Number.isFinite(rawTimeout) && rawTimeout >= 1000 && rawTimeout <= 120000 + ? Math.round(rawTimeout) + : 25000; + const rawTemp = config.temperature ?? 0.1; + this.temperature = Number.isFinite(rawTemp) && rawTemp >= 0 && rawTemp <= 2 ? rawTemp : 0.1; } private resolveProvider(explicit?: SupportedAiProvider): SupportedAiProvider { @@ -675,8 +709,8 @@ REQUIRED JSON SCHEMA: return null; } - const safeParamStorageGb = toBoundedNumberOrUndefined(params.storageGb, 65536) ?? null; - const safeParamRamGb = toBoundedNumberOrUndefined(params.ramGb, 2048) ?? null; + const safeParamStorageGb = toBoundedIntOrUndefined(params.storageGb, 65536) ?? null; + const safeParamRamGb = toBoundedIntOrUndefined(params.ramGb, 2048) ?? null; const catKey = (params.categorySlug || 'smartphones').trim().toLowerCase() || 'smartphones'; const cacheKey = `${catKey}:${params.brand.toLowerCase()}:${params.model.toLowerCase()}:${safeParamStorageGb || 'default'}:${safeParamRamGb || 'default'}`; const cached = this.memoryCache.get(cacheKey); @@ -780,13 +814,13 @@ REQUIRED JSON SCHEMA: const ramGb = safeParamRamGb ?? - toBoundedNumberOrUndefined(parsedJson.ram_gb, 2048) ?? + toBoundedIntOrUndefined(parsedJson.ram_gb, 2048) ?? specifications.hardware?.ram_gb ?? null; const storageGb = safeParamStorageGb ?? - toBoundedNumberOrUndefined(parsedJson.storage_gb, 65536) ?? + toBoundedIntOrUndefined(parsedJson.storage_gb, 65536) ?? specifications.hardware?.storage_gb ?? null; @@ -796,7 +830,7 @@ REQUIRED JSON SCHEMA: null; const batteryMah = - toBoundedNumberOrUndefined(parsedJson.battery_mah, 500000) ?? + toBoundedIntOrUndefined(parsedJson.battery_mah, 500000) ?? specifications.battery?.capacity_mah ?? null; @@ -810,28 +844,28 @@ REQUIRED JSON SCHEMA: const rawKeySpecs = parsedJson.key_specs || {}; const keySpecsCandidate: KeySpecs = { screen: - toStringOrUndefined(rawKeySpecs.screen) || + sanitizeKeySpecText(rawKeySpecs.screen) || (screenSizeInch ? `${screenSizeInch}" ${specifications.display?.type?.split(',')[0] || 'OLED'}` : undefined), chip: - toStringOrUndefined(rawKeySpecs.chip) || + sanitizeKeySpecText(rawKeySpecs.chip) || specifications.hardware?.chipset?.split('(')[0]?.trim(), camera: - toStringOrUndefined(rawKeySpecs.camera) || + sanitizeKeySpecText(rawKeySpecs.camera) || specifications.camera?.main_camera?.split(',')[0]?.trim(), battery: - toStringOrUndefined(rawKeySpecs.battery) || + sanitizeKeySpecText(rawKeySpecs.battery) || (batteryMah ? `${batteryMah} mAh${specifications.battery?.fast_charging_w ? ` (${specifications.battery.fast_charging_w}W)` : ''}` : undefined), storage: storageGb !== null - ? storageGb >= 1024 + ? storageGb >= 1024 && storageGb % 512 === 0 ? `${storageGb / 1024}TB` : `${storageGb}GB` - : toStringOrUndefined(rawKeySpecs.storage), - ram: ramGb !== null ? `${ramGb}GB` : toStringOrUndefined(rawKeySpecs.ram), + : sanitizeKeySpecText(rawKeySpecs.storage), + ram: ramGb !== null ? `${ramGb}GB` : sanitizeKeySpecText(rawKeySpecs.ram), }; const keySpecsParsed = KeySpecsSchema.safeParse(keySpecsCandidate); @@ -886,11 +920,17 @@ REQUIRED JSON SCHEMA: }; } + const validCandidateRam = toBoundedIntOrUndefined(candidate.ram_gb, 2048) ?? null; + const validCandidateStorage = toBoundedIntOrUndefined(candidate.storage_gb, 65536) ?? null; + const validCandidateScreen = + toBoundedNumberOrUndefined(candidate.screen_size_inch, 120) ?? null; + const validCandidateBattery = toBoundedIntOrUndefined(candidate.battery_mah, 500000) ?? null; + const aiResult = await this.fetchSpecsFromAi({ brand: candidate.brand, model: candidate.model, - storageGb: candidate.storage_gb, - ramGb: candidate.ram_gb, + storageGb: validCandidateStorage, + ramGb: validCandidateRam, rawTitle, rawDescription: candidate.description, categorySlug: candidate.category_slug, @@ -916,9 +956,12 @@ REQUIRED JSON SCHEMA: return { ...candidate, - ram_gb: candidate.ram_gb ?? aiResult.ram_gb ?? null, - screen_size_inch: aiResult.screen_size_inch ?? candidate.screen_size_inch, - battery_mah: aiResult.battery_mah ?? candidate.battery_mah, + ram_gb: validCandidateRam ?? toBoundedIntOrUndefined(aiResult.ram_gb, 2048) ?? null, + storage_gb: + validCandidateStorage ?? toBoundedIntOrUndefined(aiResult.storage_gb, 65536) ?? null, + screen_size_inch: + toBoundedNumberOrUndefined(aiResult.screen_size_inch, 120) ?? validCandidateScreen, + battery_mah: toBoundedIntOrUndefined(aiResult.battery_mah, 500000) ?? validCandidateBattery, has_5g: aiResult.has_5g ?? candidate.has_5g, specifications: mergedSpecifications, key_specs: { diff --git a/apps/importer/src/shopee/arena-spec-syncer.ts b/apps/importer/src/shopee/arena-spec-syncer.ts index 1c0134252..89c8cec95 100644 --- a/apps/importer/src/shopee/arena-spec-syncer.ts +++ b/apps/importer/src/shopee/arena-spec-syncer.ts @@ -83,6 +83,33 @@ function toBoundedNumber( return val; } +function normalizeArenaUrl( + rawUrl: string | undefined, + hostPattern: RegExp, + baseOrigin?: string +): string | undefined { + const trimmed = rawUrl?.trim(); + if (!trimmed || trimmed.length > 2048 || /\s/.test(trimmed)) { + return undefined; + } + for (let i = 0; i < trimmed.length; i++) { + const code = trimmed.charCodeAt(i); + if (code <= 0x1f || code === 0x7f) return undefined; + } + try { + const parsed = baseOrigin ? new URL(trimmed, baseOrigin) : new URL(trimmed); + if ( + (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') || + !hostPattern.test(parsed.hostname) + ) { + return undefined; + } + return parsed.href; + } catch { + return undefined; + } +} + function stripHtmlTags(raw: string): string { return decodeArenaHtmlEntities(raw.replace(//gi, '\n').replace(/<[^<>]+>/g, ' ')) .replace(/[ \t]+/g, ' ') @@ -189,12 +216,16 @@ export function parseGsmArenaHtmlToSpecs( // Hardware & memory parsing (`256GB 12GB RAM, 512GB 16GB RAM`) const memRaw = d['internalmemory'] || `${pa['internal storage'] || ''} ${pa['ram'] || ''}`.trim(); const ramMatches = Array.from(memRaw.matchAll(/(\d+(?:\.\d+)?)\s*GB\s*RAM/gi)) - .map(m => toBoundedNumber(Number(m[1]), 0.125, 2048)) + .map(m => { + const val = Number(m[1]); + return Number.isInteger(val) ? toBoundedNumber(val, 1, 2048, true) : undefined; + }) .filter((v): v is number => v !== undefined); const storageMatches = Array.from(memRaw.matchAll(/(\d+(?:\.\d+)?)\s*(GB|TB)(?!\s*RAM)/gi)) - .map(m => - toBoundedNumber(m[2]?.toUpperCase() === 'TB' ? Number(m[1]) * 1024 : Number(m[1]), 0.5, 65536) - ) + .map(m => { + const val = m[2]?.toUpperCase() === 'TB' ? Number(m[1]) * 1024 : Number(m[1]); + return Number.isInteger(val) ? toBoundedNumber(val, 1, 65536, true) : undefined; + }) .filter((v): v is number => v !== undefined); const ramGb = ramMatches.length > 0 ? Math.max(...ramMatches) : null; @@ -343,7 +374,7 @@ export function parseGsmArenaHtmlToSpecs( chip: d['chipset'] || pa['system chip'] || undefined, ram: ramGb ? `${ramGb}GB` : undefined, storage: storageGb - ? storageGb >= 1024 + ? storageGb >= 1024 && storageGb % 512 === 0 ? `${storageGb / 1024}TB` : `${storageGb}GB` : undefined, @@ -358,10 +389,13 @@ export function parseGsmArenaHtmlToSpecs( : batRaw || undefined, }); + const gsmarenaUrl = normalizeArenaUrl(meta?.gsmarenaUrl, /(?:^|\.)gsmarena\.com$/i); + const phonearenaUrl = normalizeArenaUrl(meta?.phonearenaUrl, /(?:^|\.)phonearena\.com$/i); + return { - source: meta?.phonearenaUrl || meta?.phonearenaHtml ? 'hybrid' : 'gsmarena', - gsmarenaUrl: meta?.gsmarenaUrl, - phonearenaUrl: meta?.phonearenaUrl, + source: phonearenaUrl || meta?.phonearenaHtml ? 'hybrid' : 'gsmarena', + gsmarenaUrl, + phonearenaUrl, brand, model, modelName, @@ -431,25 +465,23 @@ export function parseGsmArenaBrandCatalogHtml( const nameMatch = inner.match(/]*>((?:(?!<\/?span\b)[\s\S])*)<\/span>/i); const rawName = stripHtmlTags(nameMatch ? nameMatch[1] || '' : inner); if (!rawName || !href) continue; - let url = ''; - try { - const parsed = new URL(href, 'https://www.gsmarena.com/'); - if ( - (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') || - !/(?:^|\.)gsmarena\.com$/i.test(parsed.hostname) - ) { - continue; - } - url = parsed.href; - } catch { - continue; - } + const url = normalizeArenaUrl( + decodeArenaHtmlEntities(href), + /(?:^|\.)gsmarena\.com$/i, + 'https://www.gsmarena.com/' + ); + if (!url) continue; const fullName = brandPrefixRe && !brandPrefixRe.test(rawName) ? `${trimmedBrand} ${rawName}` : rawName; - const imgSrc = [...inner.matchAll(/]*)>/gi)] + const rawImgSrc = [...inner.matchAll(/]*)>/gi)] // `(?:^|\s)` so `data-src` lazy-load placeholders are not taken for src. .map(img => /(?:^|\s)src=["']([^"']+)["']/i.exec(img[1] || '')?.[1]) .find(Boolean); + const imgSrc = normalizeArenaUrl( + rawImgSrc ? decodeArenaHtmlEntities(rawImgSrc) : undefined, + /(?:^|\.)gsmarena\.com$/i, + 'https://www.gsmarena.com/' + ); results.push({ source: 'gsmarena', name: fullName, @@ -478,7 +510,12 @@ export function parsePhoneArenaCatalogHtml(html: string): DiscoveredArenaCatalog const text = stripHtmlTags(match[2] || ''); if (!rawHref || !text || text.length < 4) continue; if (/compare|benchmarks|size-compare|reviews/i.test(rawHref)) continue; - const url = rawHref.startsWith('http') ? rawHref : `https://www.phonearena.com${rawHref}`; + const url = normalizeArenaUrl( + decodeArenaHtmlEntities(rawHref), + /(?:^|\.)phonearena\.com$/i, + 'https://www.phonearena.com/' + ); + if (!url || !new URL(url).pathname.toLowerCase().startsWith('/phones/')) continue; if (seen.has(url)) continue; seen.add(url); results.push({ @@ -701,10 +738,7 @@ export const UPSTREAM_2026_ARENA_REFERENCE_URLS: ReadonlyArray<{ name: 'Apple iPad Pro M5 11 inch 256GB', announced: '2025, October 30', gsmarenaUrl: 'https://www.gsmarena.com/apple_ipad_pro_11_(2025)-14102.php', - phonearenaUrl: 'https://www.phonearena.com/ tablets/Apple-iPad-Pro-11-inch-M5_id12750'.replace( - /\s+/g, - '' - ), + phonearenaUrl: 'https://www.phonearena.com/tablets/Apple-iPad-Pro-11-inch-M5_id12750', }, { category_slug: 'tablets', diff --git a/apps/importer/src/shopee/entity-matcher.ts b/apps/importer/src/shopee/entity-matcher.ts index f0c053bca..1103734a7 100644 --- a/apps/importer/src/shopee/entity-matcher.ts +++ b/apps/importer/src/shopee/entity-matcher.ts @@ -23,6 +23,57 @@ export const ENTITY_MATCH_CANDIDATE_FLOOR = 0.65; /** Minimum pg_trgm similarity required to auto-merge into an existing Master Product */ export const ENTITY_MATCH_ACCEPT_THRESHOLD = 0.7; +const MAX_RAM_GB = 2048; +const MAX_STORAGE_GB = 65536; +const MAX_SCREEN_SIZE_INCH = 120; +const MAX_BATTERY_MAH = 500_000; + +function hasControlOrSpaceChar(val: string): boolean { + for (let i = 0; i < val.length; i++) { + const code = val.charCodeAt(i); + if (code <= 0x20 || code === 0x7f) return true; + } + return false; +} + +export function sanitizeHttpUrl(raw: unknown): string | null { + if (typeof raw !== 'string') return null; + const trimmed = raw.trim(); + if (!trimmed || trimmed.length > 2048 || hasControlOrSpaceChar(trimmed)) return null; + try { + const parsed = new URL(trimmed); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null; + if (!parsed.hostname) return null; + return trimmed; + } catch { + return null; + } +} + +export function sanitizeHttpUrlList(urls: unknown): string[] { + if (!Array.isArray(urls)) return []; + const seen = new Set(); + const out: string[] = []; + for (const item of urls) { + const valid = sanitizeHttpUrl(item); + if (valid && !seen.has(valid)) { + seen.add(valid); + out.push(valid); + } + } + return out; +} + +function toBoundedPositiveInt(value: unknown, max: number): number | null { + const n = Number(value); + return Number.isInteger(n) && n > 0 && n <= max ? n : null; +} + +function toBoundedPositiveNumber(value: unknown, max: number): number | null { + const n = Number(value); + return Number.isFinite(n) && n > 0 && n <= max ? n : null; +} + export class EntityMatcher { private enricher: AiSpecEnricher; @@ -81,27 +132,41 @@ export class EntityMatcher { throw new Error(`[EntityMatcher] Tier 2 fuzzy match failed: ${fuzzyErr.message}`); } let ambiguousFuzzy = false; - if (Array.isArray(fuzzyMatches) && fuzzyMatches.length > 0) { - const topMatch = fuzzyMatches[0]; - if (topMatch.similarity >= ENTITY_MATCH_ACCEPT_THRESHOLD) { + const validFuzzyMatches = Array.isArray(fuzzyMatches) + ? fuzzyMatches.filter(m => { + const masterId = typeof m?.master_id === 'string' ? m.master_id.trim() : ''; + const sim = Number(m?.similarity); + return ( + masterId.length > 0 && + Number.isFinite(sim) && + sim >= ENTITY_MATCH_CANDIDATE_FLOOR && + sim <= 1 + ); + }) + : []; + if (validFuzzyMatches.length > 0) { + const topMatch = validFuzzyMatches[0]!; + const topMasterId = String(topMatch.master_id).trim(); + const topSim = Number(topMatch.similarity); + if (topSim >= ENTITY_MATCH_ACCEPT_THRESHOLD) { logger.info( 'entity_match.tier2_fuzzy', - `Tier 2 Fuzzy Match found: "${candidate.name}" ~= "${topMatch.name}" (similarity: ${(topMatch.similarity * 100).toFixed(1)}%) -> ID: ${topMatch.master_id}`, + `Tier 2 Fuzzy Match found: "${candidate.name}" ~= "${topMatch.name}" (similarity: ${(topSim * 100).toFixed(1)}%) -> ID: ${topMasterId}`, { - masterId: topMatch.master_id, - similarity: topMatch.similarity, + masterId: topMasterId, + similarity: topSim, candidateName: candidate.name, } ); - return { masterId: topMatch.master_id, isNew: false }; + return { masterId: topMasterId, isNew: false }; } // Candidates exist but all below the accept threshold: identity is // uncertain -> seed as draft (plan v7 §5.1), never auto-publish. ambiguousFuzzy = true; logger.warn( 'entity_match.tier2_ambiguous', - `Tier 2 best similarity ${(topMatch.similarity * 100).toFixed(1)}% < ${(ENTITY_MATCH_ACCEPT_THRESHOLD * 100).toFixed(0)}%: seeding as draft if new.`, - { similarity: topMatch.similarity, modelKey: candidate.model_key } + `Tier 2 best similarity ${(topSim * 100).toFixed(1)}% < ${(ENTITY_MATCH_ACCEPT_THRESHOLD * 100).toFixed(0)}%: seeding as draft if new.`, + { similarity: topSim, modelKey: candidate.model_key } ); } @@ -134,6 +199,7 @@ export class EntityMatcher { // Insert-only with ignoreDuplicates: the LOSING worker must refetch the // winner WITHOUT overwriting its canonical metadata or status (Codex // blocker: a re-asserting upsert could re-publish a draft/archived master). + const safeImages = sanitizeHttpUrlList(candidate.images); const baseInsertRow = { category_id: categoryId || null, brand: candidate.brand, @@ -143,15 +209,15 @@ export class EntityMatcher { name: candidate.name, slug: candidate.slug, description: candidate.description || '', - ram_gb: candidate.ram_gb, - storage_gb: candidate.storage_gb, - screen_size_inch: candidate.screen_size_inch, - battery_mah: candidate.battery_mah, + ram_gb: toBoundedPositiveInt(candidate.ram_gb, MAX_RAM_GB), + storage_gb: toBoundedPositiveInt(candidate.storage_gb, MAX_STORAGE_GB), + screen_size_inch: toBoundedPositiveNumber(candidate.screen_size_inch, MAX_SCREEN_SIZE_INCH), + battery_mah: toBoundedPositiveInt(candidate.battery_mah, MAX_BATTERY_MAH), has_5g: candidate.has_5g, specifications: candidate.specifications, key_specs: candidate.key_specs, - images: candidate.images, - thumbnail: candidate.images[0] || null, + images: safeImages, + thumbnail: safeImages[0] || null, colors: candidate.colors, status: seedStatus, }; @@ -249,11 +315,16 @@ export class EntityMatcher { existingCategorySlug ?? detectCategorySlug(`${row.brand || ''} ${row.model || ''} ${row.name || ''}`); + const existingRamGb = toBoundedPositiveInt(row.ram_gb, MAX_RAM_GB); + const existingStorageGb = toBoundedPositiveInt(row.storage_gb, MAX_STORAGE_GB); + const existingScreenInch = toBoundedPositiveNumber(row.screen_size_inch, MAX_SCREEN_SIZE_INCH); + const existingBatteryMah = toBoundedPositiveInt(row.battery_mah, MAX_BATTERY_MAH); + const aiResult = await this.enricher.fetchSpecsFromAi({ brand: row.brand, model: row.model, - storageGb: row.storage_gb, - ramGb: row.ram_gb, + storageGb: existingStorageGb, + ramGb: existingRamGb, rawTitle: row.name, rawDescription: row.description, categorySlug: inferredCategorySlug, @@ -289,10 +360,13 @@ export class EntityMatcher { const { error: updateErr } = await this.supabase .from('master_products') .update({ - ram_gb: row.ram_gb ?? aiResult.ram_gb, - storage_gb: row.storage_gb ?? aiResult.storage_gb, - screen_size_inch: row.screen_size_inch ?? aiResult.screen_size_inch, - battery_mah: row.battery_mah ?? aiResult.battery_mah, + ram_gb: existingRamGb ?? toBoundedPositiveInt(aiResult.ram_gb, MAX_RAM_GB), + storage_gb: existingStorageGb ?? toBoundedPositiveInt(aiResult.storage_gb, MAX_STORAGE_GB), + screen_size_inch: + existingScreenInch ?? + toBoundedPositiveNumber(aiResult.screen_size_inch, MAX_SCREEN_SIZE_INCH), + battery_mah: + existingBatteryMah ?? toBoundedPositiveInt(aiResult.battery_mah, MAX_BATTERY_MAH), has_5g: row.has_5g ? true : (aiResult.has_5g ?? row.has_5g), specifications: mergedSpecifications, key_specs: mergedKeySpecs, diff --git a/apps/importer/src/shopee/product-normalizer.ts b/apps/importer/src/shopee/product-normalizer.ts index 853f67853..84cb012a4 100644 --- a/apps/importer/src/shopee/product-normalizer.ts +++ b/apps/importer/src/shopee/product-normalizer.ts @@ -111,13 +111,22 @@ export function detectProductOrigin(title: string, description?: string): Produc return 'unknown'; } +/** Apple product families, for a title whose line no Apple pattern matched (DOS/Rate#817). */ +const APPLE_FAMILIES: ReadonlyArray = [ + [/iphone/i, 'iPhone'], + [/ipad/i, 'iPad'], + [/macbook/i, 'MacBook'], + [/\bwatch\b/i, 'Watch'], + [/airpods/i, 'AirPods'], +]; + /** * Detect canonical category slug from product title across all 5 tech categories. */ export function detectCategorySlug(title: string): CanonicalCategorySlug { const t = title.toLowerCase(); if ( - /airpods|galaxy\s*buds|wf-\d+|wh-\d+|tai\s*nghe|earbuds?|freebuds|soundcore|quietcomfort|momentum\s*true/i.test( + /airpods|galaxy\s*buds|w[fh]1000xm|wf-\d+|wh-\d+|tai\s*nghe|earbuds?|freebuds|soundcore|quietcomfort|momentum\s*true/i.test( t ) ) { @@ -138,7 +147,7 @@ export function detectCategorySlug(title: string): CanonicalCategorySlug { return 'laptops'; } if ( - /\bipad\b|galaxy\s*tab|xiaomi\s*pad|redmi\s*pad|matepad|lenovo\s*tab|surface\s*pro|máy\s*tính\s*bảng|\btablet\b/i.test( + /\bipad\b|galaxy\s*tab|xiaomi\s*pad|redmi\s*pad|\bmi\s*pad\b|matepad|lenovo\s*tab|surface\s*pro|máy\s*tính\s*bảng|\btablet\b/i.test( t ) ) { @@ -154,9 +163,9 @@ export function detectCategorySlug(title: string): CanonicalCategorySlug { const CANONICAL_PHONE_REGISTRY: Record< string, { - screen_size_inch: number; + screen_size_inch?: number; battery_mah: number; - ram_gb: number; + ram_gb?: number; chipset: string; cpu: string; gpu: string; @@ -165,7 +174,7 @@ const CANONICAL_PHONE_REGISTRY: Record< dimensions: string; resolution: string; screen_type: string; - refresh_rate_hz: number; + refresh_rate_hz?: number; rear_camera: string; second_camera?: string; third_camera?: string; @@ -506,9 +515,7 @@ const CANONICAL_PHONE_REGISTRY: Record< default_storage_gb: 64, }, 'apple:airpods-pro-2': { - screen_size_inch: 1.0, battery_mah: 523, - ram_gb: 1, chipset: 'Apple H2 headphone chip + Apple U1 chip (MagSafe Case)', cpu: 'Apple H2 custom acoustic DSP', gpu: 'N/A', @@ -517,19 +524,15 @@ const CANONICAL_PHONE_REGISTRY: Record< dimensions: '45.2 x 60.6 x 21.7 mm (Case)', resolution: 'Custom high-excursion Apple driver', screen_type: 'Active Noise Cancellation (2x ANC) + Adaptive Transparency', - refresh_rate_hz: 60, rear_camera: 'Inward-facing microphone + skin-detect sensor', front_camera: 'Beamforming dual microphones', charging_w: 5, has_5g: false, release_date: 'September 2023', ip_rating: 'IP54 dust, sweat, and water resistant', - default_storage_gb: 1, }, 'sony:wf-1000xm5': { - screen_size_inch: 1.0, battery_mah: 500, - ram_gb: 1, chipset: 'Sony Integrated Processor V2 + HD Noise Cancelling Processor QN2e', cpu: 'Dual-processor 24-bit Audio DSP', gpu: 'N/A', @@ -538,7 +541,6 @@ const CANONICAL_PHONE_REGISTRY: Record< dimensions: '64.6 x 40.0 x 26.5 mm (Case)', resolution: '8.4 mm Dynamic Driver X (20Hz-40,000Hz LDAC)', screen_type: 'Industry-leading Dual-Processor ANC + Bone Conduction Sensor', - refresh_rate_hz: 60, rear_camera: 'Dual feedback microphones per earbud', second_camera: 'Feed-forward & feedback 6-mic array', front_camera: 'Bone conduction sensor + Wind noise reduction', @@ -546,7 +548,6 @@ const CANONICAL_PHONE_REGISTRY: Record< has_5g: false, release_date: 'July 2023', ip_rating: 'IPX4 water resistant', - default_storage_gb: 1, }, 'samsung:galaxy-tab-s10-ultra': { screen_size_inch: 14.6, @@ -681,9 +682,7 @@ const CANONICAL_PHONE_REGISTRY: Record< default_storage_gb: 8, }, 'samsung:galaxy-buds3-pro': { - screen_size_inch: 1.0, battery_mah: 515, - ram_gb: 1, chipset: 'Samsung Proprietary Bluetooth 5.4 SoC + Dual Amplifier', cpu: 'Galaxy AI Adaptive ANC & Real-Time Interpreter DSP', gpu: 'Blade Lights LED Controller', @@ -692,7 +691,6 @@ const CANONICAL_PHONE_REGISTRY: Record< dimensions: '48.7 x 58.9 x 24.4 mm (Case)', resolution: 'Enhanced 2-Way Speaker (10.5mm Dynamic + 6.1mm Planar Tweeter)', screen_type: 'Blade Lights LED Indicator + Transparent Cradle Lid', - refresh_rate_hz: 60, rear_camera: '3 High-SNR Microphones + VPU (Voice Pickup Unit)', second_camera: 'In-canal feedback mic + Siren/Voice Detect AI sensor', front_camera: 'Pinch & Swipe Blade Touch Controls', @@ -700,7 +698,6 @@ const CANONICAL_PHONE_REGISTRY: Record< has_5g: false, release_date: 'July 2024', ip_rating: 'IP57 dust and water resistant (1m up to 30 mins)', - default_storage_gb: 1, }, // === 2026 Flagship Lineup (Smartphones, Tablets, Laptops, Smartwatches, Earbuds) === 'xiaomi:18-pro-max': { @@ -986,9 +983,7 @@ const CANONICAL_PHONE_REGISTRY: Record< default_storage_gb: 64, }, 'apple:airpods-pro-3': { - screen_size_inch: 1.0, battery_mah: 540, - ram_gb: 1, chipset: 'Apple H3 Headphone Chip + U2 Ultra Wideband (MagSafe Case)', cpu: 'Apple H3 Ultra-Low-Latency Audio DSP (96kHz/24-bit Lossless)', gpu: 'Acoustic & Heart-Rate Biosensor Co-Processor', @@ -997,7 +992,6 @@ const CANONICAL_PHONE_REGISTRY: Record< dimensions: '45.2 x 60.6 x 21.7 mm (USB-C MagSafe Case)', resolution: 'Custom High-Excursion Apple Driver + High Dynamic Range Amplifier', screen_type: 'Status LED + Built-in Find My Precision Speaker', - refresh_rate_hz: 60, rear_camera: 'In-Ear Optical Heart-Rate & Body Temperature Sensor + 3x ANC Pro vs Gen 1', second_camera: 'Clinical-Grade Hearing Aid Mode + Conversation Awareness + Voice Isolation', front_camera: 'Capacitive Touch Stem (Swipe Volume + Press Control)', @@ -1005,12 +999,9 @@ const CANONICAL_PHONE_REGISTRY: Record< has_5g: false, release_date: 'September 2025', ip_rating: 'IP54 dust, sweat, and water resistant (earbuds and MagSafe charging case)', - default_storage_gb: 1, }, 'sony:wf-1000xm6': { - screen_size_inch: 1.0, battery_mah: 550, - ram_gb: 1, chipset: 'Sony Integrated Processor V3 + QN3e HD Noise Cancelling Processor', cpu: 'Dual-Processor 24-bit Audio Signal Processing + DSEE Ultimate AI', gpu: 'LE Audio LC3 & LDAC High-Bitrate Controller', @@ -1019,7 +1010,6 @@ const CANONICAL_PHONE_REGISTRY: Record< dimensions: '64.0 x 40.0 x 25.8 mm (Qi Wireless Case)', resolution: '8.4mm Dynamic Driver X (Dual-layer dome-edge diaphragm)', screen_type: 'Recessed Matte Acoustic Chassis + Multi-Point Status LED', - refresh_rate_hz: 60, rear_camera: 'Dual Feedback + Feedforward 6-Mic Array + Bone Conduction Sensor', second_camera: 'AI Beamforming Wind Noise Reduction + Speak-to-Chat + Head Gesture Control', front_camera: 'Customizable Capacitive Touch Panels (L/R)', @@ -1027,7 +1017,6 @@ const CANONICAL_PHONE_REGISTRY: Record< has_5g: false, release_date: 'June 2025', ip_rating: 'IPX4 splash and sweat resistant + Polyurethane Noise Isolation Tips', - default_storage_gb: 1, }, }; @@ -1051,7 +1040,10 @@ function slugify(text: string): string { return ( text .toString() + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') .toLowerCase() + .replace(/đ/g, 'd') // "Pro+" / "S24+" are Plus models: keep them apart from the base model's // key and slug, and give "S24+" the same key as "S24 Plus" (DOS/Rate#597). // Only a "+" that ends a model word counts; "A55 + Tặng" and "8+256GB" keep @@ -1074,10 +1066,11 @@ export class ProductNormalizer { // 1. Extract Brand let brand = 'Khác'; - if (/apple|iphone|ipad|macbook|airpods/i.test(title)) brand = 'Apple'; + // Not the "ipad" in Xiaomi's "Mipad" (DOS/Rate#826). + if (/apple|iphone|(? line[0]!.toUpperCase() + line.slice(1).toLowerCase() + ) + // "16ProMax" and "16 ProMax" are "16 Pro Max" (DOS/Rate#815). + .replace(/^(\d+)(?=pro|plus|mini|air)/i, '$1 ') + .replace(/\b(pro)(max)\b/i, '$1 $2'); model = `iPhone ${normalizedIphoneSuffix}`; } else if (ipadMatch && ipadMatch[1]) { - const line = ipadMatch[1].trim(); + // Apple names the base iPad by its chip from the A16 on, so an A16 or later chip + // right after the screen size or generation is the model: "iPad 11 inch (A16)" and + // "iPad 11 (A16)" are the iPad (A16). Earlier ones go by generation: "iPad 10" and + // "iPad Gen 10 (A14 Bionic)" stay the 10th (DOS/Rate#817). Each run of spaces has + // one `\s*` that can match it. + const chipAfter = /^(?:gen\s*)?\d+$/i.test(ipadMatch[1]) + ? appleTitle + .slice((ipadMatch.index ?? 0) + ipadMatch[0].length) + .match(/^(?:[.,]\d+)?\s*(?:-?(?:inch\b|in\b|"|'')\s*)?(?:\(\s*)?a(1[6-9]|2\d)\b/i) + : null; + let line = chipAfter + ? `A${chipAfter[1]}` + : ipadMatch[1] + .trim() + .replace(/^gen\s*/i, '') + .replace(/^a(\d{2})$/i, 'A$1'); + // "iPad mini 6" and "iPad mini 7" are two models, not one "iPad mini"; Apple calls the + // 7th "iPad mini (A17 Pro)", and "8.3 inch" or "8,3 inch" is the screen (DOS/Rate#819). + // One digit, not a "5G" or "4G" network or a storage ("mini 64 Wifi"). + const mini = line.match(/^mini\s*([1-9])?$/i); + if (mini) + line = `mini${mini[1] ? ` ${mini[1]}` : /\ba17\s*pro\b/i.test(appleTitle) ? ' 7' : ''}`; model = /^(pro|air)$/i.test(line) && chip ? `iPad ${line} ${chip}` : `iPad ${line}`; } else if (macbookMatch && macbookMatch[1]) { - const line = macbookMatch[1].trim(); + const line = macbookMatch[1].trim().replace(/^neo$/i, 'Neo'); model = /^(pro|air)$/i.test(line) && chip ? `MacBook ${line} ${chip}${chipMatch?.[2] ? ` ${chipMatch[2]}` : ''}` @@ -1147,7 +1181,11 @@ export class ProductNormalizer { model = `Watch ${watchMatch[1].trim()}`; } else if (airpodsMatch) { if (airpodsMatch[1]) { - model = `AirPods Pro${airpodsMatch[2] ? ` ${airpodsMatch[2]}` : ''}`; + const rawGen = airpodsMatch[2]; + // FPT Shop names 2nd-gen AirPods Pro by release/USB-C refresh year ("AirPods Pro 2022", + // "AirPods Pro 2023") (DOS/Rate#643). + const proGen = rawGen === '2022' || rawGen === '2023' ? '2' : rawGen; + model = `AirPods Pro${proGen ? ` ${proGen}` : ''}`; } else if (airpodsMatch[3]) { model = 'AirPods Max'; } else if (airpodsMatch[4]) { @@ -1156,14 +1194,19 @@ export class ProductNormalizer { model = 'AirPods'; } } else { + // No line matched: keep the family the title names first rather than call it an + // iPhone ("MacBook M4", "Apple \"iPad\" 256GB") (DOS/Rate#817). + let first = Infinity; model = 'iPhone'; + for (const [family, name] of APPLE_FAMILIES) { + const at = title.search(family); + if (at >= 0 && at < first) [first, model] = [at, name]; + } } - } else if (brand === 'Sony' && /wf-1000xm(\d+)/i.test(title)) { - const m = title.match(/wf-1000xm(\d+)/i); - model = `WF-1000XM${m?.[1] || '5'}`; - } else if (brand === 'Sony' && /wh-1000xm(\d+)/i.test(title)) { - const m = title.match(/wh-1000xm(\d+)/i); - model = `WH-1000XM${m?.[1] || '5'}`; + } else if (brand === 'Sony' && /w([fh])-?1000xm(\d+)/i.test(title)) { + // "WF1000XM5" without the hyphen is the same model (DOS/Rate#815). + const m = title.match(/w([fh])-?1000xm(\d+)/i); + model = `W${m?.[1]?.toUpperCase()}-1000XM${m?.[2]}`; } else if (brand === 'Asus' && /zenbook\s*s\s*14/i.test(title)) { model = 'Zenbook S 14'; } else if (brand === 'Lenovo' && /thinkpad\s*x1\s*carbon\s*gen\s*12/i.test(title)) { @@ -1195,6 +1238,8 @@ export class ProductNormalizer { .replace(/^(?:\s*(?:\[[^\]]*\]|\([^)]*\)|【[^】]*】))+\s*/, '') // "Vivo Book 15" is the Asus Vivobook line: key it like "Vivobook 15" (DOS/Rate#774). .replace(/\bvivo\s+book\b/gi, 'Vivobook') + // "17 ProMax" keys like "17 Pro Max" (DOS/Rate#815). + .replace(/\b(pro)(max)\b/gi, '$1 $2') .trim(); const rawTokens = cleaned.split(/\s+/).filter(Boolean); const modelTokens: string[] = []; @@ -1214,10 +1259,12 @@ export class ProductNormalizer { startsGeneration || (/^gen$/i.test(modelTokens[modelTokens.length - 1] ?? '') && /^\d{1,2}[,;]?$/.test(tok)); // A trailing "Plus" word is part of the model even past the 4-token cap, - // so "Redmi Note 13 Pro Plus" keys like "Redmi Note 13 Pro+" (#597). + // so "Redmi Note 13 Pro Plus" keys like "Redmi Note 13 Pro+" (#597); so is + // "Max" after "Pro": "Redmi Note 17 Pro Max" is not the Pro (DOS/Rate#815). if ( modelTokens.length >= 4 && !(modelTokens.length === 4 && /^plus[,;]?$/i.test(tok)) && + !(/^pro$/i.test(modelTokens[modelTokens.length - 1] ?? '') && /^max[,;]?$/i.test(tok)) && !generation ) { break; @@ -1225,7 +1272,9 @@ export class ProductNormalizer { if ( /^(?:\d+(?:[.,]\d+)?(?:GB|TB)|\d+\/\d+(?:GB|TB)?|\d{2}mm|5G|4G|LTE|GPS|RAM|ROM|OLED|WiFi|Chính|Quốc|Bản|Mới|Fullbox|Likenew|Like|Cũ|99%|98%|VN\/A|VNA|LL\/A|\(|\[|-)/i.test( tok - ) + ) || + // A size written apart, "256 GB", ends the model like "256GB" (DOS/Rate#815). + (/^\d+(?:[.,]\d+)?$/.test(tok) && /^(?:GB|TB)\b/i.test(rawTokens[i + 1] ?? '')) ) { break; } @@ -1282,6 +1331,13 @@ export class ProductNormalizer { // `\s*(?:X\s*)?` rather than `\s*X?\s*` (here and in the ROM patterns // below): the same strings, without the two adjacent \s* that made long // whitespace runs quadratic (CodeQL polynomial regex, DOS/Rate#537). + const romExplicit = title.match( + /(?:ROM|bộ nhớ trong)\s*(?:[:-]\s*)?(\d+(?:[.,]\d+)?)\s*(TB|GB)/i + ); + const romLoose = title.match( + /dung lượng\s*(?:[:-]\s*)?(\d+(?:[.,]\d+)?)\s*(TB|GB)(?!\s*(?:RAM|bộ\s*nhớ\s*RAM|dung\s*lượng\s*RAM))/i + ); + const slashRamRom = title.match(/\b(\d{1,2})\s*(?:GB\s*)?\/\s*(\d+(?:[.,]\d+)?)\s*(TB|GB)\b/i); if (slashRamRom && ramPhrases.length === 0) { const slashRam = parseInt(slashRamRom[1], 10); @@ -1297,19 +1353,29 @@ export class ProductNormalizer { if (first.value <= 32 && second.value >= 64 && /^[\s/,-]+$/.test(between)) { first.claimed = true; } + } else if ( + ramPhrases.length === 0 && + !romExplicit && + !romLoose && + claimable.filter(t => !t.claimed).length === 1 + ) { + const [only] = claimable.filter(t => !t.claimed); + // Retailers such as FPT Shop name iPads/laptops by their RAM ("iPad Pro 11 M5 2025 WiFi 12GB"): + // tablets start at >=32GB storage and laptops at >=64GB, and non-power-of-two GB values + // (6/12/18/24/36/48GB) are always RAM rather than storage (DOS/Rate#643). + const isTabletOrLaptopRamOnly = + (categorySlug === 'tablets' && only.value <= 24) || + (categorySlug === 'laptops' && only.value <= 48); + const isNonPowerOfTwoRamSize = [3, 6, 12, 18, 24, 36, 48].includes(only.value); + if (isTabletOrLaptopRamOnly || isNonPowerOfTwoRamSize) { + only.claimed = true; + } } const storageCandidates = claimable.filter( t => !t.claimed && t.value > 0 && t.value <= MAX_VALID_STORAGE_GB ); - const romExplicit = title.match( - /(?:ROM|bộ nhớ trong)\s*(?:[:-]\s*)?(\d+(?:[.,]\d+)?)\s*(TB|GB)/i - ); - const romLoose = title.match( - /dung lượng\s*(?:[:-]\s*)?(\d+(?:[.,]\d+)?)\s*(TB|GB)(?!\s*(?:RAM|bộ\s*nhớ\s*RAM|dung\s*lượng\s*RAM))/i - ); - const baseKey = `${slugify(brand)}:${slugify(model)}`; const benchmark = CANONICAL_PHONE_REGISTRY[baseKey]; @@ -1319,7 +1385,11 @@ export class ProductNormalizer { let storageGb: number | null = null; let storageConfident = false; const marker = romExplicit ?? romLoose; - if (marker) { + if (categorySlug === 'earbuds') { + // Earbuds have no storage variant and must key as :unknown with null storage (DOS/Rate#643, #657). + storageGb = null; + storageConfident = true; + } else if (marker) { const candidateStorage = toValidStorageGb( parseFloat(marker[1].replace(',', '.')) * (marker[2].toUpperCase() === 'TB' ? 1024 : 1) ); @@ -1342,9 +1412,6 @@ export class ProductNormalizer { } else if (storageCandidates.length === 0 && benchmark?.default_storage_gb != null) { storageGb = benchmark.default_storage_gb; storageConfident = true; - } else if (storageCandidates.length === 0 && categorySlug === 'earbuds') { - storageGb = 1; - storageConfident = true; } const storageKnown = storageGb !== null; @@ -1373,7 +1440,7 @@ export class ProductNormalizer { // (':unknown') instead of guessing a default (plan v7 §5.2). const modelKey = storageKnown ? `${baseKey}:${storageGb}gb` : `${baseKey}:unknown`; const storageDisplay = - categorySlug === 'earbuds' && storageGb === 1 + categorySlug === 'earbuds' ? 'Standard' : storageGb === null ? 'Không rõ dung lượng' @@ -1381,7 +1448,7 @@ export class ProductNormalizer { ? `${storageGb / 1024}TB` : `${storageGb}GB`; const canonicalName = - categorySlug === 'earbuds' && storageGb === 1 + categorySlug === 'earbuds' ? `${brand} ${model}` : storageKnown ? `${brand} ${model} ${storageDisplay}` @@ -1390,6 +1457,7 @@ export class ProductNormalizer { // 6. Enrich with 10 Field Groups (PhoneArena structure + GSMArena depth) const isApple = brand === 'Apple'; + const isSmartphone = categorySlug === 'smartphones'; const titleScreenMatch = categorySlug === 'earbuds' ? null : title.match(/\b(\d{1,2}(?:[.,]\d)?)\s*(?:inch|in\b|")/i); @@ -1405,15 +1473,21 @@ export class ProductNormalizer { : null; const claimedRamValue = claimable.find(t => t.claimed)?.value ?? null; - const ramGb = benchmark - ? benchmark.ram_gb - : claimedRamValue !== null && - Number.isFinite(claimedRamValue) && - claimedRamValue > 0 && - claimedRamValue <= MAX_VALID_RAM_GB + const validClaimedRam = + claimedRamValue !== null && + Number.isFinite(claimedRamValue) && + claimedRamValue > 0 && + claimedRamValue <= MAX_VALID_RAM_GB ? claimedRamValue : null; - const screenSize = benchmark ? benchmark.screen_size_inch : titleScreenSize; + // Title-stated RAM takes precedence over benchmark RAM; earbuds and smartwatches + // have no GB RAM spec (DOS/Rate#657). + const ramGb = + categorySlug === 'earbuds' || categorySlug === 'smartwatches' + ? null + : (validClaimedRam ?? benchmark?.ram_gb ?? null); + const screenSize = + categorySlug === 'earbuds' ? null : (benchmark?.screen_size_inch ?? titleScreenSize); const batteryMah = benchmark ? benchmark.battery_mah : null; const has5g = benchmark ? benchmark.has_5g @@ -1593,24 +1667,46 @@ export class ProductNormalizer { design: { dimensions: benchmark?.dimensions || undefined, weight_g: benchmark?.weight_g || undefined, - materials: isApple - ? 'Ceramic Shield front, glass back, metal frame' - : 'Glass front/back, metal frame', - resistance: benchmark?.ip_rating || 'IP68 dust/water resistant', - biometrics: isApple ? 'Face ID' : 'Fingerprint (under display), Face unlock', - keys: isApple ? 'Action Button, Camera Control, Volume, Power' : 'Volume, Power', + materials: isSmartphone + ? isApple + ? 'Ceramic Shield front, glass back, metal frame' + : 'Glass front/back, metal frame' + : undefined, + resistance: + benchmark?.ip_rating || (isSmartphone ? 'IP68 dust/water resistant' : undefined), + biometrics: isSmartphone + ? isApple + ? /^iPhone SE\b/i.test(model) + ? 'Touch ID' + : 'Face ID' + : 'Fingerprint (under display), Face unlock' + : undefined, + keys: isSmartphone + ? isApple + ? 'Action Button, Camera Control, Volume, Power' + : 'Volume, Power' + : undefined, colors: colors.length > 0 ? colors : ['Tiêu chuẩn'], }, display: { - size_inch: screenSize || undefined, + size_inch: categorySlug === 'earbuds' ? undefined : screenSize || undefined, type: benchmark?.screen_type || fallbackScreenType, resolution: benchmark?.resolution || undefined, - refresh_rate_hz: benchmark?.refresh_rate_hz || (isApple ? 60 : 120), - ppi: benchmark?.screen_size_inch === 6.8 ? 505 : 460, - protection: isApple ? 'Ceramic Shield' : 'Corning Gorilla Glass', - features: isApple - ? 'Dynamic Island, True Tone, Wide color (P3), Haptic Touch' - : 'Always-on display, HDR10+, Eye Comfort Shield', + refresh_rate_hz: + categorySlug === 'earbuds' + ? undefined + : benchmark?.refresh_rate_hz || (isSmartphone ? (isApple ? 60 : 120) : undefined), + ppi: isSmartphone ? (benchmark?.screen_size_inch === 6.8 ? 505 : 460) : undefined, + protection: isSmartphone + ? isApple + ? 'Ceramic Shield' + : 'Corning Gorilla Glass' + : undefined, + features: isSmartphone + ? isApple + ? 'Dynamic Island, True Tone, Wide color (P3), Haptic Touch' + : 'Always-on display, HDR10+, Eye Comfort Shield' + : undefined, }, hardware: { chipset: benchmark?.chipset || undefined, @@ -1642,9 +1738,11 @@ export class ProductNormalizer { bluetooth: '5.3, A2DP, LE', wlan: 'Wi-Fi 6E/7, dual-band, Wi-Fi Direct', usb: 'USB Type-C, OTG', - sensors: isApple - ? 'Face ID, gia tốc kế, con quay hồi chuyển, tiệm cận, la bàn, khí áp kế' - : 'Vân tay dưới màn hình, gia tốc kế, con quay hồi chuyển, tiệm cận, la bàn', + sensors: isSmartphone + ? isApple + ? 'Face ID, gia tốc kế, con quay hồi chuyển, tiệm cận, la bàn, khí áp kế' + : 'Vân tay dưới màn hình, gia tốc kế, con quay hồi chuyển, tiệm cận, la bàn' + : undefined, location: 'GPS, GLONASS, Galileo, QZSS, BeiDou', nfc: true, other: isApple ? 'Ultra Wideband (UWB), SOS vệ tinh' : 'Hỗ trợ OTG, Chia sẻ nhanh', @@ -1677,15 +1775,15 @@ export class ProductNormalizer { }, regulatory: { certifications: isApple ? 'CE, FCC ID, Apple Eco Standards' : 'CE, FCC, RoHS', - sar_head: '1.10 W/kg', - sar_body: '1.15 W/kg', + sar_head: isSmartphone ? '1.10 W/kg' : undefined, + sar_body: isSmartphone ? '1.15 W/kg' : undefined, }, }; const keySpecs: KeySpecs = { screen: - categorySlug === 'earbuds' && !benchmark - ? 'TWS In-Ear + Charging Case' + categorySlug === 'earbuds' + ? benchmark?.screen_type?.split(',')[0] || 'TWS In-Ear + Charging Case' : `${fallbackScreenSize}" ${benchmark?.screen_type?.split(',')[0] || fallbackScreenType.split(',')[0]}`, chip: benchmark?.chipset?.split('(')[0]?.trim() || fallbackChip, camera: benchmark?.rear_camera ? benchmark.rear_camera.split(',')[0] : fallbackKeyCamera, diff --git a/apps/importer/src/shopee/shopee-to-supabase.ts b/apps/importer/src/shopee/shopee-to-supabase.ts index a131a9d4d..02b20602e 100644 --- a/apps/importer/src/shopee/shopee-to-supabase.ts +++ b/apps/importer/src/shopee/shopee-to-supabase.ts @@ -17,7 +17,7 @@ import { type ValidatedScrapedProduct, } from './types.js'; import { ProductNormalizer } from './product-normalizer.js'; -import { EntityMatcher } from './entity-matcher.js'; +import { EntityMatcher, sanitizeHttpUrl, sanitizeHttpUrlList } from './entity-matcher.js'; import { AiSpecEnricher, type AiSpecEnricherConfig } from './ai-spec-enricher.js'; import { AiReviewAnalyzer, type ReviewInputItem } from './ai-review-analyzer.js'; import { createImporterLogger } from './logger.js'; @@ -116,11 +116,26 @@ export function normalizeScrapedProductPrices( return Number.isFinite(p) && p > 0 && p <= MAX_VALID_VND_PRICE; }) : []; + const images = sanitizeHttpUrlList(product.images); + const shopUrl = sanitizeHttpUrl(product.shop?.url) ?? undefined; + const reviews = Array.isArray(product.reviews) + ? product.reviews.map(r => ({ + ...r, + authorAvatar: sanitizeHttpUrl(r.authorAvatar) ?? undefined, + images: sanitizeHttpUrlList(r.images), + })) + : []; return { ...product, priceMax, priceBeforeDiscount: validBeforeDiscount, + images, models, + shop: { + ...product.shop, + url: shopUrl, + }, + reviews, }; } @@ -278,6 +293,15 @@ export class ShopeeIngestionPipeline { platform: SupportedEcommercePlatform; }> { const product = normalizeScrapedProductPrices(ScrapedProductSchema.parse(rawProduct)); + if ( + !Number.isFinite(product.priceMin) || + product.priceMin <= 0 || + product.priceMin > MAX_VALID_VND_PRICE + ) { + throw new Error( + `Cannot ingest unpriced or out-of-bounds product listing (priceMin=${product.priceMin})` + ); + } if (this.categoryIdBySlug.size === 0) { await this.init(); } diff --git a/apps/importer/src/shopee/test-ai-enricher.ts b/apps/importer/src/shopee/test-ai-enricher.ts index bbad6f763..c3b9d9dbb 100644 --- a/apps/importer/src/shopee/test-ai-enricher.ts +++ b/apps/importer/src/shopee/test-ai-enricher.ts @@ -15,6 +15,7 @@ import { ProductSpecificationsSchema, KeySpecsSchema } from '@repo/shared-data'; import { ProductNormalizer } from './product-normalizer.js'; import { AiSpecEnricher, + AiSpecEnrichmentPayloadSchema, extractJsonObjectString, sanitizeSpecifications, } from './ai-spec-enricher.js'; @@ -486,6 +487,54 @@ async function runTests() { ); assert.equal(ProductSpecificationsSchema.safeParse(boundedAiRes?.specifications).success, true); + // DOS/Rate#825: AiSpecEnrichmentPayloadSchema integer/domain bounds, baseUrl validation, fractional integer rejection, and enrichCandidate storage_gb backfill + assert.equal( + AiSpecEnrichmentPayloadSchema.safeParse({ + ram_gb: 4096, + specifications: {}, + }).success, + false, + 'AiSpecEnrichmentPayloadSchema must reject ram_gb > 2048' + ); + assert.equal( + AiSpecEnrichmentPayloadSchema.safeParse({ + ram_gb: 12.5, + specifications: {}, + }).success, + false, + 'AiSpecEnrichmentPayloadSchema must reject non-integer ram_gb' + ); + + const fallbackUrlEnricher = new AiSpecEnricher({ + provider: 'openai', + apiKey: 'test-key', + baseUrl: 'javascript:alert(1)', + timeoutMs: -50, + temperature: 99, + }); + assert.equal(fallbackUrlEnricher.getProviderInfo().baseUrl, 'https://api.openai.com/v1'); + + const fractionalSpecs = sanitizeSpecifications({ + display: { size_inch: 6.7, refresh_rate_hz: 120.5, ppi: 450.2 }, + hardware: { ram_gb: 12.5, storage_gb: 256.8 }, + battery: { capacity_mah: 5000.5 }, + }); + assert.equal(fractionalSpecs.display?.size_inch, 6.7); + assert.equal(fractionalSpecs.display?.refresh_rate_hz, undefined); + assert.equal(fractionalSpecs.display?.ppi, undefined); + assert.equal(fractionalSpecs.hardware?.ram_gb, undefined); + assert.equal(fractionalSpecs.hardware?.storage_gb, undefined); + assert.equal(fractionalSpecs.battery?.capacity_mah, undefined); + + const enrichedBackfillStorage = await enricher.enrichCandidate({ + ...xiaomi, + ram_gb: 0 as unknown as null, + storage_gb: null, + is_benchmark_enriched: false, + }); + assert.equal(enrichedBackfillStorage.ram_gb, 16); + assert.equal(enrichedBackfillStorage.storage_gb, 512); + console.log('✅ [AiSpecEnricher] Mock OpenAI-compatible enrichment & caching passed 100%!'); } finally { mockServer.close(); diff --git a/apps/web/e2e/health.spec.ts b/apps/web/e2e/health.spec.ts new file mode 100644 index 000000000..1e6c9b715 --- /dev/null +++ b/apps/web/e2e/health.spec.ts @@ -0,0 +1,20 @@ +import { expect, test } from '@playwright/test'; + +// /api/health reads the database with the app's own client (DOS/Rate#829). +// The failure paths (refused key, network error, timeout, cache window) are +// unit-tested in core-rate-architecture guard 94: the probe result is cached +// per server instance, so a simulated outage here would leak into the specs +// running in parallel against the same server. +test('/api/health reports a readable database', async ({ request }) => { + const res = await request.get('/api/health'); + expect(res.status()).toBe(200); + expect(res.headers()['cache-control']).toBe('no-store, max-age=0'); + const body = (await res.json()) as { + status: string; + service: string; + checks: { database: string }; + }; + expect(body.status).toBe('ok'); + expect(body.service).toBe('rate-web'); + expect(body.checks).toEqual({ database: 'ok' }); +}); diff --git a/apps/web/e2e/reflow-contrast.spec.ts b/apps/web/e2e/reflow-contrast.spec.ts new file mode 100644 index 000000000..2aa54c8a0 --- /dev/null +++ b/apps/web/e2e/reflow-contrast.spec.ts @@ -0,0 +1,77 @@ +import AxeBuilder from '@axe-core/playwright'; +import { expect, test } from '@playwright/test'; + +// At 320px no page scrolls sideways and the phone menu toggle stays inside +// the viewport (DOS/Rate#809, WCAG 1.4.10 Reflow). +const PHONE_PAGES = [ + '/', + '/smartphones', + '/smartphones/iphone-16-pro-max-256gb', + '/merchants', + '/compare', +]; + +for (const path of PHONE_PAGES) { + test(`${path} fits a 320px screen`, async ({ page }) => { + await page.setViewportSize({ width: 320, height: 640 }); + expect((await page.goto(path))?.status()).toBe(200); + await page.evaluate(() => document.fonts.ready); + const scrollWidth = await page.evaluate( + () => document.documentElement.scrollWidth + ); + expect(scrollWidth).toBeLessThanOrEqual(320); + // The toggle's 44px tap area (its ::after, DOS/Rate#717) stays inside the + // viewport and clear of the header icons. + const toggle = page.locator('.nav-toggle'); + await expect(toggle).toBeVisible(); + const hit = await toggle.evaluate((el) => { + const box = el.getBoundingClientRect(); + const size = parseFloat(getComputedStyle(el, '::after').width) || 0; + const icons = [...document.querySelectorAll('.mobile_nav li')].map( + (li) => li.getBoundingClientRect().right + ); + return { + size, + left: box.left + (box.width - size) / 2, + right: box.left + (box.width + size) / 2, + iconsRight: Math.max(0, ...icons), + }; + }); + expect(hit.size).toBeGreaterThanOrEqual(44); + expect(hit.left).toBeGreaterThanOrEqual(hit.iconsRight); + expect(hit.right).toBeLessThanOrEqual(320); + }); +} + +test('the review sort control fits a 320px screen', async ({ page }) => { + await page.setViewportSize({ width: 320, height: 640 }); + expect( + (await page.goto('/smartphones/iphone-16-pro-max-256gb'))?.status() + ).toBe(200); + // Measured in page coordinates: scrolling it into view could also scroll + // the page sideways and hide the overflow. + const right = await page + .locator('#review-sort-select') + .evaluate((el) => el.getBoundingClientRect().right + window.scrollX); + expect(right).toBeLessThanOrEqual(320); +}); + +// Reviews excluded from the score are set apart without dimming their text +// below 4.5:1 (DOS/Rate#811). +for (const path of [ + '/smartphones/e2e-excluded-review-phone', + '/merchants/excluded-review-store', +]) { + test(`excluded review text meets contrast on ${path}`, async ({ page }) => { + expect((await page.goto(path))?.status()).toBe(200); + const excluded = page.getByText('shop uy tín, giao nhanh nhận xu').first(); + await expect(excluded).toBeVisible(); + await page.waitForLoadState('networkidle'); + const { violations } = await new AxeBuilder({ page }) + .withRules(['color-contrast']) + .analyze(); + expect( + violations.flatMap((v) => v.nodes.map((n) => n.target.join(' '))) + ).toEqual([]); + }); +} diff --git a/apps/web/e2e/seed.sql b/apps/web/e2e/seed.sql index 64c8ca405..163a3733e 100644 --- a/apps/web/e2e/seed.sql +++ b/apps/web/e2e/seed.sql @@ -277,3 +277,31 @@ VALUES ('cccccccc-cccc-4ccc-8ccc-ccccccc07974', 'ffffffff-ffff-4fff-8fff-ffffffff7972', 'shopee', 'item_bait_4', 'E2E Bait Price Phone', 'c0000000-0000-4000-8000-000000000001', 3690000, 'https://shopee.vn/bait-4', '22222222-2222-4222-8222-222222220797'); + +-- A phone with one review excluded from the score as coin-farming spam, shown +-- set apart on the product page: its text must still meet 4.5:1 contrast +-- (DOS/Rate#811). Its rating is never refreshed, so it stays off the +-- leaderboard, and its reviews are older than every other review. +INSERT INTO rate.merchants (id, platform, platform_shop_id, name, slug, is_official) +VALUES ('ffffffff-ffff-4fff-8fff-ffffffff0811', 'shopee', 'shop_excluded_review', 'Excluded Review Store', 'excluded-review-store', false); +INSERT INTO rate.master_products + (id, category_id, brand, model, model_key, name, slug, min_price, max_price, merchant_count, status, metadata) +VALUES ('22222222-2222-4222-8222-222222220811', 'c0000000-0000-4000-8000-000000000001', 'E2E', 'Excluded Review Phone', + 'e2e_excluded_review_phone', 'E2E Excluded Review Phone', 'e2e-excluded-review-phone', 3000000, 3000000, 1, 'published', + '{"category_slug": "smartphones"}'); +INSERT INTO rate.products (id, merchant_id, platform, platform_item_id, title, category_id, price_current, url, cluster_id) +VALUES ('cccccccc-cccc-4ccc-8ccc-cccccccc0811', 'ffffffff-ffff-4fff-8fff-ffffffff0811', 'shopee', 'item_excluded_review', + 'E2E Excluded Review Phone', 'c0000000-0000-4000-8000-000000000001', 3000000, 'https://shopee.vn/item-excluded-review', + '22222222-2222-4222-8222-222222220811'); +INSERT INTO rate.reviews (id, product_id, author_name, rating_star, comment, reviewer_type, platform_created_at, created_at) +VALUES + ('44444444-4444-4444-8444-444444440811', 'cccccccc-cccc-4ccc-8ccc-cccccccc0811', 'Excluded 1', 5, + 'Sản phẩm tốt tuyệt vời, shop uy tín, giao nhanh nhận xu.', 'user', now() - interval '100 days', now() - interval '100 days'), + ('44444444-4444-4444-8444-444444440812', 'cccccccc-cccc-4ccc-8ccc-cccccccc0811', 'Excluded 2', 4, + 'Máy chạy ổn, pin dùng được một ngày, màn hình hơi tối ngoài nắng.', 'user', now() - interval '100 days', now() - interval '100 days'); +INSERT INTO rate.review_flags (review_id, verdict, fake_probability, detector, detector_version, input_hash, include_in_rating, flag_category, reason) +VALUES + ('44444444-4444-4444-8444-444444440811', 'fake', 0.9, 'e2e', '1', 'e2e-0811', false, 'seeding_fake', + 'Bình luận mang tính chất nhận xu, không mô tả trải nghiệm.'), + ('44444444-4444-4444-8444-444444440812', 'authentic', 0.05, 'e2e', '1', 'e2e-0812', true, null, + 'Đánh giá chi tiết, hợp lệ từ người dùng thực tế.'); diff --git a/apps/web/src/app/[locale]/merchants/[id]/page.tsx b/apps/web/src/app/[locale]/merchants/[id]/page.tsx index 62d0a6b2b..70d864fad 100644 --- a/apps/web/src/app/[locale]/merchants/[id]/page.tsx +++ b/apps/web/src/app/[locale]/merchants/[id]/page.tsx @@ -693,9 +693,11 @@ export default async function MerchantProfilePage({ return (
+ {/* Set apart by background, not opacity, which dropped + the text below 4.5:1 (DOS/Rate#811). */}
diff --git a/apps/web/src/app/api/health/route.ts b/apps/web/src/app/api/health/route.ts index 272799348..c5295376a 100644 --- a/apps/web/src/app/api/health/route.ts +++ b/apps/web/src/app/api/health/route.ts @@ -1,17 +1,23 @@ import { NextResponse } from 'next/server'; +import { checkDatabase } from '@/server/health'; export const dynamic = 'force-dynamic'; +// 503 when the app cannot read its database, so an uptime monitor sees an +// outage that only uncached pages would otherwise show (DOS/Rate#829). export async function GET() { + const database = await checkDatabase(); + const healthy = database === 'ok'; return NextResponse.json( { - status: 'ok', + status: healthy ? 'ok' : 'degraded', service: 'rate-web', version: '2.0.0', timestamp: new Date().toISOString(), + checks: { database }, }, { - status: 200, + status: healthy ? 200 : 503, headers: { 'Cache-Control': 'no-store, max-age=0', }, diff --git a/apps/web/src/app/style/scss/_custom.scss b/apps/web/src/app/style/scss/_custom.scss index 33cc9004f..ae20609af 100644 --- a/apps/web/src/app/style/scss/_custom.scss +++ b/apps/web/src/app/style/scss/_custom.scss @@ -1500,3 +1500,22 @@ body .input-group > .input-group-text { .filterFlex .btn { white-space: nowrap; } + +// Phone header at 320px: brand, icon group and menu toggle need 327px of +// the 296px available, so the page scrolled sideways and the toggle was cut +// off. Tighten the spacing below 360px only (DOS/Rate#809). The 38px the +// template keeps before the toggle stays: the toggle's 44px tap area reaches +// 7px past its 30px box and must not cover the icons. +@media (max-width: 359.98px) { + .navigation .nav-brand { + margin-right: 0.5rem; + } + + .navigation .mobile_nav ul { + gap: 0.25rem !important; + } + + .navigation .mobile_nav ul > li.me-1 { + margin-right: 0 !important; + } +} diff --git a/apps/web/src/components/elementary/LocaleSwitcher.tsx b/apps/web/src/components/elementary/LocaleSwitcher.tsx index f2aee7443..b73c9252c 100644 --- a/apps/web/src/components/elementary/LocaleSwitcher.tsx +++ b/apps/web/src/components/elementary/LocaleSwitcher.tsx @@ -162,10 +162,14 @@ export function LocaleSwitcher({