From cc57e2d19e205fa6e896dccbd72d5958a3c8a2ac Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:45:37 +0700 Subject: [PATCH 01/12] fix(proposals): backfill every master with a reviewed listing (#1559) * fix(proposals): backfill every master with a reviewed listing The criteria ratings backfill picked masters by the cached authentic_review_count or user_rating_weighted. A master whose review weight was written straight to rate.reviews keeps a stale count of 0 and was skipped, so V6 failed. The DOS.Me replay caught it. The backfill now also picks any master with a reviewed listing; the cached test stays for Item variants whose reviews sit on listings with no variant. The Docker replay zeroes the 256GB master's cached count before the first apply and checks the backfill still rates it; it fails on the old filter. Refs #1543 Co-Authored-By: Claude Opus 5.5 --- Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql | 7 ++++++- scripts/test-criteria-ratings-proposal.mjs | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql index c783187c..a1731631 100644 --- a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql +++ b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql @@ -462,8 +462,13 @@ ALTER TABLE rate.reviews VALIDATE CONSTRAINT reviews_criteria_version_positive; ALTER TABLE rate.reviews VALIDATE CONSTRAINT reviews_value_score_range; ALTER TABLE rate.master_products VALIDATE CONSTRAINT master_products_rating_score_range; ALTER TABLE rate.master_products VALIDATE CONSTRAINT master_products_rating_weight_sum_nonneg; +-- A master with a reviewed listing, or one the live refresh rated (an Item variant whose +-- reviews sit on listings with no variant). The first test does not trust the cached +-- authentic_review_count: a weight written straight to rate.reviews leaves it stale. SELECT rate.refresh_master_rating_extras(m.id) FROM rate.master_products m -WHERE m.authentic_review_count > 0 OR m.user_rating_weighted > 0 +WHERE EXISTS (SELECT 1 FROM rate.products p JOIN rate.reviews r ON r.product_id = p.id + WHERE p.cluster_id = m.id) + OR m.authentic_review_count > 0 OR m.user_rating_weighted > 0 ORDER BY m.id; COMMIT; diff --git a/scripts/test-criteria-ratings-proposal.mjs b/scripts/test-criteria-ratings-proposal.mjs index 6288df75..f1beee70 100644 --- a/scripts/test-criteria-ratings-proposal.mjs +++ b/scripts/test-criteria-ratings-proposal.mjs @@ -121,7 +121,15 @@ try { const per256 = '3.650|1.00|{}'; const applyAndCheckPerMaster = (when) => { + // A stale cache: the 256GB has a counted review but authentic_review_count 0 (a weight + // written straight to rate.reviews). The backfill still picks it by its reviewed listing. + sql( + `UPDATE rate.master_products SET authentic_review_count = 0, user_rating_weighted = 0 + WHERE id = '${x256}'` + ); sql(proposal); + assert.equal(row(x256), per256, `the backfill rates a master with a stale count ${when}`); + sql(`SELECT rate.refresh_master_product_rating('${x256}')`); sql(proposal); // A criteria review gets its exact star: the reviews trigger refreshes the master. asRole( From 7f14fe5bff5d659e291ea8f1fd23ccd1bc4a8b8d Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:34:23 +0700 Subject: [PATCH 02/12] feat(shared-data): cluster review pros and cons into product points (#1560) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(shared-data): cluster review pros and cons into product points Pure rules for DOS/Rate#1546 (design #1531 section 5.1, D9), with no database access: normalise lines (diacritics kept, filler words dropped, negations joined to the next word), assign a criterion by keyword, cluster at 60% shared words, count each writer once by capped trust weight halved every 12 months, and show a point with two writers and a weight sum of 1.5 or one expert. Up to three per criterion and five per side in the summary; points silent for 18 months hide. Refs #1546 * fix(shared-data): product points review findings - Keywords match without diacritics only on a line typed without any, and words whose plain form is common ("đơ", "đo", "chờ", "êm", "cầm", "cam", "mắc") are dropped, so "Pin tụt nhanh do cập nhật" stays battery. - A keyword inside a longer match ("màn" in "màn hình") counts once. - A line that negates a word the other states never joins its cluster. - Lines outside 5..150 characters, with no words, no writer, no date or a non-finite weight are not counted. - D9 reads the exact support, not the two-decimal value. Refs #1546 Co-Authored-By: Claude Opus 5.5 --- .../src/__tests__/shared-data.test.ts | 239 ++++++++ packages/shared-data/src/rules/index.ts | 1 + .../shared-data/src/rules/product-points.ts | 552 ++++++++++++++++++ packages/shared-data/src/rules/review-form.ts | 2 +- 4 files changed, 793 insertions(+), 1 deletion(-) create mode 100644 packages/shared-data/src/rules/product-points.ts diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index d09917e5..4e1204bc 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -1709,6 +1709,243 @@ function testRatingCriteriaAndReviewForm(): void { assert.ok(Date.now() - started < 1000, 'linear on an oversized con'); } +function testProductPoints(): void { + const now = new Date('2026-11-01T00:00:00Z'); + const monthsAgo = (months: number): string => + new Date(now.getTime() - months * (365.25 / 12) * 86_400_000).toISOString(); + let next = 0; + const line = ( + writerId: string, + kind: rules.PointKind, + body: string, + extra: Partial = {} + ): rules.PointLine => { + next += 1; + return { + id: `l${String(next).padStart(3, '0')}`, + reviewId: `r-${writerId}`, + writerId, + kind, + criterionKey: null, + body, + weight: 1, + isExpert: false, + writtenAt: now.toISOString(), + ...extra, + }; + }; + const run = (lines: rules.PointLine[], slug = 'smartphones') => + rules.aggregateProductPoints(slug, lines, now); + + // Normalising: filler words go, diacritics stay, a negation joins the next + // word, so "pin không trâu" never matches "pin trâu". + assert.deepEqual(rules.pointTokens('Pin KHÁ trâu!!'), ['pin', 'trâu']); + assert.deepEqual(rules.pointTokens('pin không trâu'), ['pin', 'không_trâu']); + assert.deepEqual(rules.pointTokens('ko trâu'), ['không_trâu']); + assert.notDeepEqual( + rules.pointTokens('sạc nhanh'), + rules.pointTokens('sắc nhanh') + ); + assert.equal(rules.wordOverlap(['a', 'b'], ['a', 'b']), 1); + assert.equal(rules.wordOverlap(['a', 'b'], ['c']), 0); + assert.equal(rules.wordOverlap([], []), 0); + assert.ok( + rules.wordOverlap( + rules.pointTokens('pin trâu'), + rules.pointTokens('pin không trâu') + ) < rules.POINT_CLUSTER_OVERLAP + ); + + // Criteria: the line's own key when the category has it, else keywords, + // matched with or without diacritics but never with other ones. + const assign = rules.assignPointCriterion; + assert.equal(assign('smartphones', 'camera', 'pin trâu'), 'camera'); + assert.equal(assign('smartphones', 'anc', 'pin trâu'), 'battery'); + assert.equal(assign('smartphones', null, 'Pin trâu dùng cả ngày'), 'battery'); + assert.equal(assign('smartphones', null, 'pin trau, sac nhanh'), 'battery'); + assert.equal(assign('smartphones', null, 'cảm ứng mượt'), 'performance'); + assert.equal(assign('smartphones', null, 'Màn hình sáng rõ'), 'display'); + assert.equal(assign('earbuds', null, 'chống ồn tốt'), 'anc'); + assert.equal(assign('earbuds', null, 'Màn hình sáng rõ'), null); + assert.equal( + assign('smartphones', null, 'hộp đựng đẹp, giao đúng hẹn'), + null + ); + assert.equal(assign('smartphones', 'toString', 'abc def'), null); + + // A plain word that is a keyword without its diacritics ("do", "cho", + // "em", "cam") does not route a line, and "màn" inside "màn hình" is one hit. + assert.equal( + assign('smartphones', null, 'Pin tụt nhanh do cập nhật'), + 'battery' + ); + assert.equal( + assign('smartphones', null, 'Camera mờ do chống rung kém'), + 'camera' + ); + assert.equal( + assign('smartwatches', null, 'Pin tụt nhanh do bật GPS'), + 'battery' + ); + assert.equal(assign('smartphones', null, 'Màu cam đẹp lắm'), null); + assert.equal(assign('earbuds', null, 'Mua cho em gái, mic rõ'), 'calls'); + assert.equal( + assign('smartphones', null, 'màn hình đẹp, pin trâu'), + 'battery' + ); + + // Clustering: alike lines of one criterion and kind join; a pro and a con, + // or a negated line, do not. + const negated = run([ + line('a', 'con', 'Màn hình bị ám vàng nhẹ'), + line('b', 'con', 'Màn hình không bị ám vàng'), + ]); + assert.equal(negated.clusters.length, 2, 'a negation splits one kind'); + const alike = run([ + line('a', 'pro', 'Pin trâu, dùng cả ngày'), + line('b', 'pro', 'pin rất trâu dùng cả ngày'), + line('c', 'pro', 'Pin trâu dùng cả ngày luôn'), + line('d', 'con', 'pin không trâu dùng cả ngày'), + ]); + const batteryPro = alike.clusters.find((cl) => cl.kind === 'pro')!; + assert.equal(batteryPro.criterionKey, 'battery'); + assert.equal(batteryPro.reviewerCount, 3); + assert.deepEqual(batteryPro.lineIds.length, 3); + assert.equal(alike.clusters.length, 2); + assert.equal(alike.byCriterion.battery!.pros.length, 1); + assert.equal(alike.byCriterion.battery!.cons.length, 0, 'one writer: no con'); + assert.equal(alike.reviewCount, 4); + // The label is the form most writers used, in the first one's words. + const label = run([ + line('a', 'pro', 'Màn hình sáng, rõ nét', { writtenAt: monthsAgo(3) }), + line('b', 'pro', 'màn hình rất sáng rõ nét'), + line('c', 'pro', 'Màn hình sáng rõ nét!'), + ]); + assert.equal(label.clusters.length, 1); + assert.equal(label.clusters[0]!.label, 'Màn hình sáng, rõ nét'); + + // D9: two writers and a weight sum of 1.5, or one expert. + const two = (weight: number) => + run( + [ + line('a', 'pro', 'camera chụp đêm đẹp', { weight }), + line('b', 'pro', 'camera chụp đêm rất đẹp', { weight }), + ], + 'smartphones' + ).clusters[0]!; + assert.equal(two(0.7).visible, false, '1.4 is below 1.5'); + assert.equal(two(0.75).support, 1.5); + assert.equal(two(0.75).visible, true); + // The threshold reads the exact sum: 1.4952 is stored as 1.50 but hidden. + assert.equal(two(0.7476).support, 1.5); + assert.equal(two(0.7476).visible, false); + const expert = run([ + line('x', 'con', 'Camera tele hơi mờ', { isExpert: true }), + ]); + assert.equal(expert.clusters[0]!.visible, true); + assert.equal(expert.summary.cons.length, 1); + + // A hostile writer, even at full weight over several reviews, shows no con. + const hostile = run([ + line('h', 'con', 'Pin tụt nhanh kinh khủng', { weight: 3, reviewId: 'r1' }), + line('h', 'con', 'pin tụt rất nhanh kinh khủng', { + weight: 3, + reviewId: 'r2', + }), + line('h', 'con', 'Màn hình ám vàng', { weight: 3, reviewId: 'r1' }), + line('ok1', 'pro', 'Camera đẹp'), + line('ok2', 'pro', 'camera rất đẹp'), + ]); + assert.equal(hostile.summary.cons.length, 0); + assert.equal(hostile.summary.pros.length, 1); + const hostileBattery = hostile.clusters.find( + (cl) => cl.criterionKey === 'battery' + )!; + assert.equal(hostileBattery.reviewerCount, 1); + assert.equal( + hostileBattery.support, + 3, + 'one writer counts once, capped at 3' + ); + // Gated reviews (weight 0) and dodging cons are not counted at all. + const gated = run([ + line('a', 'con', 'loa nhỏ, rè', { weight: 0 }), + line('b', 'con', 'loa nhỏ rè', { weight: 0 }), + line('c', 'con', 'Không có nhược điểm nào cả'), + line('d', 'con', 'không có gì để chê'), + ]); + assert.equal(gated.clusters.length, 0); + assert.equal(gated.reviewCount, 0); + // Lines the review_points table would refuse, or with no writer, no date or + // a weight that is not a number, are not counted either. + const malformed = run([ + line('a', 'pro', 'ok'), + line('b', 'pro', 'ok'), + line('c', 'pro', 'aaaaaaa'), + line('d', 'pro', `pin trâu ${'x'.repeat(150)}`), + line('', 'pro', 'pin trâu cả ngày'), + { ...line('e', 'pro', 'pin trâu cả ngày'), writerId: undefined as never }, + { ...line('f', 'pro', 'pin trâu cả ngày'), writtenAt: null as never }, + line('g', 'pro', 'pin trâu cả ngày', { weight: Infinity }), + line('h', 'pro', 'pin trâu cả ngày', { weight: Number.NaN }), + ]); + assert.equal(malformed.clusters.length, 0); + + // Freshness: a weight halves every 12 months; 18 silent months hide a point. + assert.ok( + Math.abs(rules.decayedWeight(1, new Date(monthsAgo(12)), now) - 0.5) < 1e-9 + ); + assert.equal(rules.decayedWeight(5, now, now), rules.POINT_WEIGHT_CAP); + assert.equal(rules.decayedWeight(-1, now, now), 0); + const aged = (months: number) => + run([ + line('a', 'pro', 'loa to rõ', { + isExpert: true, + writtenAt: monthsAgo(months), + }), + ]).clusters[0]!.visible; + assert.equal(aged(17), true); + assert.equal(aged(19), false); + // Two writers at weight 1, a year old: 0.5 + 0.5 is below 1.5. + const old = run([ + line('a', 'pro', 'sạc nhanh', { writtenAt: monthsAgo(12) }), + line('b', 'pro', 'sạc rất nhanh', { writtenAt: monthsAgo(12) }), + ]).clusters[0]!; + assert.equal(old.support, 1); + assert.equal(old.visible, false); + + // Limits: three per criterion and five per side in the summary; a passing + // con is kept however many pros pass. + const pros = [ + 'pin trâu', + 'sạc nhanh', + 'pin bền lâu chai', + 'mah lớn', + 'camera đẹp', + 'màn hình sáng', + 'hiệu năng mạnh', + 'thiết kế sang', + ]; + const many: rules.PointLine[] = []; + for (const body of pros) + for (const writer of ['a', 'b', 'c']) + many.push(line(writer, 'pro', body, { weight: 3 })); + many.push(line('d', 'con', 'giá hơi cao', { weight: 0.8 })); + many.push(line('e', 'con', 'giá cao', { weight: 0.8 })); + const crowded = run(many); + assert.equal(crowded.byCriterion.battery!.pros.length, 3); + assert.equal(crowded.summary.pros.length, rules.PRODUCT_POINTS_MAX); + assert.equal(crowded.summary.cons.length, 1); + assert.equal(crowded.summary.cons[0]!.criterionKey, 'value'); + assert.equal(crowded.updatedAt, now); + // The same lines in another order give the same clusters. + const reversed = run([...many].reverse()); + assert.deepEqual( + reversed.clusters.map((cl) => [cl.key, cl.support, cl.lineIds]), + crowded.clusters.map((cl) => [cl.key, cl.support, cl.lineIds]) + ); +} + function main(): void { console.log('🧪 Running @repo/shared-data unit tests...'); testVersionAndZeroStrapiExports(); @@ -1745,6 +1982,8 @@ function main(): void { console.log( ' ✅ [11/11] Rating criteria and review form rules (DOS/Rate#1542) verified' ); + testProductPoints(); + console.log(' ✅ [12/12] Product pros and cons (DOS/Rate#1546) verified'); console.log('🎉 All @repo/shared-data tests passed!'); } diff --git a/packages/shared-data/src/rules/index.ts b/packages/shared-data/src/rules/index.ts index 3abdc626..042f81d1 100644 --- a/packages/shared-data/src/rules/index.ts +++ b/packages/shared-data/src/rules/index.ts @@ -13,3 +13,4 @@ export * from './model-key.js'; export * from './review-summary.js'; export * from './rating-criteria.js'; export * from './review-form.js'; +export * from './product-points.js'; diff --git a/packages/shared-data/src/rules/product-points.ts b/packages/shared-data/src/rules/product-points.ts new file mode 100644 index 00000000..e4fbfb1d --- /dev/null +++ b/packages/shared-data/src/rules/product-points.ts @@ -0,0 +1,552 @@ +/** + * A product's pros and cons, built from its reviews' pro and con lines + * (DOS/Rate#1546, design #1531 section 5.1, D9). Code only, no AI: lines are + * normalised, given a criterion, clustered by shared words, counted once per + * writer by trust weight, decayed by age and filtered for display. AI later + * rewords a cluster's label; it never decides what is shown. + * + * Plain data and functions only, like the rest of `rules`, and no database: + * the importer job reads `rate.review_points` into PointLine values and + * writes the clusters to `rate.product_points`. + */ +import { getRatingCriteria } from './rating-criteria.js'; +import { + REVIEW_POINT_MAX, + REVIEW_POINT_MIN, + comparablePoint, + hasContent, + isEvasiveCon, +} from './review-form.js'; + +/** Two lines join a cluster when this share of their words is shared. */ +export const POINT_CLUSTER_OVERLAP = 0.6; +/** D9: a point shows with this many distinct writers... */ +export const POINT_MIN_WRITERS = 2; +/** ...whose decayed weights sum to at least this, or with one expert. */ +export const POINT_MIN_SUPPORT = 1.5; +/** A writer's weight counts at most this, as in the product rating. */ +export const POINT_WEIGHT_CAP = 3; +/** Pros and cons shown per criterion, each side. */ +export const PRODUCT_CRITERION_POINTS_MAX = 3; +/** Pros and cons shown in the product summary, each side. */ +export const PRODUCT_POINTS_MAX = 5; + +const DAY_MS = 24 * 60 * 60 * 1000; +const YEAR_MS = 365.25 * DAY_MS; +/** A line's weight halves every 12 months (design 5.3). */ +export const POINT_HALF_LIFE_MS = YEAR_MS; +/** A cluster nobody mentioned for 18 months is hidden (design 5.3). */ +export const POINT_SILENT_MS = 1.5 * YEAR_MS; + +export type PointKind = 'pro' | 'con'; + +/** One pro or con line of a review, as the importer reads it. */ +export interface PointLine { + /** `rate.review_points.id`. */ + id: string; + reviewId: string; + /** Who wrote it: a user id, or a marketplace reviewer key. */ + writerId: string; + kind: PointKind; + /** The line's own criterion, or null for a pro or con of the whole review. */ + criterionKey: string | null; + body: string; + /** The review's trust weight; 0 means the gate excluded it. */ + weight: number; + /** A review Rate published as an expert's (reviewer_type = 'expert'). */ + isExpert: boolean; + /** When the review was written (ISO string or Date). */ + writtenAt: string | Date; +} + +/** A cluster of lines, as `rate.product_points` stores it. */ +export interface PointCluster { + /** Stable within one run: criterion, kind and the first line's id. */ + key: string; + criterionKey: string | null; + kind: PointKind; + /** The line most writers wrote alike, in its writer's words. */ + label: string; + /** Distinct writers' decayed, capped weights, at two decimals. */ + support: number; + reviewerCount: number; + hasExpert: boolean; + firstSeen: Date; + lastSeen: Date; + /** Every counted line in the cluster, for `review_points.cluster_id`. */ + lineIds: string[]; + /** Whether it passes D9 and is not silent (design 5.3). */ + visible: boolean; +} + +export interface PointSides { + pros: PointCluster[]; + cons: PointCluster[]; +} + +export interface ProductPoints { + /** Every cluster, shown or not, in display order. */ + clusters: PointCluster[]; + /** Shown points per criterion key; '' holds the whole-review ("Chung") ones. */ + byCriterion: Record; + /** The product summary: up to five each side, a passing con always kept. */ + summary: PointSides; + /** Distinct reviews whose lines were counted ("Tóm tắt từ N review"). */ + reviewCount: number; + /** The run's clock, shown as "Cập nhật ngày ...". */ + updatedAt: Date; +} + +// The rules' import check reads quotes on an export line as a module path. +const EMPTY = ''; +const SPACE = ' '; +/** The bucket of a whole-review point ("Chung"). */ +export const GENERAL_POINTS = EMPTY; + +// Intensifiers and particles that do not change what a line says +// ("pin khá trâu" is "pin trâu"). +const FILLER = new Set([ + 'khá', + 'rất', + 'hơi', + 'quá', + 'lắm', + 'cực', + 'siêu', + 'cũng', + 'thì', + 'là', + 'ạ', + 'nhé', + 'nha', +]); +const NEGATED = 'không_'; +// A negation joins the next word, so "pin không trâu" never matches +// "pin trâu" by sharing "pin" and "trâu". +const NEGATION = new Set([ + 'không', + 'ko', + 'k', + 'kh', + 'khum', + 'hok', + 'chưa', + 'chẳng', + 'chả', +]); + +/** A word without diacritics, "đ" as "d". */ +function plainWord(word: string): string { + return word + .normalize('NFKD') + .replace(/\p{M}+/gu, '') + .replace(/đ/g, 'd'); +} + +/** + * The words two lines are compared by: the comparable line (diacritics + * kept, design 5.1) without filler words, each negation joined to the word + * after it ("không_trâu"). + */ +export function pointTokens(text: string): string[] { + const words = comparablePoint(text) + .split(SPACE) + .filter((word) => word && !FILLER.has(word)); + const tokens: string[] = []; + for (let i = 0; i < words.length; i += 1) { + const word = words[i]!; + if (NEGATION.has(word) && i + 1 < words.length) { + tokens.push(`${NEGATED}${words[i + 1]}`); + i += 1; + } else tokens.push(NEGATION.has(word) ? 'không' : word); + } + return tokens; +} + +/** + * The share of words two lines have in common: twice the shared distinct + * words over both lines' distinct words (the Dice coefficient), from 0 to 1. + */ +export function wordOverlap( + a: readonly string[], + b: readonly string[] +): number { + const left = new Set(a); + const right = new Set(b); + if (left.size + right.size === 0) return 0; + let shared = 0; + for (const word of left) if (right.has(word)) shared += 1; + return (2 * shared) / (left.size + right.size); +} + +// Keywords per criterion, written with diacritics. The analyzer's aspect +// words (battery, display, camera, audio, performance) are the start. A word +// whose plain form is a common word is left out: "đơ" and "đo" read "do" +// (because), "chờ" reads "cho", "êm" reads "em", "cầm" and "cam" read +// the colour, "mắc" reads "Mac". +const CRITERION_KEYWORDS: Readonly> = { + battery: ['pin', 'sạc', 'mah', 'sạc nhanh', 'thời lượng', 'củ sạc'], + display: [ + 'màn hình', + 'màn', + 'hiển thị', + 'độ sáng', + 'tần số quét', + 'nit', + 'ám màu', + 'chói', + ], + camera: [ + 'camera', + 'chụp', + 'chụp ảnh', + 'quay', + 'quay phim', + 'selfie', + 'zoom', + 'góc rộng', + ], + performance: [ + 'hiệu năng', + 'mượt', + 'lag', + 'giật', + 'chip', + 'game', + 'chơi game', + 'cấu hình', + 'ram', + 'xử lý', + ], + design: [ + 'thiết kế', + 'bền', + 'nhẹ', + 'nặng', + 'vỏ', + 'khung', + 'mỏng', + 'chắc chắn', + 'chống nước', + 'ngoại hình', + ], + value: ['giá', 'tiền', 'đáng tiền', 'rẻ', 'đắt', 'hời'], + sound: ['âm thanh', 'chất âm', 'loa', 'bass', 'treble', 'âm lượng'], + software: ['phần mềm', 'ứng dụng', 'cập nhật', 'ios', 'android', 'giao diện'], + keyboard: ['bàn phím', 'phím', 'touchpad', 'trackpad'], + thermals: ['nóng', 'nhiệt', 'quạt', 'ồn', 'tản nhiệt'], + health: [ + 'sức khoẻ', + 'sức khỏe', + 'nhịp tim', + 'spo2', + 'giấc ngủ', + 'ngủ', + 'bước chân', + ], + apps: [ + 'ứng dụng', + 'app', + 'kết nối', + 'thông báo', + 'bluetooth', + 'wifi', + 'gps', + 'nfc', + ], + anc: ['chống ồn', 'anc', 'xuyên âm', 'cách âm'], + comfort: ['đeo', 'thoải mái', 'đau tai', 'vừa tai', 'cấn'], + calls: ['mic', 'micro', 'gọi', 'cuộc gọi', 'đàm thoại', 'nghe gọi'], + service_quality: ['chất lượng', 'dịch vụ', 'sạch', 'ngon'], + price: ['giá', 'tiền', 'rẻ', 'đắt'], + attitude: ['thái độ', 'nhân viên', 'phục vụ', 'nhiệt tình', 'lịch sự'], + speed: ['nhanh', 'chậm', 'đợi', 'lâu'], +}; + +/** + * Whether a keyword's words appear in a line's words, in order. A line typed + * without any diacritics ("pin trau, sac nhanh") matches the plain keyword; + * a line with diacritics matches only as written, so "cảm" (cảm ứng) is not + * "cam" and "do" (because) is not "đo". + */ +function containsKeyword( + words: readonly string[], + keyword: string, + plainLine: boolean +): boolean { + const wanted = (plainLine ? plainWord(keyword) : keyword).split(SPACE); + for (let start = 0; start + wanted.length <= words.length; start += 1) { + if (wanted.every((want, offset) => words[start + offset] === want)) + return true; + } + return false; +} + +/** Matched keywords, not counting one inside a longer match ("màn" in "màn hình"). */ +function keywordHits( + words: readonly string[], + keywords: readonly string[], + plainLine: boolean +): number { + const matched = keywords.filter((kw) => + containsKeyword(words, kw, plainLine) + ); + return matched.filter( + (kw) => + !matched.some((other) => other !== kw && ` ${other} `.includes(` ${kw} `)) + ).length; +} + +/** + * The criterion a line belongs to in a category: its own key when the + * category has it, otherwise the criterion whose keywords it mentions most + * (ties go to the category's order), otherwise null ("Chung"). + */ +export function assignPointCriterion( + categorySlug: string | null | undefined, + criterionKey: string | null, + body: string +): string | null { + const criteria = getRatingCriteria(categorySlug); + if (criterionKey && criteria.some((cr) => cr.key === criterionKey)) + return criterionKey; + const words = comparablePoint(body).split(SPACE); + const plainLine = words.every((word) => word === plainWord(word)); + let best: string | null = null; + let bestHits = 0; + for (const criterion of criteria) { + const keywords = Object.prototype.hasOwnProperty.call( + CRITERION_KEYWORDS, + criterion.key + ) + ? CRITERION_KEYWORDS[criterion.key]! + : []; + const hits = keywordHits(words, keywords, plainLine); + if (hits > bestHits) { + best = criterion.key; + bestHits = hits; + } + } + return best; +} + +/** A weight after decay: half per POINT_HALF_LIFE_MS of age, capped. */ +export function decayedWeight( + weight: number, + writtenAt: Date, + now: Date +): number { + if (!Number.isFinite(weight) || weight <= 0) return 0; + const age = Math.max(0, now.getTime() - writtenAt.getTime()); + return Math.min(weight, POINT_WEIGHT_CAP) * 0.5 ** (age / POINT_HALF_LIFE_MS); +} + +interface Line { + source: PointLine; + criterionKey: string | null; + tokens: string[]; + form: string; + writtenAt: Date; +} + +function asDate(value: string | Date | null | undefined): Date { + if (value instanceof Date) return value; + // new Date(null) is 1970, not an unknown date. + return typeof value === 'string' ? new Date(value) : new Date(Number.NaN); +} + +/** + * Two lines that say opposite things: one negates a word the other states + * ("không_bị" against "bị"), however many other words they share. + */ +function contradicts(a: readonly string[], b: readonly string[]): boolean { + const negates = (from: readonly string[], to: Set): boolean => + from.some( + (token) => + token.startsWith(NEGATED) && to.has(token.slice(NEGATED.length)) + ); + return negates(a, new Set(b)) || negates(b, new Set(a)); +} + +/** + * The lines that may count: a writer, a finite positive weight, a date, a + * length rate.review_points allows, words in it, and not a dodge. + */ +function countedLines( + categorySlug: string | null | undefined, + lines: readonly PointLine[] +): Line[] { + const counted: Line[] = []; + for (const source of lines) { + if (source.kind !== 'pro' && source.kind !== 'con') continue; + if (!Number.isFinite(source.weight) || source.weight <= 0) continue; + if (typeof source.writerId !== 'string' || !source.writerId.trim()) + continue; + const writtenAt = asDate(source.writtenAt); + if (Number.isNaN(writtenAt.getTime())) continue; + const body = String(source.body ?? EMPTY).trim(); + const length = [...body.normalize('NFKC')].length; + if (length < REVIEW_POINT_MIN || length > REVIEW_POINT_MAX) continue; + if (!hasContent(body)) continue; + if (source.kind === 'con' && isEvasiveCon(body)) continue; + const tokens = pointTokens(body); + if (tokens.length === 0) continue; + counted.push({ + source, + criterionKey: assignPointCriterion( + categorySlug, + source.criterionKey, + source.body + ), + tokens, + form: tokens.join(SPACE), + writtenAt, + }); + } + // Oldest first, so a cluster's key and leader do not depend on read order. + return counted.sort( + (a, b) => + a.writtenAt.getTime() - b.writtenAt.getTime() || + (a.source.id < b.source.id ? -1 : a.source.id > b.source.id ? 1 : 0) + ); +} + +/** + * Greedy clustering within one criterion and kind: a line joins the cluster + * whose first line it overlaps most, at POINT_CLUSTER_OVERLAP or more, or + * starts a new one. Comparing with the first line, not any member, keeps a + * chain of small steps from joining two different points. + */ +function clusterLines(lines: readonly Line[]): Line[][] { + const clusters: Line[][] = []; + for (const line of lines) { + let best: Line[] | undefined; + let bestOverlap = POINT_CLUSTER_OVERLAP; + for (const cluster of clusters) { + const leader = cluster[0]!; + if ( + leader.criterionKey !== line.criterionKey || + leader.source.kind !== line.source.kind + ) + continue; + const overlap = contradicts(leader.tokens, line.tokens) + ? 0 + : wordOverlap(leader.tokens, line.tokens); + if (overlap >= bestOverlap && (!best || overlap > bestOverlap)) { + best = cluster; + bestOverlap = overlap; + } + } + if (best) best.push(line); + else clusters.push([line]); + } + return clusters; +} + +/** The label: the form most writers used, in its first writer's words. */ +function clusterLabel(lines: readonly Line[]): string { + const forms = new Map; first: Line }>(); + for (const line of lines) { + const entry = forms.get(line.form); + if (entry) entry.writers.add(line.source.writerId); + else + forms.set(line.form, { + writers: new Set([line.source.writerId]), + first: line, + }); + } + let best: { writers: Set; first: Line } | undefined; + for (const entry of forms.values()) + if (!best || entry.writers.size > best.writers.size) best = entry; + return best!.first.source.body.trim().replace(/\s+/g, ' '); +} + +function summarise(lines: readonly Line[], now: Date): PointCluster { + const leader = lines[0]!; + // Each writer counts once, with their strongest line in the cluster. + const writers = new Map(); + let hasExpert = false; + for (const line of lines) { + const weight = decayedWeight(line.source.weight, line.writtenAt, now); + const writer = line.source.writerId; + writers.set(writer, Math.max(writers.get(writer) ?? 0, weight)); + if (line.source.isExpert) hasExpert = true; + } + let rawSupport = 0; + for (const weight of writers.values()) rawSupport += weight; + const support = Math.round(rawSupport * 100) / 100; + const firstSeen = leader.writtenAt; + const lastSeen = lines[lines.length - 1]!.writtenAt; + const fresh = now.getTime() - lastSeen.getTime() <= POINT_SILENT_MS; + const passes = + (writers.size >= POINT_MIN_WRITERS && rawSupport >= POINT_MIN_SUPPORT) || + hasExpert; + return { + key: `${leader.criterionKey ?? GENERAL_POINTS}:${leader.source.kind}:${leader.source.id}`, + criterionKey: leader.criterionKey, + kind: leader.source.kind, + label: clusterLabel(lines), + support, + reviewerCount: writers.size, + hasExpert, + firstSeen, + lastSeen, + lineIds: lines.map((line) => line.source.id), + visible: passes && fresh, + }; +} + +/** Display order: more support, more writers, an expert, then newer. */ +function byRank(a: PointCluster, b: PointCluster): number { + return ( + b.support - a.support || + b.reviewerCount - a.reviewerCount || + Number(b.hasExpert) - Number(a.hasExpert) || + b.lastSeen.getTime() - a.lastSeen.getTime() || + (a.key < b.key ? -1 : a.key > b.key ? 1 : 0) + ); +} + +function sides(clusters: readonly PointCluster[], max: number): PointSides { + return { + pros: clusters.filter((cl) => cl.kind === 'pro').slice(0, max), + cons: clusters.filter((cl) => cl.kind === 'con').slice(0, max), + }; +} + +/** + * A product's pros and cons from its review lines (design 5.1 and 5.3): + * lines of gated reviews (weight 0), dodging cons and lines with no words + * are left out; the rest are clustered per criterion and kind, counted once + * per writer and shown under D9. `now` is the run's clock, so a run is + * repeatable. + */ +export function aggregateProductPoints( + categorySlug: string | null | undefined, + lines: readonly PointLine[], + now: Date +): ProductPoints { + const counted = countedLines(categorySlug, lines); + const clusters = clusterLines(counted) + .map((cluster) => summarise(cluster, now)) + .sort(byRank); + const shown = clusters.filter((cl) => cl.visible); + const byCriterion: Record = {}; + for (const key of new Set( + shown.map((cl) => cl.criterionKey ?? GENERAL_POINTS) + )) + byCriterion[key] = sides( + shown.filter((cl) => (cl.criterionKey ?? GENERAL_POINTS) === key), + PRODUCT_CRITERION_POINTS_MAX + ); + // Each side has its own five places, so a passing con is always kept in + // the summary however many pros pass (design 5.1, "luôn giữ nhược điểm"). + const summary = sides(shown, PRODUCT_POINTS_MAX); + return { + clusters, + byCriterion, + summary, + reviewCount: new Set(counted.map((line) => line.source.reviewId)).size, + updatedAt: now, + }; +} diff --git a/packages/shared-data/src/rules/review-form.ts b/packages/shared-data/src/rules/review-form.ts index 3a4453c3..9576d9ef 100644 --- a/packages/shared-data/src/rules/review-form.ts +++ b/packages/shared-data/src/rules/review-form.ts @@ -68,7 +68,7 @@ function visibleLength(text: string): number { * Whether a line has words in it: at least three letters or digits, and * not one character repeated ("aaaaa", "....", "🔥🔥🔥🔥🔥"). */ -function hasContent(text: string): boolean { +export function hasContent(text: string): boolean { const letters = comparablePoint(text).match(/[\p{L}\p{N}]/gu) ?? []; return letters.length >= 3 && new Set(letters).size >= 2; } From 2877e90a97cb2d6e05bdebe4330d6d549f6626ca Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:37:50 +0700 Subject: [PATCH 03/12] fix(importer): reject used, like-new and refurbished listings at ingest (#1561) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(importer): reject used, like-new and refurbished listings at ingest (#1430) rate.products stores no condition, so a non-new listing linked to a published master (exact model_key, variant or a master hint) fed its lower price into the new device's min_price and the deal radar. - ingestProduct throws NonNewListingError (code NON_NEW_LISTING) before anything is written when the normalizer detects a non-new condition, with or without a master hint (option A of #1430, no DDL). - Trade-in offers for the buyer's old device ("nhận thu máy cũ", "thu cũ lên đời", "đổi máy cũ lấy máy mới") no longer read as used, so new listings are not dropped for them. - The error is exported from @repo/crawlers (dist smoke). * fix(importer): only a non-new title rejects a listing; 2nd Gen is new (#1430 review) - The ingest check reads the title alone. Description boilerplate such as "không bán hàng cũ" or "99% khách hàng hài lòng" would drop new listings, so a description hit still only sets needs_review. - "2nd" counts as used only in "2nd hand": "AirPods Pro 2nd Gen" and "iPhone SE 2nd generation" are new devices. - The trade-in strip also covers "thu máy cũ đổi máy mới" and "thu mua máy cũ". * fix(importer): check the sanitized title; a bare 98% is not a condition (#1430 Codex) - The normalizer sets title_condition from the sanitized title, and the ingest check reads it, so a control character inside "Máy cũ" no longer slips a used listing past the check. - A bare "98%" or "99%" (such as "Chống ồn 98%") no longer reads as like-new; a percentage counts only after máy, ngoại hình, hình thức, keng or pin, besides the existing mới, đẹp and zin. * fix(importer): "thu mua điện thoại cũ" is a trade-in offer too (#1430 Codex) Co-Authored-By: Claude Opus 5.5 --- .../shopee/__tests__/dist-exports.smoke.mjs | 1 + .../shopee/__tests__/entity-matcher.test.ts | 34 ++++++++++++++ .../__tests__/shopee-to-supabase.test.ts | 44 ++++++++++++++++++- .../importer/src/shopee/product-normalizer.ts | 39 +++++++++++++--- .../importer/src/shopee/shopee-to-supabase.ts | 9 ++++ 5 files changed, 120 insertions(+), 7 deletions(-) diff --git a/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs b/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs index a7a985ba..ab05ac84 100644 --- a/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs +++ b/apps/importer/src/shopee/__tests__/dist-exports.smoke.mjs @@ -27,6 +27,7 @@ for (const name of [ 'computeRetryDelayMs', 'resolveItemModelMode', 'UnkeyableProductTitleError', + 'NonNewListingError', ]) { assert.equal( typeof crawlers[name], diff --git a/apps/importer/src/shopee/__tests__/entity-matcher.test.ts b/apps/importer/src/shopee/__tests__/entity-matcher.test.ts index 7b38c261..55785b15 100644 --- a/apps/importer/src/shopee/__tests__/entity-matcher.test.ts +++ b/apps/importer/src/shopee/__tests__/entity-matcher.test.ts @@ -261,6 +261,40 @@ async function runEntityMatcherSuite() { 'new', 'Trade-in promotion ("thu cũ đổi mới") must not misclassify a new phone as used' ); + for (const description of [ + 'Nhận thu máy cũ giá cao', + 'Hỗ trợ thu mua máy cũ, trả góp 0%', + 'Thu cũ lên đời, bảo hành 12 tháng', + 'Đổi máy cũ lấy máy mới tại cửa hàng', + 'Thu máy cũ đổi máy mới, trợ giá', + 'Thu mua máy cũ giá cao', + 'Mới 100% - Thu mua điện thoại cũ giá cao', + ]) { + assert.equal( + detectProductCondition('Điện thoại Samsung Galaxy S24 Ultra 256GB', description), + 'new', + `A trade-in offer (${description}) does not make a new listing used (DOS/Rate#1430)` + ); + } + assert.equal( + detectProductCondition('Điện thoại Samsung Galaxy S24 Ultra 256GB', 'Máy cũ, pin 90%'), + 'used' + ); + for (const title of [ + 'Tai nghe Apple AirPods Pro 2nd Gen USB-C Chính hãng VN/A', + 'Apple iPhone SE 2nd generation 64GB', + ]) { + assert.equal(detectProductCondition(title), 'new', `${title} is a new device`); + } + assert.equal(detectProductCondition('iPhone 13 128GB 2nd hand'), 'used'); + // A bare percentage is a spec or an ad, not a condition; one about the device is. + assert.equal( + detectProductCondition( + 'Tai nghe Anker Soundcore Liberty 4 NC Mới 100% Chống ồn 98% Chính hãng' + ), + 'new' + ); + assert.equal(detectProductCondition('iPhone 13 Pro 128GB máy 98% pin 99%'), 'like_new_99'); assert.equal( detectProductCondition('Điện thoại Apple iPhone 15 Pro Max 256GB Likenew 99% Zin Nguyên Bản'), 'like_new_99' diff --git a/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts b/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts index c5e3aee9..68826178 100644 --- a/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts +++ b/apps/importer/src/shopee/__tests__/shopee-to-supabase.test.ts @@ -17,7 +17,7 @@ import { normalizeScrapedProductPrices, platformReviewIdFor, } from '../shopee-to-supabase.js'; -import { UnkeyableProductTitleError } from '../product-normalizer.js'; +import { NonNewListingError, UnkeyableProductTitleError } from '../product-normalizer.js'; import type { RateSupabaseClient, ShopeeProduct, ShopeeReview } from '../types.js'; function createBaseProduct(overrides: Partial = {}): ShopeeProduct { @@ -532,6 +532,48 @@ async function runTests() { UnkeyableProductTitleError ); assert.equal(accessoryEvents.productUpserts.length, 0); + // A used, like-new or refurbished listing is rejected before anything is written, with + // or without a hint: its price would become the new device's min_price (DOS/Rate#1430). + for (const [title, hint, condition] of [ + ['Like new 99% iPhone 15 128GB', undefined, 'like_new_99'], + ['Điện thoại iPhone 14 Pro Max 128GB Máy cũ đã qua sử dụng', 'master-hint-verified', 'used'], + ['Apple iPhone 15 Pro 256GB Hàng Tân Trang Apple CPO', 'master-hint-verified', 'refurbished'], + // A control character inside the phrase is stripped before the check. + ['Apple iPhone 14 Pro Max 128GB Máy\u0000 cũ', 'master-hint-verified', 'used'], + ] as const) { + const { stub: nonNewStub, events: nonNewEvents } = createPipelineStubSupabase({ + hintRow: { id: 'master-hint-verified' }, + }); + await assert.rejects( + new ShopeeIngestionPipeline({ supabaseClient: nonNewStub, apiKey: '' }).ingestProduct( + createBaseProduct({ title }), + undefined, + hint ? { masterIdHint: hint } : {} + ), + (err: unknown) => + err instanceof NonNewListingError && + err.code === 'NON_NEW_LISTING' && + err.condition === condition + ); + assert.equal(nonNewEvents.productUpserts.length, 0, `${title} writes no listing`); + } + // Only the title rejects: description boilerplate that reads as non-new (a negation, + // "99% khách hàng") keeps a new listing (DOS/Rate#1430). + for (const description of [ + 'Nhận thu máy cũ, hỗ trợ trả góp 0%', + 'Cam kết hàng chính hãng, không bán hàng cũ, hàng tân trang', + '99% khách hàng hài lòng', + ]) { + const { stub: descStub, events: descEvents } = createPipelineStubSupabase({ + hintRow: { id: 'master-hint-verified' }, + }); + await new ShopeeIngestionPipeline({ supabaseClient: descStub, apiKey: '' }).ingestProduct( + createBaseProduct({ description }), + undefined, + { masterIdHint: 'master-hint-verified' } + ); + assert.equal(descEvents.productUpserts.length, 1, `${description} keeps the listing`); + } assert.equal(hintOkEvents.productUpserts[0]?.cluster_id, 'master-hint-verified'); assert.equal(hintOkEvents.productUpserts[0]?.category_id, 'cat-smartphones'); assert.equal( diff --git a/apps/importer/src/shopee/product-normalizer.ts b/apps/importer/src/shopee/product-normalizer.ts index d5392bda..166d3b2a 100644 --- a/apps/importer/src/shopee/product-normalizer.ts +++ b/apps/importer/src/shopee/product-normalizer.ts @@ -96,6 +96,8 @@ export interface NormalizedProductCandidate { storage_gb: number | null; // null = unparseable from the title; identity carries ':unknown' needs_review: boolean; // true when identity could not be confidently derived or listing is non-new (plan v7 §5.2, Audit C6) condition?: ProductCondition; + /** The condition the sanitized title alone states; the ingest check reads it (DOS/Rate#1430). */ + title_condition?: ProductCondition; origin?: ProductOrigin; ram_gb: number | null; screen_size_inch: number | null; @@ -134,16 +136,40 @@ export class UnkeyableProductTitleError extends Error { } } +/** + * A used, like-new or refurbished listing. `rate.products` stores no condition, so + * linked to a master its lower price would become the new device's `min_price` and + * feed the deal radar. The listing is rejected before anything is written until + * condition is stored (DOS/Rate#1430). + */ +export class NonNewListingError extends Error { + readonly code = 'NON_NEW_LISTING'; + readonly condition: Exclude; + + constructor(title: unknown, condition: Exclude) { + super( + `Cannot ingest a non-new listing (condition=${condition}): ${JSON.stringify(String(title ?? '').slice(0, 200))}` + ); + this.name = 'NonNewListingError'; + this.condition = condition; + } +} + +/** A trade-in offer for the buyer's old device, which says nothing about the listing's own. */ +const TRADE_IN_PHRASE = + /(?:thu|đổi)\s*(?:mua\s*)?(?:(?:máy|điện\s*thoại)\s*)?cũ\s*(?:lên\s*đời|(?:lấy|đổi)\s*(?:(?:máy|điện\s*thoại)\s*)?mới)|thu\s*(?:mua\s*)?(?:máy|điện\s*thoại)\s*cũ/gi; + /** * Detect listing physical condition from product title and optional description (Audit C6). - * Strips promotional trade-in phrases ("thu cũ đổi mới") before matching so new retail listings - * offering trade-in programs are not misclassified as used devices. + * Strips promotional trade-in phrases ("thu cũ đổi mới", "thu máy cũ lên đời") before matching + * so new retail listings offering trade-in programs are not misclassified as used devices; a + * non-new listing is rejected at ingest (DOS/Rate#1430). */ export function detectProductCondition(title: string, description?: string): ProductCondition { const safeTitle = typeof title === 'string' ? title : ''; const safeDesc = typeof description === 'string' ? description : ''; - const cleanTitle = safeTitle.replace(/thu\s*cũ\s*đổi\s*mới/gi, ' '); - const cleanDesc = safeDesc.replace(/thu\s*cũ\s*đổi\s*mới/gi, ' '); + const cleanTitle = safeTitle.replace(TRADE_IN_PHRASE, ' '); + const cleanDesc = safeDesc.replace(TRADE_IN_PHRASE, ' '); const checkSource = (text: string): ProductCondition | null => { if ( @@ -154,14 +180,14 @@ export function detectProductCondition(title: string, description?: string): Pro return 'refurbished'; } if ( - /(?:\blike\s*new\b|\blikenew\b|mới\s*9[5-9]%|đẹp\s*9[5-9]%|zin\s*9[5-9]%|\b9[89]%|siêu\s*lướt|hàng\s*lướt|máy\s*lướt|hàng\s*trưng\s*bày|máy\s*trưng\s*bày|\bopen\s*box\b|đã\s*kích\s*hoạt\s*bảo\s*hành)/i.test( + /(?:\blike\s*new\b|\blikenew\b|mới\s*9[5-9]%|đẹp\s*9[5-9]%|zin\s*9[5-9]%|(?:máy|ngoại\s*hình|hình\s*thức|keng|pin)\s*9[5-9]%|siêu\s*lướt|hàng\s*lướt|máy\s*lướt|hàng\s*trưng\s*bày|máy\s*trưng\s*bày|\bopen\s*box\b|đã\s*kích\s*hoạt\s*bảo\s*hành)/i.test( text ) ) { return 'like_new_99'; } if ( - /(?:máy\s*cũ|hàng\s*cũ|điện\s*thoại\s*cũ|đã\s*qua\s*sử\s*dụng|\bsecond\s*hand\b|\b2nd\b|thanh\s*lý\s*máy\s*cũ|máy\s*trần\s*cũ)/i.test( + /(?:máy\s*cũ|hàng\s*cũ|điện\s*thoại\s*cũ|đã\s*qua\s*sử\s*dụng|\bsecond\s*hand\b|\b2nd\s*hand\b|thanh\s*lý\s*máy\s*cũ|máy\s*trần\s*cũ)/i.test( text ) ) { @@ -3218,6 +3244,7 @@ export class ProductNormalizer { storage_gb: storageKnown ? storageGb : null, needs_review: !keyable || !storageConfident || condition !== 'new', condition, + title_condition: detectProductCondition(title), origin, ram_gb: ramGb, screen_size_inch: screenSize, diff --git a/apps/importer/src/shopee/shopee-to-supabase.ts b/apps/importer/src/shopee/shopee-to-supabase.ts index f2730ced..7f453324 100644 --- a/apps/importer/src/shopee/shopee-to-supabase.ts +++ b/apps/importer/src/shopee/shopee-to-supabase.ts @@ -18,6 +18,7 @@ import { } from './types.js'; import { isAccessoryTitle, + NonNewListingError, ProductNormalizer, UnkeyableProductTitleError, } from './product-normalizer.js'; @@ -538,6 +539,14 @@ export class ShopeeIngestionPipeline { if (candidate.keyable === false && (!options.masterIdHint || isAccessoryTitle(product.title))) { throw new UnkeyableProductTitleError(product.title); } + // A used, like-new or refurbished listing is rejected even with a hint: with no + // stored condition, its price would become the new device's min_price (DOS/Rate#1430). + // Only the title decides: description boilerplate ("không bán hàng cũ", "99% khách + // hàng hài lòng") would drop new listings, so a description hit still only sets needs_review. + const condition = candidate.title_condition ?? 'new'; + if (condition !== 'new') { + throw new NonNewListingError(product.title, condition); + } // 1. Upsert merchant const entityHash = hashEntity(`${platform}:${product.shop.shopId}`); From d0b10349b5a96129059d487a26c2ccf28511dffa Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:53:06 +0700 Subject: [PATCH 04/12] fix(importer): trade-in titles in any order stay new (#1562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(importer): trade-in titles in any order stay new (#1430) detectProductCondition read new-device titles such as "Trợ giá lên đời máy cũ", "Lên đời máy cũ trợ giá" or "Hỗ trợ đổi máy cũ" as used, so ingest threw NON_NEW_LISTING on new listings. - The trade-in strip now takes a trade-in verb (thu, thu mua, đổi, lên đời, trợ giá, hỗ trợ) right before "(máy|điện thoại|đt) cũ", with a few filler words allowed, or "cũ" right before "lên đời" or "đổi/lấy/sang mới". - Fixtures: nine trade-in titles stay new, and five real used titles stay used. Refs #1430 * fix(importer): keep warranty swaps used; allow amounts and hyphens (#1430 review) - "Mới" followed by a number or a warranty term ("đổi mới 99%", "đổi mới trong 30 ngày") is not a trade-in, so those used listings stay used or like-new. - A bare "hỗ trợ" is no longer a trade-in verb; "hỗ trợ thu/đổi" still is. - "lại", "giá cao" and an amount ("2 triệu") may sit between the verb and the device, and a hyphen, colon or comma may separate the words. Refs #1430 Co-Authored-By: Claude Opus 5.5 --- .../shopee/__tests__/entity-matcher.test.ts | 33 +++++++++++++++++++ .../importer/src/shopee/product-normalizer.ts | 27 +++++++++++++-- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/apps/importer/src/shopee/__tests__/entity-matcher.test.ts b/apps/importer/src/shopee/__tests__/entity-matcher.test.ts index 55785b15..e6cdb7c5 100644 --- a/apps/importer/src/shopee/__tests__/entity-matcher.test.ts +++ b/apps/importer/src/shopee/__tests__/entity-matcher.test.ts @@ -280,6 +280,39 @@ async function runEntityMatcherSuite() { detectProductCondition('Điện thoại Samsung Galaxy S24 Ultra 256GB', 'Máy cũ, pin 90%'), 'used' ); + // A new device's title offering a trade-in in any order stays new (DOS/Rate#1430). + for (const title of [ + 'Samsung Galaxy S24 Ultra 256GB - Trợ giá lên đời máy cũ', + 'Samsung Galaxy S24 Ultra 256GB Lên đời máy cũ trợ giá 2 triệu', + 'Samsung Galaxy S24 Ultra 256GB Hỗ trợ đổi máy cũ', + 'iPhone 16 Pro Max 256GB Chính hãng VN/A - Đổi máy cũ', + 'iPhone 16 Pro Max 256GB Trợ giá máy cũ 1 triệu', + 'iPhone 16 Pro Max 256GB Thu cũ lên đời', + 'iPhone 16 Pro Max 256GB Máy cũ lên đời trợ giá', + 'iPhone 16 Pro Max 256GB Lên đời từ điện thoại cũ', + 'iPhone 16 Pro Max 256GB Thu đt cũ đổi mới', + 'iPhone 16 Pro Max 256GB Thu mua giá cao máy cũ', + 'iPhone 16 Pro Max 256GB Thu mua lại máy cũ', + 'iPhone 16 Pro Max 256GB Trợ giá 2 triệu cho máy cũ', + 'iPhone 16 Pro Max 256GB Lên đời - máy cũ trợ giá', + 'iPhone 16 Pro Max 256GB Lên-đời máy cũ', + ]) { + assert.equal(detectProductCondition(title), 'new', `${title} is a new device`); + } + // A real used listing stays used. + for (const title of [ + 'Điện thoại iPhone 12 Pro 128GB máy cũ đẹp', + 'iPhone 11 64GB Máy cũ giá rẻ bảo hành 3 tháng', + 'Samsung Galaxy S21 điện thoại cũ đã qua sử dụng', + 'Thanh lý máy cũ iPhone XS Max', + 'iPhone 13 cũ - máy cũ zin, trợ giá ship', + // A warranty swap is not a trade-in, and a bare "hỗ trợ" is not a trade-in verb. + 'iPhone 11 64GB máy cũ đổi mới trong 30 ngày', + 'iPhone 12 Pro hỗ trợ máy cũ', + ]) { + assert.equal(detectProductCondition(title), 'used', `${title} is a used device`); + } + assert.equal(detectProductCondition('Điện thoại cũ đổi mới 99%'), 'like_new_99'); for (const title of [ 'Tai nghe Apple AirPods Pro 2nd Gen USB-C Chính hãng VN/A', 'Apple iPhone SE 2nd generation 64GB', diff --git a/apps/importer/src/shopee/product-normalizer.ts b/apps/importer/src/shopee/product-normalizer.ts index 166d3b2a..5321efaf 100644 --- a/apps/importer/src/shopee/product-normalizer.ts +++ b/apps/importer/src/shopee/product-normalizer.ts @@ -155,9 +155,30 @@ export class NonNewListingError extends Error { } } -/** A trade-in offer for the buyer's old device, which says nothing about the listing's own. */ -const TRADE_IN_PHRASE = - /(?:thu|đổi)\s*(?:mua\s*)?(?:(?:máy|điện\s*thoại)\s*)?cũ\s*(?:lên\s*đời|(?:lấy|đổi)\s*(?:(?:máy|điện\s*thoại)\s*)?mới)|thu\s*(?:mua\s*)?(?:máy|điện\s*thoại)\s*cũ/gi; +/** + * A trade-in offer for the buyer's old device, which says nothing about the listing's own: + * a trade-in verb right before "(máy|điện thoại) cũ" ("thu mua máy cũ", "lên đời máy cũ", + * "trợ giá máy cũ", "hỗ trợ đổi máy cũ"), or "cũ" right before "lên đời" or "đổi/lấy mới" + * ("thu cũ đổi mới", "máy cũ lên đời"). A few filler words or an amount ("2 triệu") may sit + * between verb and device, and a hyphen, colon or comma may separate the words. "Mới" is not + * a trade-in when a number or a warranty term follows it ("đổi mới 99%", "đổi mới trong 30 + * ngày"), and a bare "hỗ trợ" is not a trade-in verb ("hỗ trợ máy cũ"). + */ +const SEP = String.raw`[\s\-:,]*`; +const OLD_DEVICE = String.raw`(?:(?:máy|điện\s*thoại|đt)${SEP})?cũ`; +const TRADE_IN_FILLER = String.raw`(?:(?:từ|cho|mọi|các|bằng|khi|lại|giá\s*cao|\d+(?:[.,]\d+)?\s*(?:triệu|tr|k|đ))${SEP})*`; +const TRADE_IN_TO_NEW = String.raw`(?:lên${SEP}đời|(?:lấy|đổi|sang)${SEP}(?:(?:máy|điện\s*thoại)${SEP})?mới(?!\s*(?:\d|trong|sau|nếu|khi)))`; +const TRADE_IN_PHRASE = new RegExp( + String.raw`(?:(?:hỗ\s*trợ${SEP})?(?:thu(?:\s*mua)?|đổi|lên${SEP}đời)|trợ\s*giá)${SEP}` + + TRADE_IN_FILLER + + OLD_DEVICE + + String.raw`(?:${SEP}${TRADE_IN_TO_NEW})?` + + '|' + + OLD_DEVICE + + SEP + + TRADE_IN_TO_NEW, + 'gi' +); /** * Detect listing physical condition from product title and optional description (Audit C6). From b2abe73bb3568749a0856f02b648588ffd0cdfd7 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:51:01 +0700 Subject: [PATCH 05/12] fix(shared-data): product points second review findings (#1563) Refs #1546 --- .../src/__tests__/shared-data.test.ts | 50 +++++++++++ .../shared-data/src/rules/product-points.ts | 83 ++++++++++++++++--- 2 files changed, 121 insertions(+), 12 deletions(-) diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index 4e1204bc..511ce1ce 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -1801,6 +1801,28 @@ function testProductPoints(): void { line('b', 'con', 'Màn hình không bị ám vàng'), ]); assert.equal(negated.clusters.length, 2, 'a negation splits one kind'); + assert.deepEqual(rules.pointTokens('máy không hề nóng'), [ + 'máy', + 'không_nóng', + ]); + // Lines that share a frame but state different facts stay apart: another + // aspect ("nóng" against "lag") or another number. + const claims = (a: string, b: string) => + run([line('a', 'con', a), line('b', 'con', b)]).clusters.length; + assert.equal(claims('máy nóng khi chơi game', 'máy lag khi chơi game'), 2); + assert.equal(claims('pin dùng được 1 ngày', 'pin dùng được 2 ngày'), 2); + // A short line inside a longer one joins it, whichever is older. + for (const ids of [ + ['x1', 'x2'], + ['x2', 'x1'], + ]) + assert.equal( + run([ + line('a', 'pro', 'pin trâu', { id: ids[0] }), + line('b', 'pro', 'pin trâu dùng cả ngày', { id: ids[1] }), + ]).clusters.length, + 1 + ); const alike = run([ line('a', 'pro', 'Pin trâu, dùng cả ngày'), line('b', 'pro', 'pin rất trâu dùng cả ngày'), @@ -1890,6 +1912,34 @@ function testProductPoints(): void { line('h', 'pro', 'pin trâu cả ngày', { weight: Number.NaN }), ]); assert.equal(malformed.clusters.length, 0); + // A line read twice counts once; writer ids are compared trimmed. + const twice = line('a', 'con', 'loa rè khi mở to'); + const dup = run([ + twice, + twice, + { ...twice, writerId: 'b' }, + line(' c', 'con', 'loa rè khi mở to'), + line('c ', 'con', 'loa rè khi mở to'), + ]).clusters[0]!; + assert.deepEqual(dup.lineIds.length, 3); + assert.equal(dup.reviewerCount, 2); + // An old expert line does not lift one fresh writer's point, and a future + // date counts as now. + const stale = run([ + line('x', 'con', 'pin tụt nhanh', { + isExpert: true, + writtenAt: monthsAgo(60), + }), + line('h', 'con', 'pin tụt rất nhanh'), + ]).clusters[0]!; + assert.equal(stale.hasExpert, false); + assert.equal(stale.visible, false); + const future = run([ + line('a', 'pro', 'loa to rõ', { + writtenAt: new Date(now.getTime() + 86_400_000 * 400).toISOString(), + }), + ]).clusters[0]!; + assert.equal(future.lastSeen.getTime(), now.getTime()); // Freshness: a weight halves every 12 months; 18 silent months hide a point. assert.ok( diff --git a/packages/shared-data/src/rules/product-points.ts b/packages/shared-data/src/rules/product-points.ts index e4fbfb1d..5cc26a62 100644 --- a/packages/shared-data/src/rules/product-points.ts +++ b/packages/shared-data/src/rules/product-points.ts @@ -119,6 +119,7 @@ const FILLER = new Set([ 'ạ', 'nhé', 'nha', + 'hề', ]); const NEGATED = 'không_'; // A negation joins the next word, so "pin không trâu" never matches @@ -343,6 +344,8 @@ export function decayedWeight( interface Line { source: PointLine; + /** The writer id, trimmed. */ + writer: string; criterionKey: string | null; tokens: string[]; form: string; @@ -368,22 +371,74 @@ function contradicts(a: readonly string[], b: readonly string[]): boolean { return negates(a, new Set(b)) || negates(b, new Set(a)); } +/** Single-word keywords: a word that names an aspect ("nóng", "lag"). */ +const ASPECT_WORDS: ReadonlySet = new Set( + Object.values(CRITERION_KEYWORDS) + .flat() + .filter((keyword) => !keyword.includes(SPACE)) +); + +/** + * Two lines that state different facts while sharing most words: each names + * an aspect the other does not ("máy nóng khi chơi game" against "máy lag + * khi chơi game"), or their numbers differ ("1 ngày" against "2 ngày"). + */ +function differentClaims(a: readonly string[], b: readonly string[]): boolean { + const left = new Set(a); + const right = new Set(b); + const only = ( + from: Set, + to: Set, + test: (word: string) => boolean + ): boolean => [...from].some((word) => test(word) && !to.has(word)); + const isAspect = (word: string): boolean => ASPECT_WORDS.has(word); + const isNumber = (word: string): boolean => /^\p{N}+$/u.test(word); + return ( + (only(left, right, isAspect) && only(right, left, isAspect)) || + only(left, right, isNumber) || + only(right, left, isNumber) + ); +} + +/** + * How well a line matches a cluster's first line: 0 when they contradict or + * state different facts; at least POINT_CLUSTER_OVERLAP when every word of + * the shorter one (two words or more) is in the longer one ("pin trâu" in + * "pin trâu dùng cả ngày"); otherwise their word overlap. + */ +function matchScore(a: readonly string[], b: readonly string[]): number { + if (contradicts(a, b) || differentClaims(a, b)) return 0; + const overlap = wordOverlap(a, b); + const [shorter, longer] = a.length <= b.length ? [a, b] : [b, a]; + const inside = new Set(longer); + const contained = + new Set(shorter).size >= 2 && shorter.every((word) => inside.has(word)); + return contained ? Math.max(overlap, POINT_CLUSTER_OVERLAP) : overlap; +} + /** * The lines that may count: a writer, a finite positive weight, a date, a * length rate.review_points allows, words in it, and not a dodge. */ function countedLines( categorySlug: string | null | undefined, - lines: readonly PointLine[] + lines: readonly PointLine[], + now: Date ): Line[] { const counted: Line[] = []; + const seen = new Set(); for (const source of lines) { + // A line read twice counts once. + if (seen.has(source.id)) continue; + seen.add(source.id); if (source.kind !== 'pro' && source.kind !== 'con') continue; if (!Number.isFinite(source.weight) || source.weight <= 0) continue; if (typeof source.writerId !== 'string' || !source.writerId.trim()) continue; - const writtenAt = asDate(source.writtenAt); - if (Number.isNaN(writtenAt.getTime())) continue; + const dated = asDate(source.writtenAt); + if (Number.isNaN(dated.getTime())) continue; + // A future date is now: it neither escapes decay nor keeps a point fresh. + const writtenAt = dated.getTime() > now.getTime() ? now : dated; const body = String(source.body ?? EMPTY).trim(); const length = [...body.normalize('NFKC')].length; if (length < REVIEW_POINT_MIN || length > REVIEW_POINT_MAX) continue; @@ -393,6 +448,7 @@ function countedLines( if (tokens.length === 0) continue; counted.push({ source, + writer: source.writerId.trim(), criterionKey: assignPointCriterion( categorySlug, source.criterionKey, @@ -429,9 +485,7 @@ function clusterLines(lines: readonly Line[]): Line[][] { leader.source.kind !== line.source.kind ) continue; - const overlap = contradicts(leader.tokens, line.tokens) - ? 0 - : wordOverlap(leader.tokens, line.tokens); + const overlap = matchScore(leader.tokens, line.tokens); if (overlap >= bestOverlap && (!best || overlap > bestOverlap)) { best = cluster; bestOverlap = overlap; @@ -448,10 +502,10 @@ function clusterLabel(lines: readonly Line[]): string { const forms = new Map; first: Line }>(); for (const line of lines) { const entry = forms.get(line.form); - if (entry) entry.writers.add(line.source.writerId); + if (entry) entry.writers.add(line.writer); else forms.set(line.form, { - writers: new Set([line.source.writerId]), + writers: new Set([line.writer]), first: line, }); } @@ -468,9 +522,14 @@ function summarise(lines: readonly Line[], now: Date): PointCluster { let hasExpert = false; for (const line of lines) { const weight = decayedWeight(line.source.weight, line.writtenAt, now); - const writer = line.source.writerId; - writers.set(writer, Math.max(writers.get(writer) ?? 0, weight)); - if (line.source.isExpert) hasExpert = true; + writers.set(line.writer, Math.max(writers.get(line.writer) ?? 0, weight)); + // An expert counts while their line is fresh: an old expert line does not + // lift a new single writer's point (design 5.3). + if ( + line.source.isExpert && + now.getTime() - line.writtenAt.getTime() <= POINT_SILENT_MS + ) + hasExpert = true; } let rawSupport = 0; for (const weight of writers.values()) rawSupport += weight; @@ -526,7 +585,7 @@ export function aggregateProductPoints( lines: readonly PointLine[], now: Date ): ProductPoints { - const counted = countedLines(categorySlug, lines); + const counted = countedLines(categorySlug, lines, now); const clusters = clusterLines(counted) .map((cluster) => summarise(cluster, now)) .sort(byRank); From a3afba26757ff3aca58086d33e0bb8d13cb056f7 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:07:42 +0700 Subject: [PATCH 06/12] fix(shared-data): each writer counts at most 1 toward the D9 sum (#1565) Refs #1546, #1543 --- ...6-10-31-dos-me-criteria-ratings-verify.sql | 3 ++- .../2026-10-31-dos-me-criteria-ratings.md | 2 +- .../2026-10-31-dos-me-criteria-ratings.sql | 7 ++++-- .../src/__tests__/shared-data.test.ts | 23 +++++++++++++++++++ .../shared-data/src/rules/product-points.ts | 19 ++++++++++++--- scripts/test-criteria-ratings-proposal.mjs | 19 ++++++++++++++- 6 files changed, 65 insertions(+), 8 deletions(-) diff --git a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings-verify.sql b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings-verify.sql index 9fe8ff51..24a1ba7a 100644 --- a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings-verify.sql +++ b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings-verify.sql @@ -44,7 +44,8 @@ WHERE n.nspname = 'rate' AND NOT t.tgisinternal AND t.tgenabled <> 'D' 'trg_review_flags_rating_extras_upd', 'trg_review_flags_rating_extras_del', 'trg_master_products_rating_extras'); --- V5 Every criterion score meets the display thresholds of D9. expect 0 +-- V5 Every criterion score has 3 writers and a stored weight sum of 1.5. expect 0 +-- (D9 also caps each review at 1 toward that sum; V6 checks it.) SELECT count(*) AS criteria_below_threshold FROM rate.master_products m, jsonb_each(m.criteria_scores) c(key, v) WHERE (c.v ->> 'writers')::int < 3 diff --git a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.md b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.md index b8c8bbe1..280b920d 100644 --- a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.md +++ b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.md @@ -38,7 +38,7 @@ Design: [`2026-10-31-criteria-stars-pros-cons.html`](./2026-10-31-criteria-stars - `criteria_scores`: `{key: {score, weight_sum, writers}}`. A criterion appears only when all of these hold (D9): - 3 or more reviews rated it; - each rating is a number from 1 to 5, under a key shaped like the shared-data keys; - - the reviews' weight sum is 1.5 or more. + - the reviews' weight sum is 1.5 or more, each review counting at most 1 toward it, so one heavy review plus near-zero ones never passes. The stored `weight_sum` is the plain sum. Readers apply the ranking threshold of 5 from `writers`. - **Which reviews count:** the same reviews `refresh_master_product_rating` counts. diff --git a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql index a1731631..7f1fe320 100644 --- a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql +++ b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql @@ -161,7 +161,9 @@ END $$; -- criteria = the same estimate per criterion of criteria_ratings, over the reviews that -- rated it with a number from 1 to 5 under a key shaped like the shared-data -- keys (^[a-z_]{2,32}$); kept only with 3 or more reviews and a weight sum of --- 1.5 or more (D9). "writers" lets a reader apply the ranking threshold (5). A +-- 1.5 or more (D9), each review counting at most 1 toward that sum, so one heavy +-- review plus near-zero ones never passes. The stored weight_sum is the plain +-- sum. "writers" lets a reader apply the ranking threshold (5). A -- Rate user writes one review per product, so a review is a writer. The prior -- mean is 4.2 for every criterion until per-category means exist. @@ -221,6 +223,7 @@ BEGIN SELECT c.key, count(*) AS writers, round(sum(e.w), 2) AS weight_sum, + sum(LEAST(1.0, e.w)) AS shown_sum, round((3 * 4.2 + sum(e.w * (c.value #>> '{}')::numeric)) / (3 + sum(e.w)), 3) AS score FROM eligible e CROSS JOIN LATERAL jsonb_each( @@ -238,7 +241,7 @@ BEGIN (SELECT jsonb_object_agg(pc.key, jsonb_build_object( 'score', pc.score, 'weight_sum', pc.weight_sum, 'writers', pc.writers)) FROM per_criterion pc - WHERE pc.writers >= 3 AND pc.weight_sum >= 1.5), + WHERE pc.writers >= 3 AND pc.shown_sum >= 1.5), '{}'::jsonb) FROM overall o $q$, CASE WHEN v_item IS NULL THEN 'p.cluster_id' ELSE 'p.item_id' END) diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index 511ce1ce..358135c6 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -1861,6 +1861,29 @@ function testProductPoints(): void { // The threshold reads the exact sum: 1.4952 is stored as 1.50 but hidden. assert.equal(two(0.7476).support, 1.5); assert.equal(two(0.7476).visible, false); + // Each writer adds at most 1 toward 1.5: 3 + 0.01 is one voice, not two; + // 1 + 0.5 passes. Support (for ranking) keeps the full sum. + const heavy = run([ + line('a', 'pro', 'camera chụp đêm đẹp', { weight: 3 }), + line('b', 'pro', 'camera chụp đêm rất đẹp', { weight: 0.01 }), + ]).clusters[0]!; + assert.equal(heavy.support, 3.01); + assert.equal(heavy.visible, false, '3 + 0.01 does not pass'); + const even = run([ + line('a', 'pro', 'camera chụp đêm đẹp', { weight: 3 }), + line('b', 'pro', 'camera chụp đêm rất đẹp', { weight: 0.5 }), + ]).clusters[0]!; + assert.equal(even.visible, true, '1 + 0.5 passes'); + // The cap applies after decay: weight 3 two years old is 0.75, which counts + // in full, so with another 0.75 it passes. + const decayedHeavy = run([ + line('a', 'pro', 'camera chụp đêm đẹp', { + weight: 3, + writtenAt: monthsAgo(24), + }), + line('b', 'pro', 'camera chụp đêm rất đẹp', { weight: 0.75 }), + ]).clusters[0]!; + assert.equal(decayedHeavy.visible, true); const expert = run([ line('x', 'con', 'Camera tele hơi mờ', { isExpert: true }), ]); diff --git a/packages/shared-data/src/rules/product-points.ts b/packages/shared-data/src/rules/product-points.ts index 5cc26a62..5fa334d0 100644 --- a/packages/shared-data/src/rules/product-points.ts +++ b/packages/shared-data/src/rules/product-points.ts @@ -22,8 +22,17 @@ import { export const POINT_CLUSTER_OVERLAP = 0.6; /** D9: a point shows with this many distinct writers... */ export const POINT_MIN_WRITERS = 2; -/** ...whose decayed weights sum to at least this, or with one expert. */ +/** + * ...whose decayed weights, each writer counting at most + * POINT_WRITER_SHARE_CAP, sum to at least this; or with one expert. + */ export const POINT_MIN_SUPPORT = 1.5; +/** + * The most one writer adds toward POINT_MIN_SUPPORT, so one heavy writer and + * a near-zero one (3 + 0.01) are not two real voices. Ranking still uses the + * full support. + */ +export const POINT_WRITER_SHARE_CAP = 1; /** A writer's weight counts at most this, as in the product rating. */ export const POINT_WEIGHT_CAP = 3; /** Pros and cons shown per criterion, each side. */ @@ -532,13 +541,17 @@ function summarise(lines: readonly Line[], now: Date): PointCluster { hasExpert = true; } let rawSupport = 0; - for (const weight of writers.values()) rawSupport += weight; + let shownSupport = 0; + for (const weight of writers.values()) { + rawSupport += weight; + shownSupport += Math.min(weight, POINT_WRITER_SHARE_CAP); + } const support = Math.round(rawSupport * 100) / 100; const firstSeen = leader.writtenAt; const lastSeen = lines[lines.length - 1]!.writtenAt; const fresh = now.getTime() - lastSeen.getTime() <= POINT_SILENT_MS; const passes = - (writers.size >= POINT_MIN_WRITERS && rawSupport >= POINT_MIN_SUPPORT) || + (writers.size >= POINT_MIN_WRITERS && shownSupport >= POINT_MIN_SUPPORT) || hasExpert; return { key: `${leader.criterionKey ?? GENERAL_POINTS}:${leader.source.kind}:${leader.source.id}`, diff --git a/scripts/test-criteria-ratings-proposal.mjs b/scripts/test-criteria-ratings-proposal.mjs index f1beee70..418ccf3c 100644 --- a/scripts/test-criteria-ratings-proposal.mjs +++ b/scripts/test-criteria-ratings-proposal.mjs @@ -128,7 +128,11 @@ try { WHERE id = '${x256}'` ); sql(proposal); - assert.equal(row(x256), per256, `the backfill rates a master with a stale count ${when}`); + assert.equal( + row(x256), + per256, + `the backfill rates a master with a stale count ${when}` + ); sql(`SELECT rate.refresh_master_product_rating('${x256}')`); sql(proposal); // A criteria review gets its exact star: the reviews trigger refreshes the master. @@ -159,6 +163,19 @@ try { `UPDATE rate.review_flags SET include_in_rating = true WHERE review_id = '${review(3)}'` ); assert.equal(row(x128), per128, `review 3 back ${when}`); + // D9: each review counts at most 1 toward the 1.5 sum, so one heavy review and two + // near-zero ones (3 + 0.01 + 0.01) show no display score. + sql(`UPDATE rate.reviews SET helpfulness_weight = 3 WHERE id = '${review(1)}'; + UPDATE rate.reviews SET helpfulness_weight = 0.01 + WHERE id IN ('${review(2)}', '${review(3)}')`); + assert.match( + row(x128), + /\|\{\}$/, + `one heavy writer shows no criterion ${when}` + ); + sql(`UPDATE rate.reviews SET helpfulness_weight = 1 + WHERE id IN ('${review(1)}', '${review(2)}', '${review(3)}')`); + assert.equal(row(x128), per128, `weights back ${when}`); for (const fn of [ `rate.refresh_master_rating_extras('${x128}')`, From 189247d7a40686df11e748efaf10cbfb50125e68 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:13:34 +0700 Subject: [PATCH 07/12] fix(ops): merge-when-green lints the squash header without blocking (#1566) Closes #1564 --- scripts/ops/merge-when-green.mjs | 111 +++++++++++++--- scripts/ops/merge-when-green.test.mjs | 179 +++++++++++++++++++++++--- 2 files changed, 252 insertions(+), 38 deletions(-) diff --git a/scripts/ops/merge-when-green.mjs b/scripts/ops/merge-when-green.mjs index 6cd7700d..f1439bf2 100644 --- a/scripts/ops/merge-when-green.mjs +++ b/scripts/ops/merge-when-green.mjs @@ -13,7 +13,7 @@ // --ignore lets a known-broken, non-required check through; name it in the PR. // Prints one line and exits 0 when merged, 1 when it cannot merge. -import { spawnSync } from 'node:child_process'; +import { spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -209,18 +209,69 @@ export function squashTitle(title, number) { const REPO_ROOT = fileURLToPath(new URL('../../', import.meta.url)); const COMMITLINT_CLI = 'node_modules/@commitlint/cli/cli.js'; -export const LINT_TIMEOUT_MS = 60_000; +/** + * How long commitlint may take before the header fails closed. The CLI spends + * its time opening about 750 small module files at almost no CPU; on a cold + * Windows checkout with antivirus scanning that took up to two minutes of wall + * time (DOS/Rate#1564). MERGE_WHEN_GREEN_LINT_TIMEOUT_MS overrides it. + */ +export const LINT_TIMEOUT_MS = 180_000; + +export function lintTimeoutMs(env = process.env) { + const raw = env.MERGE_WHEN_GREEN_LINT_TIMEOUT_MS; + const ms = Number(raw); + return raw && Number.isInteger(ms) && ms > 0 ? ms : LINT_TIMEOUT_MS; +} + +/** Runs the commitlint CLI on stdin without blocking; resolves when it ends. */ +function runCommitlint(header, { cwd, spawnFn, timeoutMs }) { + return new Promise((resolve) => { + let child; + try { + child = spawnFn(process.execPath, [COMMITLINT_CLI], { + cwd, + stdio: ['pipe', 'pipe', 'pipe'], + windowsHide: true, + }); + } catch (error) { + resolve({ error, status: null, stdout: '', stderr: '' }); + return; + } + let stdout = ''; + let stderr = ''; + let settled = false; + const finish = (r) => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve({ stdout, stderr, ...r }); + }; + // A hung commitlint is killed and fails closed. + const timer = setTimeout(() => { + child.kill(); + finish({ timedOut: true, status: null }); + }, timeoutMs); + child.stdout?.setEncoding('utf8').on('data', (d) => (stdout += d)); + child.stderr?.setEncoding('utf8').on('data', (d) => (stderr += d)); + child.on('error', (error) => finish({ error, status: null })); + child.on('close', (status) => finish({ status })); + child.stdin?.on('error', () => {}); + child.stdin?.end(header); + }); +} /** * Lints a squash header with the repository's own commitlint config * (commitlint.config.ts), as the commitlint check will: type-enum, case, * length and the rest. The shape check in squashTitle let "ux(web): ..." through - * (DOS/Rate#1513). Fails closed when commitlint cannot run, so no header is - * merged unchecked. `problems` holds commitlint's own lines. + * (DOS/Rate#1513). commitlint runs as a child process, so the run keeps polling + * while it loads and the timeout can stop it (DOS/Rate#1564). Fails closed when + * commitlint cannot run or times out, so no header is merged unchecked. + * `problems` holds commitlint's own lines. */ -export function lintHeader( +export async function lintHeader( header, - { cwd = REPO_ROOT, spawnFn = spawnSync } = {} + { cwd = REPO_ROOT, spawnFn = spawn, timeoutMs = lintTimeoutMs() } = {} ) { if (!existsSync(path.join(cwd, COMMITLINT_CLI))) { return { @@ -230,15 +281,15 @@ export function lintHeader( ], }; } - // A hung commitlint fails closed after a minute instead of holding the merge. - const r = spawnFn(process.execPath, [COMMITLINT_CLI], { - cwd, - input: header, - encoding: 'utf8', - timeout: LINT_TIMEOUT_MS, - }); + const r = await runCommitlint(header, { cwd, spawnFn, timeoutMs }); + if (r.timedOut) { + return { + ok: false, + problems: [`commitlint timed out after ${timeoutMs / 1000} s`], + }; + } if (r.error || r.status === null) { - return { ok: false, problems: ['commitlint could not run or timed out'] }; + return { ok: false, problems: ['commitlint could not run'] }; } if (r.status === 0) return { ok: true, problems: [] }; // commitlint prints "✖ [rule]" lines, coloured on a terminal. @@ -452,9 +503,24 @@ export async function mergeWhenGreen({ }; } // Linted once per header: the title is re-read each poll and may change. - if (!headerLints.has(header)) headerLints.set(header, lint(header)); - const linted = headerLints.get(header); - if (!linted.ok) { + // The lint runs while the checks are awaited; a rejection it has + // reported stops the run at once, and a merge waits for its verdict. + if (!headerLints.has(header)) { + const entry = { result: null }; + entry.promise = Promise.resolve() + .then(() => lint(header)) + .catch((err) => ({ + ok: false, + problems: [`commitlint could not run: ${err?.message ?? err}`], + })) + .then((r) => { + entry.result = r; + return r; + }); + headerLints.set(header, entry); + } + const linted = headerLints.get(header).result; + if (linted && !linted.ok) { return { ok: false, message: `PR ${number}: commitlint rejects the squash header "${header}": ${linted.problems.join('; ')}`, @@ -510,6 +576,14 @@ export async function mergeWhenGreen({ // mixes their trailers into the body: send one commitlint accepts. const squash = {}; if (method === 'squash') { + const header = squashTitle(pr.title, number); + const linted = await headerLints.get(header).promise; + if (!linted.ok) { + return { + ok: false, + message: `PR ${number}: commitlint rejects the squash header "${header}": ${linted.problems.join('; ')}`, + }; + } let messages; try { messages = io @@ -638,4 +712,7 @@ if (isMain(import.meta.url)) { }); if (!result.ok) fail(result.message); console.log(result.message); + // A commitlint child still running (the PR was found merged, say) must not + // hold the process open (DOS/Rate#1564). + process.exit(0); } diff --git a/scripts/ops/merge-when-green.test.mjs b/scripts/ops/merge-when-green.test.mjs index 6a623bd2..9b6c69f6 100644 --- a/scripts/ops/merge-when-green.test.mjs +++ b/scripts/ops/merge-when-green.test.mjs @@ -1,7 +1,11 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; +import { EventEmitter } from 'node:events'; +import { mkdtempSync, rmSync, symlinkSync } from 'node:fs'; import os from 'node:os'; +import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { PassThrough, Writable } from 'node:stream'; import test from 'node:test'; import { @@ -17,6 +21,7 @@ import { squashBody, LINT_TIMEOUT_MS, lintHeader, + lintTimeoutMs, mergeWhenGreen, squashTitle, wrapLine, @@ -403,14 +408,16 @@ test('the squash message of PR bodies with lines over 100 characters passes comm assert.doesNotMatch(result.output, /warning/); }); -test('lints the squash header with the repository commitlint, type-enum included', () => { +test('lints the squash header with the repository commitlint, type-enum included', async () => { // A valid header passes. - assert.deepEqual(lintHeader('fix(web): a thing (#7)'), { + assert.deepEqual(await lintHeader('fix(web): a thing (#7)'), { ok: true, problems: [], }); // "ux" fits the shape check but not commitlint's type-enum (DOS/Rate#1513). - const ux = lintHeader('ux(web): the same product shows one rating (#1475)'); + const ux = await lintHeader( + 'ux(web): the same product shows one rating (#1475)' + ); assert.equal( squashTitle('ux(web): the same product shows one rating', 1475), 'ux(web): the same product shows one rating (#1475)' @@ -418,49 +425,139 @@ test('lints the squash header with the repository commitlint, type-enum included assert.equal(ux.ok, false); assert.ok( ux.problems.some( - (p) => /type must be one of/.test(p) && /type-enum/.test(p) + (p) => /type must be one of/.test(p) && /\[type-enum\]/.test(p) ), ux.problems.join('\n') ); // A start-case subject is refused with commitlint's reason. - const startCase = lintHeader('fix(web): Add a sentence-case subject (#8)'); + const startCase = await lintHeader( + 'fix(web): Add a sentence-case subject (#8)' + ); assert.equal(startCase.ok, false); assert.ok( - startCase.problems.some((p) => /subject-case/.test(p)), + startCase.problems.some((p) => /\[subject-case\]/.test(p)), startCase.problems.join('\n') ); + // The config's own ignores apply: a Dependabot subject is skipped (DOS/Rate#725). + assert.equal( + (await lintHeader('chore(deps): Bump foo from 1.0.0 to 1.0.1 (#9)')).ok, + true + ); }); -test('a squash header is refused, not merged, when commitlint cannot run', () => { - const missing = lintHeader('fix(web): a thing (#7)', { cwd: os.tmpdir() }); +test('a squash header is refused, not merged, when commitlint cannot run', async () => { + const missing = await lintHeader('fix(web): a thing (#7)', { + cwd: os.tmpdir(), + }); assert.equal(missing.ok, false); assert.match(missing.problems[0], /commitlint is not installed/); - const failed = lintHeader('fix(web): a thing (#7)', { - spawnFn: () => ({ error: new Error('spawn failed'), status: null }), + // commitlint installed but no commitlint.config.ts: refused, as the CLI does. + const bare = mkdtempSync(path.join(os.tmpdir(), 'mwg-no-config-')); + try { + symlinkSync( + path.join(repoRoot, 'node_modules'), + path.join(bare, 'node_modules'), + 'junction' + ); + const noRules = await lintHeader('fix(web): a thing (#7)', { cwd: bare }); + assert.equal(noRules.ok, false, noRules.problems.join('\n')); + } finally { + rmSync(bare, { recursive: true, force: true }); + } + const failed = await lintHeader('fix(web): a thing (#7)', { + spawnFn: () => { + throw new Error('spawn failed'); + }, }); assert.deepEqual(failed, { ok: false, - problems: ['commitlint could not run or timed out'], + problems: ['commitlint could not run'], + }); + const errored = await lintHeader('fix(web): a thing (#7)', { + spawnFn: () => fakeChild({ error: new Error('ENOENT') }), }); - const silent = lintHeader('fix(web): a thing (#7)', { - spawnFn: () => ({ status: 2, stdout: '', stderr: '' }), + assert.deepEqual(errored.problems, ['commitlint could not run']); + const silent = await lintHeader('fix(web): a thing (#7)', { + spawnFn: () => fakeChild({ status: 2 }), }); assert.deepEqual(silent, { ok: false, problems: ['commitlint exited 2'] }); - // A broken config says why in its first lines, and the lint is bounded in time. - const broken = lintHeader('fix(web): a thing (#7)', { - spawnFn: (_cmd, _args, opts) => { - assert.equal(opts.timeout, LINT_TIMEOUT_MS); - return { + // A broken config says why in its first lines; the header goes in on stdin. + let stdin = ''; + const broken = await lintHeader('fix(web): a thing (#7)', { + spawnFn: () => + fakeChild({ status: 9, - stdout: '', stderr: 'Error: cannot load config\n at x', - }; - }, + onInput: (text) => (stdin = text), + }), }); + assert.equal(stdin, 'fix(web): a thing (#7)'); assert.deepEqual(broken, { ok: false, problems: ['commitlint exited 9: Error: cannot load config | at x'], }); + // A commitlint that never ends is killed and fails closed (DOS/Rate#1564). + let killed = false; + const hung = await lintHeader('fix(web): a thing (#7)', { + spawnFn: () => fakeChild({ hang: true, onKill: () => (killed = true) }), + timeoutMs: 20, + }); + assert.deepEqual(hung, { + ok: false, + problems: ['commitlint timed out after 0.02 s'], + }); + assert.equal(killed, true); +}); + +/** A child process double: exits with `status`, or errors, or hangs until killed. */ +function fakeChild({ + status = 0, + stdout = '', + stderr = '', + error, + hang, + onInput, + onKill, +} = {}) { + const child = new EventEmitter(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + let input = ''; + child.stdin = new Writable({ + write(chunk, _enc, done) { + input += chunk; + done(); + }, + final(done) { + onInput?.(input); + done(); + if (hang) return; + setImmediate(() => { + if (error) return child.emit('error', error); + child.stdout.end(stdout); + child.stderr.end(stderr); + setImmediate(() => child.emit('close', status)); + }); + }, + }); + child.kill = () => onKill?.(); + return child; +} + +test('the lint timeout defaults to three minutes and takes a positive override', () => { + assert.equal(LINT_TIMEOUT_MS, 180_000); + assert.equal(lintTimeoutMs({}), LINT_TIMEOUT_MS); + assert.equal( + lintTimeoutMs({ MERGE_WHEN_GREEN_LINT_TIMEOUT_MS: '300000' }), + 300_000 + ); + for (const bad of ['0', '-5', 'abc', '1.5', '']) { + assert.equal( + lintTimeoutMs({ MERGE_WHEN_GREEN_LINT_TIMEOUT_MS: bad }), + LINT_TIMEOUT_MS, + bad + ); + } }); /** @@ -561,3 +658,43 @@ test('no branch update goes out once the deadline has passed', async () => { assert.match(result.message, /still not mergeable after/); assert.deepEqual(writes, [], 'the branch was not updated'); }); + +test('a merge waits for a slow lint, and a rejection it reports writes nothing', async () => { + // The lint resolves only after the checks are green: the merge waits for it. + let release; + const slow = new Promise((resolve) => (release = resolve)); + const { io, writes } = fakeGitHub(); + const pending = mergeWhenGreen({ + number: 41, + timeoutMs: 500, + intervalMs: 10, + lint: () => slow, + io, + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(writes, [], 'no merge before the lint has a verdict'); + release({ ok: true, problems: [] }); + assert.equal((await pending).ok, true); + // A rejected header is never merged, and a throwing lint fails closed. + for (const lint of [ + async () => ({ + ok: false, + problems: ['type must be one of [fix] [type-enum]'], + }), + () => { + throw new Error('boom'); + }, + ]) { + const rejected = fakeGitHub(); + const result = await mergeWhenGreen({ + number: 41, + timeoutMs: 500, + intervalMs: 10, + lint, + io: rejected.io, + }); + assert.equal(result.ok, false); + assert.match(result.message, /commitlint rejects the squash header/); + assert.deepEqual(rejected.writes, []); + } +}); From ac24fe5d4be6f5cf0176e071891a44411d2299e6 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:02:46 +0700 Subject: [PATCH 08/12] docs(proposals): dos-me review submission v2 rpc (p3) (#1568) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs(proposals): dos-me review submission v2 rpc (p3) The DOS-Me migration P3 needs, as a proposal: rate.submit_community_review_v2 checks a Rate review under D5 in the database (criterion stars in half steps, the required criterion, at most two left out, a 20 to 300 character verdict, one to five pros and cons, up to three per criterion, no dodging con, no line twice or in both lists) and writes rating_exact, verdict, criteria_version and the review_points rows in one transaction. rate.rating_criteria holds the versioned lists. The replay test keeps the lists, the line rules and the dodge pattern equal to @repo/shared-data. v1 stays unchanged. Nothing is applied. Refs #1544 * fix(proposals): review v2 rpc review findings Two fresh-context reviews of the P3 proposal: - Lines are stored and measured in one form (NFC, no invisible characters, trimmed), on both sides, so a stored line always fits the review_points check. - Both sides strip the same invisible set: every format character, emoji variation selectors and the keycap mark. "Không có gì ☺️" was a dodge that passed in review-form.ts, and it differed from the SQL. - The SQL helpers read text under pg_c_utf8, so they do not depend on the database locale; verify V7 checks them on the target database. - Keyword rules cover criterion lines; criterion stars are stored at 0.1. - The replay test checks the RPC's star on random sets, every invisible code point, emoji and decomposed input; verify V2 and V3 cover every helper. Refs #1544 * fix(proposals): track the review v2 proposal files past .gitignore Refs #1544 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 3 + .gitignore | 3 + ...1-dos-me-review-submission-v2-rollback.sql | 18 + ...-01-dos-me-review-submission-v2-verify.sql | 69 +++ .../2026-11-01-dos-me-review-submission-v2.md | 90 +++ ...2026-11-01-dos-me-review-submission-v2.sql | 573 ++++++++++++++++++ .../src/__tests__/shared-data.test.ts | 18 + packages/shared-data/src/rules/review-form.ts | 44 +- .../test-review-submission-v2-proposal.mjs | 524 ++++++++++++++++ 9 files changed, 1332 insertions(+), 10 deletions(-) create mode 100644 Docs/Proposals/2026-11-01-dos-me-review-submission-v2-rollback.sql create mode 100644 Docs/Proposals/2026-11-01-dos-me-review-submission-v2-verify.sql create mode 100644 Docs/Proposals/2026-11-01-dos-me-review-submission-v2.md create mode 100644 Docs/Proposals/2026-11-01-dos-me-review-submission-v2.sql create mode 100644 scripts/test-review-submission-v2-proposal.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81ceff8a..7df3dfdf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,6 +76,9 @@ jobs: - name: Replay the DOS-Me criteria ratings proposal (Docker) run: node scripts/test-criteria-ratings-proposal.mjs + - name: Replay the DOS-Me review submission v2 proposal (Docker) + run: pnpm --filter @repo/shared-data build && node scripts/test-review-submission-v2-proposal.mjs + - name: Check generated rate schema types match the migrations run: pnpm check:db-types diff --git a/.gitignore b/.gitignore index eb9fd6ef..9a949f4f 100644 --- a/.gitignore +++ b/.gitignore @@ -152,6 +152,9 @@ playwright-report-item-model/ !Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql !Docs/Proposals/2026-10-31-dos-me-criteria-ratings-rollback.sql !Docs/Proposals/2026-10-31-dos-me-criteria-ratings-verify.sql +!Docs/Proposals/2026-11-01-dos-me-review-submission-v2.sql +!Docs/Proposals/2026-11-01-dos-me-review-submission-v2-rollback.sql +!Docs/Proposals/2026-11-01-dos-me-review-submission-v2-verify.sql * - Copy* * - Bản sao* *.dump diff --git a/Docs/Proposals/2026-11-01-dos-me-review-submission-v2-rollback.sql b/Docs/Proposals/2026-11-01-dos-me-review-submission-v2-rollback.sql new file mode 100644 index 00000000..61906277 --- /dev/null +++ b/Docs/Proposals/2026-11-01-dos-me-review-submission-v2-rollback.sql @@ -0,0 +1,18 @@ +-- PROPOSAL, NOT APPLIED. Rollback of 2026-11-01-dos-me-review-submission-v2.sql (DOS/Rate#1544). +-- Drops the v2 RPC, the line helpers, the criteria table and the position key. Reviews written +-- through v2 stay, with their rating_exact, verdict, criteria_version and review_points rows +-- (P2 columns and table). Run it only while no web build calls v2 (the form falls back to v1). +-- One transaction, re-runnable. + +BEGIN; + +DROP FUNCTION IF EXISTS rate.submit_community_review_v2(uuid, jsonb, text, jsonb, jsonb, jsonb, text, text, text, text); +DROP FUNCTION IF EXISTS rate.review_point_list(jsonb, text, integer, integer, text); +DROP FUNCTION IF EXISTS rate.review_point_is_evasive_con(text); +DROP FUNCTION IF EXISTS rate.review_point_has_content(text); +DROP FUNCTION IF EXISTS rate.review_text_length(text); +DROP FUNCTION IF EXISTS rate.review_point_compare(text); +DROP INDEX IF EXISTS rate.review_points_position_key; +DROP TABLE IF EXISTS rate.rating_criteria; + +COMMIT; diff --git a/Docs/Proposals/2026-11-01-dos-me-review-submission-v2-verify.sql b/Docs/Proposals/2026-11-01-dos-me-review-submission-v2-verify.sql new file mode 100644 index 00000000..87e6f542 --- /dev/null +++ b/Docs/Proposals/2026-11-01-dos-me-review-submission-v2-verify.sql @@ -0,0 +1,69 @@ +-- PROPOSAL, NOT APPLIED. Read-only checks for 2026-11-01-dos-me-review-submission-v2.sql +-- (DOS/Rate#1544). Run after the migration on preview, beta and prod. V1 to V7 must return 0. + +-- V1 The criteria lists are complete. expect 0 +-- Version 1 has 36 rows; each category weighs 100 with one required criterion. +SELECT count(*) AS rating_criteria_incomplete +FROM (SELECT 1) one +WHERE (SELECT count(*) FROM rate.rating_criteria WHERE criteria_version = 1) <> 36 + OR EXISTS (SELECT 1 FROM rate.rating_criteria GROUP BY criteria_version, category_slug + HAVING sum(weight) <> 100 OR count(*) FILTER (WHERE required) <> 1); + +-- V2 The functions exist, the RPC is SECURITY DEFINER with an empty search_path. expect 0 +SELECT 7 - count(DISTINCT p.proname) AS functions_missing +FROM pg_proc p +WHERE p.pronamespace = 'rate'::regnamespace + AND (p.proname IN ('review_text_stored', 'review_text_length', 'review_point_compare', + 'review_point_has_content', 'review_point_is_evasive_con', 'review_point_list') + OR (p.proname = 'submit_community_review_v2' AND p.prosecdef + AND p.proconfig @> ARRAY['search_path=""'])); + +-- V3 anon cannot run the RPC or the helpers; authenticated and service_role can. expect 0 +SELECT count(*) AS grants_wrong +FROM (VALUES ('anon', false), ('authenticated', true), ('service_role', true)) AS g(role, allowed) +CROSS JOIN (VALUES + ('rate.submit_community_review_v2(uuid, jsonb, text, jsonb, jsonb, jsonb, text, text, text, text)'), + ('rate.review_point_list(jsonb, text, integer, integer, text)'), + ('rate.review_text_stored(text)'), ('rate.review_text_length(text)'), + ('rate.review_point_compare(text)'), ('rate.review_point_has_content(text)'), + ('rate.review_point_is_evasive_con(text)')) AS f(fn) +WHERE has_function_privilege(g.role, f.fn, 'EXECUTE') <> g.allowed; + +-- V4 Every v2 review (criteria_version set) has a verdict, a pro and a con. expect 0 +SELECT count(*) AS v2_reviews_incomplete +FROM rate.reviews r +WHERE r.criteria_version IS NOT NULL + AND (r.verdict IS NULL OR r.rating_exact IS NULL + OR NOT EXISTS (SELECT 1 FROM rate.review_points p + WHERE p.review_id = r.id AND p.criterion_key IS NULL AND p.kind = 'pro') + OR NOT EXISTS (SELECT 1 FROM rate.review_points p + WHERE p.review_id = r.id AND p.criterion_key IS NULL AND p.kind = 'con')); + +-- V5 Every v2 review's star is its 0.1 star rounded. expect 0 +SELECT count(*) AS v2_star_mismatch +FROM rate.reviews +WHERE criteria_version IS NOT NULL AND rating_star <> round(rating_exact); + +-- V6 The position key exists (one row per review, criterion, kind and position). expect 0 +SELECT 1 - count(*) AS position_key_missing +FROM pg_indexes +WHERE schemaname = 'rate' AND indexname = 'review_points_position_key'; + +-- V7 The line helpers read Vietnamese the same as shared-data on this database. expect 0 +-- Upper case folds, diacritics stay, a dodge is caught, emoji marks are invisible. +SELECT count(*) AS line_helpers_wrong +FROM (VALUES + (rate.review_point_compare('KHÔNG CÓ'), 'không có'), + (rate.review_point_compare('Sạc NHANH!!'), 'sạc nhanh'), + (rate.review_point_compare('Pin tốt ❤️'), 'pin tốt'), + (rate.review_point_is_evasive_con('Không có gì ☺️')::text, 'true'), + (rate.review_point_is_evasive_con('Giá hơi cao')::text, 'false'), + (rate.review_point_has_content('ưu điểm')::text, 'true'), + (rate.review_text_length('Ổn ' || chr(8203) || '😀')::text, '4')) AS t(got, want) +WHERE got IS DISTINCT FROM want; + +-- V8 For information: v2 reviews and their lines so far. +SELECT count(*) AS v2_reviews, + (SELECT count(*) FROM rate.review_points p JOIN rate.reviews r ON r.id = p.review_id + WHERE r.criteria_version IS NOT NULL) AS v2_lines +FROM rate.reviews WHERE criteria_version IS NOT NULL; diff --git a/Docs/Proposals/2026-11-01-dos-me-review-submission-v2.md b/Docs/Proposals/2026-11-01-dos-me-review-submission-v2.md new file mode 100644 index 00000000..725aebc7 --- /dev/null +++ b/Docs/Proposals/2026-11-01-dos-me-review-submission-v2.md @@ -0,0 +1,90 @@ +# DOS-Me migration proposal: review submission v2 (P3) + +**Not applied.** DOS/Rate#1544, phase P3 of the criteria ratings design (`2026-10-31-criteria-stars-pros-cons.html`, sections 2 and 4.2; D1 and D5, approved by JOY on 2026-10-31). JOY reviews it and applies it through DOS-Me: preview, then beta, then production. DOS/Rate runs no DDL against production. + +- **Needs:** P2 (`2026-10-31-dos-me-criteria-ratings.sql`, DOS/DOS.Me#1489) applied first. P3 writes P2's columns (`rating_exact`, `verdict`, `criteria_version`) and P2's `rate.review_points` table. +- **Files:** + - `2026-11-01-dos-me-review-submission-v2.sql`: the migration, one transaction, re-runnable. + - `-verify.sql`: read-only checks; V1 to V7 must return 0. + - `-rollback.sql`: drops what the migration adds, one transaction, re-runnable. +- **Replay test:** `scripts/test-review-submission-v2-proposal.mjs`, on Docker, in DOS/Rate CI. + +## What it adds + +- **`rate.rating_criteria`:** the criteria of each category, with weight, the required flag and order, versioned. Version 1 is `packages/shared-data/src/rules/rating-criteria.ts`, 36 rows. A version never changes; a new list is a new version. RLS is on, with public read. +- **Line helpers**, the SQL side of `review-form.ts`. They read text under the built-in `pg_c_utf8` collation (PostgreSQL 17), so they behave the same whatever the database's locale is: + - `rate.review_text_stored` (`storedPoint`): a line as stored, composed (NFC), without invisible characters, trimmed; + - `rate.review_point_compare` (`comparablePoint`): lower case, diacritics kept, anything but letters, digits and spaces as a space; + - `rate.review_point_has_content` (`hasContent`); + - `rate.review_point_is_evasive_con` (`isEvasiveCon`); its pattern is `EVASIVE_CON_PATTERN`, character for character; + - `rate.review_text_length`: code points of the stored form, so a stored line always fits the `review_points` length check; + - `rate.review_point_list`, which checks one list of pros or cons. +- **`rate.submit_community_review_v2(master, criteria, verdict, pros, cons, criterion_points, comment, author_name, variation, reviewer_type)`**, SECURITY DEFINER, callable by `authenticated` and `service_role`. It: + - checks the criterion stars: keys of the master's category only, 1 to 5 in half steps, the required criterion given, at most 2 left out. It stores each star at 0.1, whatever scale the caller sent; + - stores `rating_exact` as their weighted average at 0.1 (the same as `weightedReviewStar`), and `rating_star` as that value rounded; + - checks the verdict: 20 to 300 characters, with words in it; + - checks pros and cons: 1 to 5 each, 5 to 150 characters each, no line twice, no line in both lists, and no con that dodges; + - checks the lines of one criterion: up to 3 pros and 3 cons, under the same rules. As in `review-form.ts`, nothing compares a criterion's lines with the overall lists or with another criterion's, and a criterion the writer left unrated may still have lines; + - writes the review, one `rate.review_points` row per line, the author row and the excluded flag, all in one transaction. A refused call writes nothing. + - Without free text, the verdict becomes the review's `comment`, so the analyzer and v1 readers still have text. +- **`review_points_position_key`:** one row per review, criterion, kind and position. + +## Same as v1 + +- Only a signed-in, non-anonymous user or the service role can submit. +- The review goes on a crawled listing of the master. +- One review per user and master, and at most 5 reviews per user in 24 hours. v1 and v2 share the per-user lock. +- The keyword rules apply, now over the verdict and the lines too. They only exclude. +- A new review starts excluded until the analyzer decides. + +v1 stays unchanged and granted, so the current form works until the web calls v2. A later migration revokes v1. + +## Errors + +- Messages are in Vietnamese, with SQLSTATE 22023. +- The field is in `DETAIL`, so the API can mark it: + - `criteria`, or `criteria.`; + - `verdict`; + - `pros`, or `pros.`; + - `cons`, or `cons.`; + - `criterion_points...`; + - `comment`. +- The gates keep v1's codes: + - 42501: not signed in, or an anonymous account; + - 23505: the user already reviewed this master; + - 54000: more than 5 reviews in 24 hours; + - P0002: no product, or no listing. + +## Differences from the TypeScript rules + +- **Invisible characters:** both sides strip the same set: every format character (`\p{Cf}`, including the tags of flag emoji), the emoji variation selectors and the keycap mark. + - The replay test checks every such code point. + - With this PR, `review-form.ts` also strips them. Before, "Không có gì ☺️" passed as a con, because the variation selector survived. +- **Same rules on both sides:** comparison turns anything but letters, digits and spaces into a space, and lengths count code points of the stored form. +- **Known differences:** both are rare in Vietnamese reviews, and V7 checks the cases that matter. + - Case folding follows Unicode simple mapping in SQL; JavaScript gives a final sigma ("ς"). + - Some scripts' combining marks are kept by JavaScript composition but turned into spaces by the SQL class. +- **Corpus:** the replay test compares the compared form, the dodge check, the stored form and the length on a corpus of real and tricky lines: + - diacritics, dodges, upper case and fullwidth letters; + - zero-width and tag characters, variation-selector, keycap and flag emoji; + - decomposed input, and astral private-use characters. +- **Unknown category:** the RPC refuses a master whose category has no list (or no category). `getRatingCriteria` falls back to the smartphone list, so the API must check `hasRatingCriteria` before it shows the form. + +## Rollout + +1. Preview: Supabase Preview applies it on the DOS-Me PR. Then run the verify file there, read-only. +2. Beta: the native GitHub Integration applies it on merge to `dev`. Then run the verify file. +3. Production: a selective promote PR into `main`, after JOY's approval, as for P2. +4. Web (P3, #1544): the form and `POST /api/v1/rate/reviews` call v2, and map `DETAIL` to the field. +5. Rollback: `-rollback.sql`, only while no web build calls v2. Reviews written through v2 keep their columns and lines. + +## Verification + +- **V1:** 36 version 1 rows; each category weighs 100, with one required criterion. +- **V2:** the seven functions exist; the RPC is SECURITY DEFINER with an empty `search_path`. +- **V3:** `anon` cannot run the RPC or the helpers; `authenticated` and `service_role` can. +- **V4:** every v2 review has a verdict, a 0.1 star, a pro and a con. +- **V5:** every v2 review's star equals its 0.1 star rounded. +- **V6:** the position key exists. +- **V7:** the helpers read Vietnamese as shared-data does on this database: upper case, diacritics, a dodge, emoji marks, length. +- **V8** (for information): the number of v2 reviews and lines. diff --git a/Docs/Proposals/2026-11-01-dos-me-review-submission-v2.sql b/Docs/Proposals/2026-11-01-dos-me-review-submission-v2.sql new file mode 100644 index 00000000..79ae8433 --- /dev/null +++ b/Docs/Proposals/2026-11-01-dos-me-review-submission-v2.sql @@ -0,0 +1,573 @@ +-- PROPOSAL, NOT APPLIED. DOS-Me migration for phase P3 of the criteria ratings design +-- (DOS/Rate#1544; design Docs/Proposals/2026-10-31-criteria-stars-pros-cons.html, sections 2 +-- and 4.2; decisions D1, D5 approved by JOY on 2026-10-31). It needs P2 (DOS/Rate#1543, +-- DOS/DOS.Me#1489) applied first. JOY reviews and applies it through DOS-Me (preview, then beta, +-- then prod). Zero DDL on prod from DOS/Rate. +-- Rollback: 2026-11-01-dos-me-review-submission-v2-rollback.sql. +-- Verification: 2026-11-01-dos-me-review-submission-v2-verify.sql. +-- Replay test: scripts/test-review-submission-v2-proposal.mjs (Docker, never a real database). +-- +-- What it adds: +-- rate.rating_criteria the criteria of each category with weights, versioned: +-- version 1 equals packages/shared-data/src/rules/ +-- rating-criteria.ts (the replay test keeps them equal) +-- rate.review_point_compare(text) the compared form of a line (shared-data comparablePoint) +-- rate.review_point_has_content(text) at least three letters or digits, two of them different +-- rate.review_point_is_evasive_con(text) a con that only says there is none (shared-data +-- isEvasiveCon, the same pattern text) +-- rate.review_text_stored(text) a line as stored (shared-data storedPoint) +-- rate.review_text_length(text) its length in characters as the writer sees them +-- rate.submit_community_review_v2(...) a Rate review under D5: criterion stars (half steps, +-- the required criterion, at most 2 left out), the review +-- star as their weighted average at 0.1, a verdict of 20 to +-- 300 characters, 1 to 5 pros and cons of 5 to 150 +-- characters, up to 3 pros and 3 cons per criterion. It +-- stores rating_exact, verdict, criteria_version and one +-- rate.review_points row per line, in one transaction. +-- review_points_position_key one row per review, criterion, kind and position +-- +-- rate.submit_community_review (v1) is kept unchanged, so the current web form keeps working +-- until P3's web side calls v2; a later migration revokes v1. v2 has its own name rather than an +-- overload: PostgREST cannot choose between two functions of one name by their arguments. +-- +-- Same gates as v1 (20261018153000_rate_row_provenance.sql): a signed-in, non-anonymous user or +-- the service role; a crawled listing of the master; one review per user and master; 5 reviews +-- per user in 24 hours; keyword rules that exclude, never include; the review starts excluded +-- from the rating until the analyzer decides. Errors are in Vietnamese with SQLSTATE 22023 and +-- the field in DETAIL ('verdict', 'pros', 'pros.2', 'criteria.battery', +-- 'criterion_points.battery.cons.0'), so the API can point at the field. +-- +-- Idempotent: IF NOT EXISTS, ON CONFLICT DO NOTHING for the version 1 rows, CREATE OR REPLACE for +-- functions. One transaction. + +BEGIN; + +-- 1. rate.rating_criteria ----------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS rate.rating_criteria ( + criteria_version integer NOT NULL CHECK (criteria_version >= 1), + category_slug text NOT NULL CHECK (category_slug ~ '^[a-z_-]{2,64}$'), + key text NOT NULL CHECK (key ~ '^[a-z_]{2,32}$'), + weight smallint NOT NULL CHECK (weight BETWEEN 1 AND 100), + required boolean NOT NULL DEFAULT false, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 19), + PRIMARY KEY (criteria_version, category_slug, key) +); + +ALTER TABLE rate.rating_criteria ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS "Public can view rating_criteria" ON rate.rating_criteria; +CREATE POLICY "Public can view rating_criteria" ON rate.rating_criteria FOR SELECT USING (true); +GRANT SELECT ON rate.rating_criteria TO anon, authenticated, rate_worker; +GRANT ALL ON rate.rating_criteria TO service_role; + +-- A version never changes once written: a new list is a new version (RATING_CRITERIA_VERSION). +INSERT INTO rate.rating_criteria (criteria_version, category_slug, key, weight, required, position) +VALUES + (1, 'smartphones', 'performance', 20, false, 0), + (1, 'smartphones', 'camera', 20, false, 1), + (1, 'smartphones', 'battery', 20, false, 2), + (1, 'smartphones', 'display', 15, false, 3), + (1, 'smartphones', 'design', 10, false, 4), + (1, 'smartphones', 'value', 15, true, 5), + (1, 'tablets', 'display', 20, false, 0), + (1, 'tablets', 'performance', 20, false, 1), + (1, 'tablets', 'battery', 15, false, 2), + (1, 'tablets', 'sound', 10, false, 3), + (1, 'tablets', 'software', 10, false, 4), + (1, 'tablets', 'design', 10, false, 5), + (1, 'tablets', 'value', 15, true, 6), + (1, 'laptops', 'performance', 25, false, 0), + (1, 'laptops', 'display', 15, false, 1), + (1, 'laptops', 'battery', 15, false, 2), + (1, 'laptops', 'keyboard', 10, false, 3), + (1, 'laptops', 'thermals', 10, false, 4), + (1, 'laptops', 'design', 10, false, 5), + (1, 'laptops', 'value', 15, true, 6), + (1, 'smartwatches', 'health', 25, false, 0), + (1, 'smartwatches', 'battery', 20, false, 1), + (1, 'smartwatches', 'design', 15, false, 2), + (1, 'smartwatches', 'apps', 15, false, 3), + (1, 'smartwatches', 'display', 10, false, 4), + (1, 'smartwatches', 'value', 15, true, 5), + (1, 'earbuds', 'sound', 30, false, 0), + (1, 'earbuds', 'anc', 15, false, 1), + (1, 'earbuds', 'battery', 15, false, 2), + (1, 'earbuds', 'comfort', 15, false, 3), + (1, 'earbuds', 'calls', 10, false, 4), + (1, 'earbuds', 'value', 15, true, 5), + (1, 'services', 'service_quality', 35, false, 0), + (1, 'services', 'price', 25, true, 1), + (1, 'services', 'attitude', 20, false, 2), + (1, 'services', 'speed', 20, false, 3) +ON CONFLICT (criteria_version, category_slug, key) DO NOTHING; + +-- 2. One row per review, criterion, kind and position ------------------------------------------ + +CREATE UNIQUE INDEX IF NOT EXISTS review_points_position_key + ON rate.review_points (review_id, COALESCE(criterion_key, ''), kind, position); + +-- 3. Line helpers (packages/shared-data/src/rules/review-form.ts) ------------------------------ + +-- Every helper reads text under the built-in pg_c_utf8 collation (PostgreSQL 17), so lower case +-- and letter classes follow Unicode whatever the database's own locale is. The invisible set is +-- review-form.ts INVISIBLE: format characters (zero-width, bidi marks, flag tags), emoji +-- variation selectors and the keycap mark; the replay test checks every such code point. + +-- storedPoint: canonical composition, no invisible characters, trimmed. A line is stored in this +-- form and its length counted on it, so it always fits the review_points length check. +CREATE OR REPLACE FUNCTION rate.review_text_stored(p_text text) +RETURNS text +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +SET search_path = '' +AS $$ + SELECT pg_catalog.regexp_replace( + pg_catalog.regexp_replace(normalize(COALESCE(p_text, ''), NFC), + '[­؀-؅؜۝܏࢐-࢑࣢᠎​-‏‪-‮⁠-⁤⁦-⃣︀-️-\U000110BD\U000110CD\U00013430-\U0001343F\U0001BCA0-\U0001BCA3\U0001D173-\U0001D17A\U000E0001\U000E0020-\U000E007F\U000E0100-\U000E01EF]', + '', 'g') COLLATE pg_catalog.pg_c_utf8, + '^[[:space:]]+|[[:space:]]+$', '', 'g') +$$; + +-- Length in characters as the writer sees them: code points of the stored form. +CREATE OR REPLACE FUNCTION rate.review_text_length(p_text text) +RETURNS integer +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +SET search_path = '' +AS $$ + SELECT char_length(rate.review_text_stored(p_text)) +$$; + +-- comparablePoint: compatibility composition, no invisible characters, lower case, anything but +-- letters, digits and spaces as a space, spaces collapsed. Diacritics stay: "sạc" and "sắc" +-- differ. +CREATE OR REPLACE FUNCTION rate.review_point_compare(p_text text) +RETURNS text +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +SET search_path = '' +AS $$ + SELECT pg_catalog.btrim(pg_catalog.regexp_replace( + pg_catalog.regexp_replace( + pg_catalog.lower(pg_catalog.regexp_replace(normalize(COALESCE(p_text, ''), NFKC), + '[­؀-؅؜۝܏࢐-࢑࣢᠎​-‏‪-‮⁠-⁤⁦-⃣︀-️-\U000110BD\U000110CD\U00013430-\U0001343F\U0001BCA0-\U0001BCA3\U0001D173-\U0001D17A\U000E0001\U000E0020-\U000E007F\U000E0100-\U000E01EF]', + '', 'g') COLLATE pg_catalog.pg_c_utf8), + '[^[:alnum:][:space:]]+', ' ', 'g'), + '[[:space:]]+', ' ', 'g')) +$$; + +-- hasContent: at least three letters or digits, and not one character repeated. +CREATE OR REPLACE FUNCTION rate.review_point_has_content(p_text text) +RETURNS boolean +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +SET search_path = '' +AS $$ + SELECT pg_catalog.char_length(c) >= 3 + AND (SELECT count(DISTINCT ch) FROM pg_catalog.regexp_split_to_table(c, '') AS ch) >= 2 + FROM (SELECT pg_catalog.regexp_replace(rate.review_point_compare(p_text) COLLATE pg_catalog.pg_c_utf8, + '[^[:alnum:]]', '', 'g') AS c) s +$$; + +-- isEvasiveCon: no words, or the whole compared line is a dodge ("không có", "chưa thấy gì để +-- chê", "mọi thứ đều ổn", "none", "10/10"). The pattern text equals EVASIVE_CON_PATTERN. +CREATE OR REPLACE FUNCTION rate.review_point_is_evasive_con(p_text text) +RETURNS boolean +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +SET search_path = '' +AS $$ + SELECT NOT rate.review_point_has_content(p_text) + OR rate.review_point_compare(p_text) ~ '^(?:(?:hiện tại|hien tai|tạm thời|tam thoi|đến giờ|den gio|vẫn|van|thì|thi|thực sự|thuc su|thật sự|that su|mình|minh|em|tôi|toi|tui|anh|chị|chi|e|m) )*(?:không|khong|ko|k|kh|khum|chưa|chua|chẳng|chang|chả|cha|hok|hông|hong|hem)(?: (?:có|co|thấy|thay|gặp|gap|phát hiện|phat hien|phát hiện ra|phat hien ra|chê được|che duoc|chê|che))*(?: (?:gì|gi|j|nhược điểm|nhuoc diem|điểm trừ|diem tru|khuyết điểm|khuyet diem|điểm yếu|diem yeu|vấn đề|van de|lỗi|loi|điểm|diem|ý kiến|y kien))*(?:(?: (?:để|de|mà|ma|đáng|dang|phải|phai|cần|can))? (?:chê|che|phàn nàn|phan nan|nói|noi|kể|ke))?(?: (?:cả|ca|hết|het|nào|nao|luôn|luon|ạ|a|nhé|nhe|nha|nhen|đâu|dau))*$|^(?:(?:mọi thứ|moi thu|tất cả|tat ca|all) (?:(?:đều|deu) )?(?:ổn|on|tốt|tot|ok|good|fine)|(?:kcj|no|none|nothing|nothing to complain|nothing to complain about|no con|no cons|no complaint|no complaints|everything is fine|n a|0|ok|ổn|on|tốt|tot|rất tốt|rat tot|tuyệt vời|tuyet voi|hoàn hảo|hoan hao|perfect|10 10|5 sao))$' +$$; + +REVOKE ALL ON FUNCTION rate.review_text_stored(text) FROM PUBLIC, anon; +REVOKE ALL ON FUNCTION rate.review_text_length(text) FROM PUBLIC, anon; +REVOKE ALL ON FUNCTION rate.review_point_compare(text) FROM PUBLIC, anon; +REVOKE ALL ON FUNCTION rate.review_point_has_content(text) FROM PUBLIC, anon; +REVOKE ALL ON FUNCTION rate.review_point_is_evasive_con(text) FROM PUBLIC, anon; +GRANT EXECUTE ON FUNCTION rate.review_text_stored(text) TO authenticated, service_role; +GRANT EXECUTE ON FUNCTION rate.review_text_length(text) TO authenticated, service_role; +GRANT EXECUTE ON FUNCTION rate.review_point_compare(text) TO authenticated, service_role; +GRANT EXECUTE ON FUNCTION rate.review_point_has_content(text) TO authenticated, service_role; +GRANT EXECUTE ON FUNCTION rate.review_point_is_evasive_con(text) TO authenticated, service_role; + +-- 4. The lines of one list, checked and normalised ---------------------------------------------- + +-- A list of pros or cons: a JSON array of strings, each 5 to 150 characters with words in it, +-- no line twice; a con must not dodge. Returns the lines as stored (review_text_stored), in +-- order. p_field names the +-- list in errors ('pros', 'criterion_points.battery.cons'). +CREATE OR REPLACE FUNCTION rate.review_point_list( + p_lines jsonb, p_kind text, p_min integer, p_max integer, p_field text +) +RETURNS text[] +LANGUAGE plpgsql +IMMUTABLE +SET search_path = '' +AS $$ +DECLARE + v_lines text[] := '{}'; + v_seen text[] := '{}'; + v_line text; + v_compared text; + v_len integer; + v_i integer := 0; +BEGIN + IF p_lines IS NULL OR jsonb_typeof(p_lines) = 'null' THEN + p_lines := '[]'::jsonb; + END IF; + IF jsonb_typeof(p_lines) <> 'array' THEN + RAISE EXCEPTION 'Danh sách % không hợp lệ', CASE p_kind WHEN 'pro' THEN 'ưu điểm' ELSE 'nhược điểm' END + USING ERRCODE = '22023', DETAIL = p_field; + END IF; + IF jsonb_array_length(p_lines) < p_min OR jsonb_array_length(p_lines) > p_max THEN + RAISE EXCEPTION 'Cần từ % đến % %', p_min, p_max, + CASE p_kind WHEN 'pro' THEN 'ưu điểm' ELSE 'nhược điểm' END + USING ERRCODE = '22023', DETAIL = p_field; + END IF; + FOR v_i IN 0 .. jsonb_array_length(p_lines) - 1 LOOP + IF jsonb_typeof(p_lines -> v_i) <> 'string' THEN + RAISE EXCEPTION 'Mỗi dòng phải là chữ' USING ERRCODE = '22023', DETAIL = p_field || '.' || v_i; + END IF; + v_line := rate.review_text_stored(p_lines ->> v_i); + v_len := rate.review_text_length(v_line); + IF v_len < 5 OR v_len > 150 THEN + RAISE EXCEPTION 'Mỗi dòng từ 5 đến 150 ký tự' USING ERRCODE = '22023', DETAIL = p_field || '.' || v_i; + END IF; + IF p_kind = 'pro' AND NOT rate.review_point_has_content(v_line) THEN + RAISE EXCEPTION 'Dòng này chưa có nội dung' USING ERRCODE = '22023', DETAIL = p_field || '.' || v_i; + END IF; + IF p_kind = 'con' AND rate.review_point_is_evasive_con(v_line) THEN + RAISE EXCEPTION 'Hãy ghi một điểm chưa tốt cụ thể, kể cả điểm nhỏ' + USING ERRCODE = '22023', DETAIL = p_field || '.' || v_i; + END IF; + v_compared := rate.review_point_compare(v_line); + IF v_compared = ANY (v_seen) THEN + RAISE EXCEPTION 'Dòng này bị trùng' USING ERRCODE = '22023', DETAIL = p_field || '.' || v_i; + END IF; + v_seen := v_seen || v_compared; + v_lines := v_lines || v_line; + END LOOP; + RETURN v_lines; +END; +$$; + +REVOKE ALL ON FUNCTION rate.review_point_list(jsonb, text, integer, integer, text) FROM PUBLIC, anon; +GRANT EXECUTE ON FUNCTION rate.review_point_list(jsonb, text, integer, integer, text) + TO authenticated, service_role; + +-- 5. rate.submit_community_review_v2 ----------------------------------------------------------- + +CREATE OR REPLACE FUNCTION rate.submit_community_review_v2( + p_master_product_id uuid, + p_criteria_ratings jsonb, + p_verdict text, + p_pros jsonb, + p_cons jsonb, + p_criterion_points jsonb DEFAULT '{}'::jsonb, + p_comment text DEFAULT NULL, + p_author_name text DEFAULT NULL, + p_variation text DEFAULT NULL, + p_reviewer_type text DEFAULT 'user' +) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_uid uuid := auth.uid(); + v_claims jsonb := COALESCE(NULLIF(current_setting('request.jwt.claims', true), '')::jsonb, '{}'::jsonb); + v_is_service boolean; + v_category text; + v_version integer; + v_criterion record; + v_value jsonb; + v_star numeric; + v_clean_criteria jsonb := '{}'::jsonb; + v_weighted numeric := 0; + v_weights integer := 0; + v_skipped integer := 0; + v_rating_exact numeric(2, 1); + v_rating_star smallint; + v_verdict text; + v_verdict_len integer; + v_pros text[]; + v_cons text[]; + v_key text; + v_points jsonb; + v_cr_pros text[]; + v_cr_cons text[]; + v_criterion_lines jsonb := '[]'::jsonb; + v_comment text; + v_comment_len integer; + v_all_text text; + v_clean_author text; + v_clean_variation text; + v_clean_type text; + v_product_id uuid; + v_recent integer; + v_review_id uuid; + v_created_at timestamptz; + v_line_count integer := 0; + v_i integer; + v_flag_verdict text := 'unanalyzed'; + v_flag_category text := NULL; + v_reason text := 'Đánh giá cộng đồng đang chờ phân tích, chưa được tính vào điểm trung bình.'; +BEGIN + v_is_service := COALESCE(v_claims ->> 'role', '') = 'service_role'; + + -- The gates of v1: a signed-in, non-anonymous user, or the service role. + IF v_uid IS NULL AND NOT v_is_service THEN + RAISE EXCEPTION 'Bạn cần đăng nhập để gửi đánh giá' USING ERRCODE = '42501'; + END IF; + IF NOT v_is_service AND COALESCE(v_claims ->> 'is_anonymous', 'false') = 'true' THEN + RAISE EXCEPTION 'Bạn cần đăng nhập bằng tài khoản thật để gửi đánh giá' USING ERRCODE = '42501'; + END IF; + IF p_master_product_id IS NULL THEN + RAISE EXCEPTION 'master_product_id is required' USING ERRCODE = '22023', DETAIL = 'master_product_id'; + END IF; + + -- The master's category and its newest criteria list. A category with no list is refused + -- rather than rated as a phone (shared-data hasRatingCriteria). + SELECT c.slug INTO v_category + FROM rate.master_products m + LEFT JOIN rate.categories c ON c.id = m.category_id + WHERE m.id = p_master_product_id; + IF NOT FOUND THEN + RAISE EXCEPTION 'Không tìm thấy sản phẩm' USING ERRCODE = 'P0002', DETAIL = 'master_product_id'; + END IF; + SELECT max(criteria_version) INTO v_version + FROM rate.rating_criteria WHERE category_slug = v_category; + IF v_version IS NULL THEN + RAISE EXCEPTION 'Danh mục này chưa có tiêu chí chấm điểm' USING ERRCODE = '22023', DETAIL = 'criteria'; + END IF; + + -- Criterion stars (D5): keys of the list only, 1 to 5 in half steps, the required criterion + -- given, at most 2 left out. The review star is their weighted average at 0.1. + IF p_criteria_ratings IS NULL OR jsonb_typeof(p_criteria_ratings) <> 'object' THEN + RAISE EXCEPTION 'Hãy chấm điểm các tiêu chí' USING ERRCODE = '22023', DETAIL = 'criteria'; + END IF; + FOR v_key IN SELECT jsonb_object_keys(p_criteria_ratings) LOOP + IF NOT EXISTS (SELECT 1 FROM rate.rating_criteria + WHERE criteria_version = v_version AND category_slug = v_category AND key = v_key) THEN + RAISE EXCEPTION 'Tiêu chí không thuộc danh mục này' USING ERRCODE = '22023', DETAIL = 'criteria.' || left(v_key, 40); + END IF; + END LOOP; + FOR v_criterion IN + SELECT key, weight, required FROM rate.rating_criteria + WHERE criteria_version = v_version AND category_slug = v_category + ORDER BY position + LOOP + v_value := p_criteria_ratings -> v_criterion.key; + IF v_value IS NULL OR jsonb_typeof(v_value) = 'null' THEN + IF v_criterion.required THEN + RAISE EXCEPTION 'Tiêu chí này là bắt buộc' USING ERRCODE = '22023', DETAIL = 'criteria.' || v_criterion.key; + END IF; + v_skipped := v_skipped + 1; + CONTINUE; + END IF; + IF jsonb_typeof(v_value) <> 'number' THEN + RAISE EXCEPTION 'Số sao phải từ 1 đến 5, bước 0,5' USING ERRCODE = '22023', DETAIL = 'criteria.' || v_criterion.key; + END IF; + v_star := (v_value #>> '{}')::numeric; + IF v_star < 1 OR v_star > 5 OR v_star * 2 <> trunc(v_star * 2) THEN + RAISE EXCEPTION 'Số sao phải từ 1 đến 5, bước 0,5' USING ERRCODE = '22023', DETAIL = 'criteria.' || v_criterion.key; + END IF; + v_star := round(v_star, 1); + v_clean_criteria := v_clean_criteria || jsonb_build_object(v_criterion.key, v_star); + v_weighted := v_weighted + v_star * v_criterion.weight; + v_weights := v_weights + v_criterion.weight; + END LOOP; + IF v_skipped > 2 THEN + RAISE EXCEPTION 'Chỉ được bỏ qua tối đa 2 tiêu chí' USING ERRCODE = '22023', DETAIL = 'criteria'; + END IF; + v_rating_exact := round(v_weighted / v_weights, 1); + v_rating_star := round(v_rating_exact)::smallint; + + -- Verdict, 20 to 300 characters with words in it. + v_verdict := rate.review_text_stored(p_verdict); + v_verdict_len := rate.review_text_length(v_verdict); + IF v_verdict_len < 20 OR v_verdict_len > 300 THEN + RAISE EXCEPTION 'Phần kết luận cần từ 20 đến 300 ký tự' USING ERRCODE = '22023', DETAIL = 'verdict'; + END IF; + IF NOT rate.review_point_has_content(v_verdict) THEN + RAISE EXCEPTION 'Phần kết luận chưa có nội dung' USING ERRCODE = '22023', DETAIL = 'verdict'; + END IF; + + -- Pros and cons of the whole review: 1 to 5 each; no line in both lists. + v_pros := rate.review_point_list(p_pros, 'pro', 1, 5, 'pros'); + v_cons := rate.review_point_list(p_cons, 'con', 1, 5, 'cons'); + FOR v_i IN 1 .. array_length(v_cons, 1) LOOP + IF rate.review_point_compare(v_cons[v_i]) IN (SELECT rate.review_point_compare(x) FROM unnest(v_pros) x) THEN + RAISE EXCEPTION 'Một dòng không thể vừa là ưu điểm vừa là nhược điểm' + USING ERRCODE = '22023', DETAIL = 'cons.' || (v_i - 1); + END IF; + END LOOP; + + -- Pros and cons of one criterion: optional, up to 3 each, criteria of the list only. + IF p_criterion_points IS NOT NULL AND jsonb_typeof(p_criterion_points) NOT IN ('object', 'null') THEN + RAISE EXCEPTION 'Ưu nhược theo tiêu chí không hợp lệ' USING ERRCODE = '22023', DETAIL = 'criterion_points'; + END IF; + IF p_criterion_points IS NOT NULL AND jsonb_typeof(p_criterion_points) = 'object' THEN + FOR v_key, v_points IN SELECT * FROM jsonb_each(p_criterion_points) LOOP + IF NOT EXISTS (SELECT 1 FROM rate.rating_criteria + WHERE criteria_version = v_version AND category_slug = v_category AND key = v_key) THEN + RAISE EXCEPTION 'Tiêu chí không thuộc danh mục này' + USING ERRCODE = '22023', DETAIL = 'criterion_points.' || left(v_key, 40); + END IF; + IF jsonb_typeof(v_points) <> 'object' THEN + RAISE EXCEPTION 'Ưu nhược theo tiêu chí không hợp lệ' + USING ERRCODE = '22023', DETAIL = 'criterion_points.' || v_key; + END IF; + v_cr_pros := rate.review_point_list(v_points -> 'pros', 'pro', 0, 3, 'criterion_points.' || v_key || '.pros'); + v_cr_cons := rate.review_point_list(v_points -> 'cons', 'con', 0, 3, 'criterion_points.' || v_key || '.cons'); + FOR v_i IN 1 .. COALESCE(array_length(v_cr_cons, 1), 0) LOOP + IF rate.review_point_compare(v_cr_cons[v_i]) IN (SELECT rate.review_point_compare(x) FROM unnest(v_cr_pros) x) THEN + RAISE EXCEPTION 'Một dòng không thể vừa là ưu điểm vừa là nhược điểm' + USING ERRCODE = '22023', DETAIL = 'criterion_points.' || v_key || '.cons.' || (v_i - 1); + END IF; + END LOOP; + v_criterion_lines := v_criterion_lines || jsonb_build_object('key', v_key, 'pros', to_jsonb(v_cr_pros), 'cons', to_jsonb(v_cr_cons)); + END LOOP; + END IF; + + -- The free text is optional; without it the verdict is the review's comment, so readers of v1 + -- reviews and the analyzer always have text. + v_comment := rate.review_text_stored(p_comment); + v_comment_len := char_length(v_comment); + IF v_comment_len > 3000 THEN + RAISE EXCEPTION 'Nội dung đánh giá tối đa 3000 ký tự' USING ERRCODE = '22023', DETAIL = 'comment'; + END IF; + IF v_comment_len = 0 THEN + v_comment := v_verdict; + END IF; + + v_clean_author := left(btrim(COALESCE(p_author_name, '')), 80); + IF char_length(v_clean_author) < 2 THEN + v_clean_author := 'Người dùng Rate.vn'; + END IF; + v_clean_variation := NULLIF(left(btrim(COALESCE(p_variation, '')), 120), ''); + v_clean_type := CASE WHEN p_reviewer_type = 'expert' AND v_is_service THEN 'expert' ELSE 'user' END; + + -- Only a crawled listing of the master, as v1. + SELECT id INTO v_product_id + FROM rate.products + WHERE cluster_id = p_master_product_id AND origin = 'crawl' + ORDER BY price_current ASC NULLS LAST, created_at ASC + LIMIT 1; + IF v_product_id IS NULL THEN + RAISE EXCEPTION 'Không tìm thấy listing liên kết cho sản phẩm chuẩn hóa này' USING ERRCODE = 'P0002'; + END IF; + + IF v_uid IS NOT NULL THEN + -- The same lock key as v1, so v1 and v2 submissions of one user are serialized together. + PERFORM pg_advisory_xact_lock(hashtextextended('rate.submit_community_review:' || v_uid::text, 0)); + IF EXISTS ( + SELECT 1 FROM rate.review_authors a + JOIN rate.reviews r ON r.id = a.review_id + JOIN rate.products p ON p.id = r.product_id + WHERE a.author_user_id = v_uid AND p.cluster_id = p_master_product_id + ) THEN + RAISE EXCEPTION 'Bạn đã gửi đánh giá cho sản phẩm này' USING ERRCODE = '23505'; + END IF; + SELECT count(*) INTO v_recent FROM rate.review_authors + WHERE author_user_id = v_uid AND created_at > now() - interval '24 hours'; + IF v_recent >= 5 THEN + RAISE EXCEPTION 'Bạn đã gửi quá 5 đánh giá trong 24 giờ, vui lòng thử lại sau' USING ERRCODE = '54000'; + END IF; + END IF; + + -- v1's keyword rules, over every line the writer wrote. They only exclude. + v_all_text := concat_ws(E'\n', v_comment, v_verdict, array_to_string(v_pros, E'\n'), array_to_string(v_cons, E'\n'), + (SELECT string_agg(line, E'\n') FROM jsonb_array_elements(v_criterion_lines) e, + LATERAL (SELECT jsonb_array_elements_text(e -> 'pros') UNION ALL SELECT jsonb_array_elements_text(e -> 'cons')) AS t(line))); + IF v_all_text ~* '(nhận xu|kiếm xu|lấy xu|comment lấy xu|hình ảnh chỉ mang tính chất|video chỉ mang tính chất)' THEN + v_flag_verdict := 'suspicious'; + v_flag_category := 'low_value_coin_spam'; + v_reason := 'Phát hiện cụm từ bình luận cày xu mẫu, loại khỏi điểm trung bình.'; + ELSIF v_all_text ~* '(inbox zalo|liên hệ zalo|nhắn zalo|http://|https://|shopee\.ee|s\.shopee)' THEN + v_flag_verdict := 'suspicious'; + v_flag_category := 'seeding_fake'; + v_reason := 'Phát hiện đường dẫn quảng cáo hoặc điều hướng ngoài luồng, loại khỏi điểm trung bình.'; + ELSIF v_rating_star <= 2 AND v_all_text ~* '(shop lừa đảo sang shop|mua bên shop khác tốt hơn|đừng mua shop này qua shop)' THEN + v_flag_verdict := 'suspicious'; + v_flag_category := 'competitor_sabotage'; + v_reason := 'Nghi vấn điều hướng sang đối thủ, loại khỏi điểm trung bình.'; + END IF; + + INSERT INTO rate.reviews ( + product_id, platform_review_id, author_name, rating_star, rating_exact, comment, verdict, + variation, is_verified_buyer, reviewer_type, criteria_ratings, criteria_version, + upvotes, downvotes, report_count, helpfulness_weight + ) + VALUES ( + v_product_id, 'rate_comm_' || replace(gen_random_uuid()::text, '-', ''), v_clean_author, + v_rating_star, v_rating_exact, v_comment, v_verdict, v_clean_variation, NULL, v_clean_type, + v_clean_criteria, v_version, 0, 0, 0, 0.00 + ) + RETURNING id, created_at INTO v_review_id, v_created_at; + + INSERT INTO rate.review_points (review_id, criterion_key, kind, body, position, method) + SELECT v_review_id, NULL, 'pro', line, ord - 1, 'writer' FROM unnest(v_pros) WITH ORDINALITY AS t(line, ord) + UNION ALL + SELECT v_review_id, NULL, 'con', line, ord - 1, 'writer' FROM unnest(v_cons) WITH ORDINALITY AS t(line, ord) + UNION ALL + SELECT v_review_id, e ->> 'key', 'pro', line, ord - 1, 'writer' + FROM jsonb_array_elements(v_criterion_lines) e, + jsonb_array_elements_text(e -> 'pros') WITH ORDINALITY AS t(line, ord) + UNION ALL + SELECT v_review_id, e ->> 'key', 'con', line, ord - 1, 'writer' + FROM jsonb_array_elements(v_criterion_lines) e, + jsonb_array_elements_text(e -> 'cons') WITH ORDINALITY AS t(line, ord); + GET DIAGNOSTICS v_line_count = ROW_COUNT; + + IF v_uid IS NOT NULL THEN + INSERT INTO rate.review_authors (review_id, author_user_id) VALUES (v_review_id, v_uid); + END IF; + + INSERT INTO rate.review_flags ( + review_id, verdict, fake_probability, flag_category, include_in_rating, reason, signals, + detector, detector_version, input_hash + ) + VALUES ( + v_review_id, v_flag_verdict, NULL, v_flag_category, false, v_reason, '{}'::jsonb, + 'rate-community-guard', 'v3', + rate.review_input_hash(v_clean_author, v_rating_star, v_comment) + ); + + -- No rating refresh: an excluded review cannot change the rating. + RETURN jsonb_build_object( + 'id', v_review_id, + 'product_id', v_product_id, + 'author_name', v_clean_author, + 'rating_star', v_rating_star, + 'rating_exact', v_rating_exact, + 'criteria_ratings', v_clean_criteria, + 'criteria_version', v_version, + 'verdict', v_verdict, + 'comment', v_comment, + 'variation', v_clean_variation, + 'reviewer_type', v_clean_type, + 'points', v_line_count, + 'flag_verdict', v_flag_verdict, + 'flag_category', v_flag_category, + 'include_in_rating', false, + 'reason', v_reason, + 'created_at', v_created_at + ); +END; +$$; + +REVOKE ALL ON FUNCTION rate.submit_community_review_v2(uuid, jsonb, text, jsonb, jsonb, jsonb, text, text, text, text) + FROM PUBLIC, anon; +GRANT EXECUTE ON FUNCTION rate.submit_community_review_v2(uuid, jsonb, text, jsonb, jsonb, jsonb, text, text, text, text) + TO authenticated, service_role; + +COMMIT; diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index 358135c6..0a072de8 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -1707,6 +1707,24 @@ function testRatingCriteriaAndReviewForm(): void { const started = Date.now(); assert.ok(!rules.isEvasiveCon('khong ' + 'gi '.repeat(50_000) + 'x')); assert.ok(Date.now() - started < 1000, 'linear on an oversized con'); + // Emoji marks and flag tags are invisible: "☺️" leaves no mark behind, so + // the dodge is caught, and "Pin tốt ❤️" is "Pin tốt" twice. + assert.ok(rules.isEvasiveCon('Không có gì ☺️')); + assert.ok(rules.isEvasiveCon('10/10 ❤️')); + assert.equal(rules.comparablePoint('Pin tốt ❤️'), 'pin tốt'); + assert.equal(rules.storedPoint(' Loa​ to 🏴󠁧󠁢󠁳󠁣󠁴󠁿 '), 'Loa to 🏴'); + // A line is measured as stored: 150 decomposed letters are 150 letters. + const decomposed = 'ướ'.repeat(75).normalize('NFD'); + assert.deepEqual( + rules + .validateReviewText({ + verdict: 'Máy tốt, pin trâu, camera chụp đêm ổn định.', + pros: [decomposed], + cons: ['Giá hơi cao so với đối thủ'], + }) + .filter((issue) => issue.kind === 'point_length'), + [] + ); } function testProductPoints(): void { diff --git a/packages/shared-data/src/rules/review-form.ts b/packages/shared-data/src/rules/review-form.ts index 9576d9ef..96213055 100644 --- a/packages/shared-data/src/rules/review-form.ts +++ b/packages/shared-data/src/rules/review-form.ts @@ -33,35 +33,52 @@ const EMPTY = ''; // Compatibility composition: fullwidth letters ("ＫＨÔＮＧ") fold to plain // ones and Vietnamese letters stay whole. const COMPOSED = 'NFKC'; +// Canonical composition: the text as stored and measured. +const CANONICAL = 'NFC'; // Compatibility decomposition, so diacritics split off as marks. const DECOMPOSED = 'NFKD'; +// Invisible characters: format characters (zero-width, bidi marks, the tags +// of flag emoji), emoji variation selectors and the keycap mark. The DOS-Me +// review RPC strips the same set (DOS/Rate#1544). +const INVISIBLE = /[\p{Cf}︀-️⃣\u{E0100}-\u{E01EF}]+/gu; /** - * The form two lines are compared in: lower case, no zero-width characters, - * punctuation and symbols as spaces, spaces collapsed. Diacritics stay - * ("sạc" and "sắc" differ, design 5.1). Only for comparing lines; what is - * stored and shown keeps the writer's text. + * The form two lines are compared in: lower case, no invisible characters, + * anything but letters, digits and spaces as a space, spaces collapsed. + * Diacritics stay ("sạc" and "sắc" differ, design 5.1). Only for comparing + * lines; what is stored and shown is storedPoint. */ export function comparablePoint(text: string): string { const composed = String(text ?? EMPTY).normalize(COMPOSED); return composed - .replace(/\p{Cf}+/gu, '') + .replace(INVISIBLE, '') .toLowerCase() - .replace(/[\p{P}\p{S}]+/gu, ' ') + .replace(/[^\p{L}\p{N}\s]+/gu, ' ') .replace(/\s+/g, ' ') .trim(); } +/** + * A line as stored: the writer's words, composed, without invisible + * characters, trimmed. Lengths are counted on this form, so a stored line + * always fits the table's length check. + */ +export function storedPoint(text: string): string { + return String(text ?? EMPTY) + .normalize(CANONICAL) + .replace(INVISIBLE, EMPTY) + .trim(); +} + /** A compared line without diacritics, "đ" as "d": for spotting dodges. */ function plainPoint(text: string): string { const decomposed = comparablePoint(text).normalize(DECOMPOSED); return decomposed.replace(/\p{M}+/gu, '').replace(/đ/g, 'd'); } -/** Length in characters as the writer sees them (code points, composed). */ +/** Length in characters as the writer sees them (code points of storedPoint). */ function visibleLength(text: string): number { - const composed = String(text ?? EMPTY).normalize(COMPOSED); - return [...composed.replace(/\p{Cf}+/gu, '').trim()].length; + return [...storedPoint(text)].length; } /** @@ -110,6 +127,13 @@ const EVASIVE_CON = new RegExp( ].join('') ); +/** + * The dodge pattern's source, matched against comparablePoint(line). The + * DOS-Me review RPC (rate.submit_community_review_v2) carries the same + * pattern; DOS/Rate's proposal test keeps the two equal. + */ +export const EVASIVE_CON_PATTERN: string = EVASIVE_CON.source; + /** * Whether a con line says only that there is no con. A dodge is short: a * line longer than a con may be is never one, which also keeps the check @@ -117,7 +141,7 @@ const EVASIVE_CON = new RegExp( */ export function isEvasiveCon(text: string): boolean { const compared = comparablePoint(text); - if (compared.length > REVIEW_POINT_MAX) return false; + if ([...compared].length > REVIEW_POINT_MAX) return false; return !hasContent(text) || EVASIVE_CON.test(compared); } diff --git a/scripts/test-review-submission-v2-proposal.mjs b/scripts/test-review-submission-v2-proposal.mjs new file mode 100644 index 00000000..391318a4 --- /dev/null +++ b/scripts/test-review-submission-v2-proposal.mjs @@ -0,0 +1,524 @@ +#!/usr/bin/env node +// Replays the DOS-Me review submission v2 PROPOSAL (DOS/Rate#1544, phase P3 of #1531) on the +// migration replay stack with the P2 proposal applied: the proposal twice, the criteria lists +// and the line rules against @repo/shared-data (the same lists, the same dodge pattern, the same +// verdicts on a corpus of lines, the same weighted star), a v2 review end to end, every refusal +// with its field, the v1 gates, the grants, every verification query, and the rollback twice. +// Nothing touches a real database. +// +// Usage: pnpm --filter @repo/shared-data build && node scripts/test-review-submission-v2-proposal.mjs +// (needs Docker) + +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; + +import { repoRoot, startRateDb } from './lib/rate-db.mjs'; + +const rules = await import( + pathToFileURL( + path.join(repoRoot, 'packages', 'shared-data', 'dist', 'rules', 'index.js') + ).href +); +const read = (name) => + readFileSync(path.join(repoRoot, 'Docs', 'Proposals', name), 'utf8'); +const p2 = read('2026-10-31-dos-me-criteria-ratings.sql'); +const proposal = read('2026-11-01-dos-me-review-submission-v2.sql'); +const rollback = read('2026-11-01-dos-me-review-submission-v2-rollback.sql'); +const verify = read('2026-11-01-dos-me-review-submission-v2-verify.sql'); + +const container = `rate-review-v2-proposal-${process.pid}`; +const removeContainer = () => + spawnSync('docker', ['rm', '--force', container], { timeout: 30_000 }); +for (const signal of ['SIGINT', 'SIGTERM']) { + process.on(signal, () => { + removeContainer(); + process.exit(130); + }); +} + +const checks = [ + ...verify.matchAll( + /^-- (V\d+) [^\n]*expect 0\n(?:--[^\n]*\n)*(SELECT[\s\S]*?;)/gm + ), +].map(([, id, query]) => ({ id, query })); +assert.equal(checks.length, 7, 'the verification lists its expect-0 queries'); + +// The proposal carries the dodge pattern of shared-data, character for character. +const patternInSql = proposal.match( + /review_point_compare\(p_text\) ~ '((?:[^']|'')*)'/ +)?.[1]; +assert.equal( + patternInSql?.replaceAll("''", "'"), + rules.EVASIVE_CON_PATTERN, + 'rate.review_point_is_evasive_con uses EVASIVE_CON_PATTERN' +); + +const PHONES = 'c0000000-0000-4000-8000-000000000001'; +const EARBUDS = 'c0000000-0000-4000-8000-000000000002'; +const GADGETS = 'c0000000-0000-4000-8000-000000000003'; +const MERCHANT = 'e0000000-0000-4000-8000-000000000001'; +const phone = 'a0000000-0000-4000-8000-000000000001'; +const phone2 = 'a0000000-0000-4000-8000-000000000002'; +const buds = 'a0000000-0000-4000-8000-000000000003'; +const gadget = 'a0000000-0000-4000-8000-000000000004'; +const noListing = 'a0000000-0000-4000-8000-000000000005'; +const userA = 'f0000000-0000-4000-8000-00000000000a'; +const userB = 'f0000000-0000-4000-8000-00000000000b'; +const RPC = + 'rate.submit_community_review_v2(uuid, jsonb, text, jsonb, jsonb, jsonb, text, text, text, text)'; + +const q = (value) => `'${String(value).replaceAll("'", "''")}'`; +const j = (value) => `${q(JSON.stringify(value))}::jsonb`; + +try { + const { sql, asRole } = startRateDb({ name: container }); + const checkAll = (when) => { + for (const { id, query } of checks) + assert.equal(sql(query), '0', `${id} ${when}`); + }; + sql(p2); + sql(proposal); + sql(proposal); + + // 1. The criteria lists equal rating-criteria.ts. + const stored = sql( + `SELECT string_agg(category_slug || ':' || key || ':' || weight || ':' || required || ':' || position, + ',' ORDER BY category_slug, position) FROM rate.rating_criteria WHERE criteria_version = ${rules.RATING_CRITERIA_VERSION}` + ); + const expected = rules.RATING_CRITERIA_CATEGORIES.flatMap((slug) => + rules + .getRatingCriteria(slug) + .map((c, i) => `${slug}:${c.key}:${c.weight}:${c.required}:${i}`) + ) + .sort((a, b) => { + const [sa, , , , pa] = a.split(':'); + const [sb, , , , pb] = b.split(':'); + return sa < sb ? -1 : sa > sb ? 1 : Number(pa) - Number(pb); + }) + .join(','); + assert.equal( + stored, + expected, + 'rate.rating_criteria equals rating-criteria.ts' + ); + + // 2. The line rules agree with review-form.ts on a corpus of real and tricky lines. + const corpus = [ + 'Pin trâu, dùng cả ngày', + 'không có', + 'Không có nhược điểm nào cả', + 'khong co gi de che', + 'chưa thấy ạ', + 'mình không thấy gì để chê', + 'Chả có gì', + 'khum có', + 'không chê được gì', + 'chưa gặp vấn đề gì', + 'không có ý kiến', + 'Mọi thứ đều ổn', + 'none', + '10/10', + 'ok', + 'giá hơi cao', + 'không có jack 3.5mm', + 'chưa có gì nổi bật', + 'hỏng luôn', + 'aaaaa', + '.....', + '🔥🔥🔥🔥🔥', + 'ＫＨÔＮＧ CÓ', + 'không​ có', + 'Sạc nhanh!!!', + 'sắc nét', + 'Không có nhược điểm', + 'Tất cả đều tốt', + 'nothing to complain about', + 'Camera chụp đêm hơi nhiễu', + 'KHÔNG CÓ', + 'Không có gì ☺️', + 'không có gì ❤️', + '10/10 ❤️', + 'Pin tốt ❤️', + '1️⃣ màn hình hơi ám vàng', + 'Loa to 🏴󠁧󠁢󠁳󠁣󠁴󠁿 rõ', + 'khô\u{E0020}ng có', + 'a̲b̲c̲ d', + 'Ổn định
lắm', + '\u{F0000}'.repeat(76), + 'ước mơ'.normalize('NFD'), + ]; + for (const line of corpus) { + const row = sql( + `SELECT rate.review_point_compare(${q(line)}) || '|' || rate.review_point_is_evasive_con(${q(line)}) + || '|' || rate.review_text_stored(${q(line)}) || '|' || rate.review_text_length(${q(line)})` + ); + assert.equal( + row, + `${rules.comparablePoint(line)}|${rules.isEvasiveCon(line)}|${rules.storedPoint(line)}|${[...rules.storedPoint(line)].length}`, + `compare, dodge and stored form for ${JSON.stringify(line)}` + ); + } + // Every invisible code point of review-form.ts is invisible to the SQL helpers too. + const invisible = []; + for (let c = 0; c < 0x110000; c += 1) + if ( + (c < 0xd800 || c > 0xdfff) && + /[\p{Cf}︀-️⃣\u{E0100}-\u{E01EF}]/u.test(String.fromCodePoint(c)) + ) + invisible.push(String.fromCodePoint(c)); + assert.ok(invisible.length > 300, 'the invisible set is the full one'); + assert.equal( + sql( + `SELECT rate.review_text_stored(${q(`a${invisible.join('')}b`)}) || '|' || rate.review_point_compare(${q(`A${invisible.join('')}B`)})` + ), + 'ab|ab' + ); + assert.notEqual( + sql(`SELECT rate.review_point_compare('sạc')`), + sql(`SELECT rate.review_point_compare('sắc')`), + 'diacritics stay' + ); + + // 3. The weighted star equals weightedReviewStar on many random sets. + sql(` + INSERT INTO rate.categories (id, name, slug) VALUES + ('${PHONES}', 'Điện thoại', 'smartphones'), + ('${EARBUDS}', 'Tai nghe', 'earbuds'), + ('${GADGETS}', 'Đồ chơi', 'gadgets'); + INSERT INTO rate.merchants (id, platform, platform_shop_id, name, slug) + VALUES ('${MERCHANT}', 'shopee', 's1', 'Shop 1', 'shop-1'); + INSERT INTO rate.master_products (id, brand, model, model_key, name, slug, category_id, status) + VALUES + ('${phone}', 'Acme', 'Phone X', 'acme:phone-x', 'Acme Phone X', 'acme-phone-x', '${PHONES}', 'published'), + ('${phone2}', 'Acme', 'Phone Y', 'acme:phone-y', 'Acme Phone Y', 'acme-phone-y', '${PHONES}', 'published'), + ('${buds}', 'Acme', 'Buds', 'acme:buds', 'Acme Buds', 'acme-buds', '${EARBUDS}', 'published'), + ('${gadget}', 'Acme', 'Toy', 'acme:toy', 'Acme Toy', 'acme-toy', '${GADGETS}', 'published'), + ('${noListing}', 'Acme', 'Phone Z', 'acme:phone-z', 'Acme Phone Z', 'acme-phone-z', '${PHONES}', 'published'); + INSERT INTO rate.products (id, merchant_id, platform, platform_item_id, title, price_current, url, cluster_id) + VALUES + ('b0000000-0000-4000-8000-000000000001', '${MERCHANT}', 'shopee', 'p1', 'Acme Phone X', 10000000, 'https://shopee.vn/p1', '${phone}'), + ('b0000000-0000-4000-8000-000000000002', '${MERCHANT}', 'shopee', 'p2', 'Acme Phone Y', 9000000, 'https://shopee.vn/p2', '${phone2}'), + ('b0000000-0000-4000-8000-000000000003', '${MERCHANT}', 'shopee', 'p3', 'Acme Buds', 1000000, 'https://shopee.vn/p3', '${buds}'), + ('b0000000-0000-4000-8000-000000000004', '${MERCHANT}', 'shopee', 'p4', 'Acme Toy', 100000, 'https://shopee.vn/p4', '${gadget}'); + INSERT INTO auth.users (id, email, email_confirmed_at, created_at) VALUES + ('${userA}', 'a@example.test', now(), now() - interval '30 days'), + ('${userB}', 'b@example.test', now(), now() - interval '30 days'); + `); + const call = (claims, args) => + sql(`SET ROLE authenticated; SET request.jwt.claims = ${q(JSON.stringify(claims))}; + SELECT rate.submit_community_review_v2(${args})::text; RESET ROLE;`); + const asA = { sub: userA, role: 'authenticated' }; + const asB = { sub: userB, role: 'authenticated' }; + const good = { + criteria: { + performance: 5, + camera: 4, + battery: 3, + display: 5, + design: 4, + value: 4, + }, + verdict: 'Máy mạnh, camera ổn, pin chỉ vừa đủ một ngày.', + pros: ['Hiệu năng mạnh, chơi game mượt', 'Màn hình sáng rõ ngoài nắng'], + cons: ['Pin chỉ đủ một ngày dùng vừa'], + points: { + battery: { + pros: ['Sạc nhanh 30 phút được 60%'], + cons: ['Máy hơi nóng khi sạc'], + }, + }, + }; + const args = (r, master = phone) => + [ + q(master), + j(r.criteria), + q(r.verdict), + j(r.pros), + j(r.cons), + j(r.points ?? {}), + ].join(', '); + const fails = (claims, r, pattern, detail, master = phone) => { + assert.throws( + () => call(claims, args(r, master)), + (err) => { + assert.match(err.message, pattern); + if (detail) + assert.match( + err.message, + new RegExp(`DETAIL:\\s+${detail.replaceAll('.', '\\.')}`) + ); + return true; + }, + `${pattern} ${detail ?? ''}` + ); + }; + + // Refusals, each with its field. Nothing is written by a refused call. + fails( + asA, + { ...good, criteria: { ...good.criteria, value: undefined } }, + /bắt buộc/, + 'criteria.value' + ); + fails( + asA, + { ...good, criteria: { value: 4, performance: 5, camera: 4 } }, + /tối đa 2 tiêu chí/, + 'criteria' + ); + fails( + asA, + { ...good, criteria: { ...good.criteria, camera: 4.3 } }, + /bước 0,5/, + 'criteria.camera' + ); + fails( + asA, + { ...good, criteria: { ...good.criteria, camera: '4' } }, + /bước 0,5/, + 'criteria.camera' + ); + fails( + asA, + { ...good, criteria: { ...good.criteria, anc: 4 } }, + /không thuộc danh mục/, + 'criteria.anc' + ); + fails(asA, { ...good, verdict: 'Ngắn quá' }, /20 đến 300/, 'verdict'); + fails(asA, { ...good, pros: [] }, /Cần từ 1 đến 5 ưu điểm/, 'pros'); + fails( + asA, + { + ...good, + cons: Array(6) + .fill(0) + .map((_, i) => `Nhược điểm số ${i}`), + }, + /Cần từ 1 đến 5 nhược điểm/, + 'cons' + ); + fails( + asA, + { ...good, cons: ['Không có nhược điểm nào cả'] }, + /cụ thể/, + 'cons.0' + ); + fails( + asA, + { ...good, pros: ['Pin trâu', 'Màn hình đẹp', 'pin trâu!'] }, + /trùng/, + 'pros.2' + ); + fails(asA, { ...good, pros: ['Tốt'] }, /5 đến 150/, 'pros.0'); + fails(asA, { ...good, pros: ['aaaaaaa'] }, /chưa có nội dung/, 'pros.0'); + fails( + asA, + { ...good, cons: ['Màn hình sáng rõ ngoài nắng!'] }, + /vừa là ưu điểm vừa là nhược điểm/, + 'cons.0' + ); + fails( + asA, + { ...good, points: { anc: { pros: ['Chống ồn tốt lắm'] } } }, + /không thuộc danh mục/, + 'criterion_points.anc' + ); + fails( + asA, + { + ...good, + points: { + battery: { + cons: ['a1', 'b', 'c', 'd'].map((x) => `Pin tụt nhanh ${x}`), + }, + }, + }, + /Cần từ 0 đến 3 nhược điểm/, + 'criterion_points.battery.cons' + ); + fails( + asA, + { ...good, points: { battery: { cons: ['chưa thấy gì'] } } }, + /cụ thể/, + 'criterion_points.battery.cons.0' + ); + fails(asA, good, /chưa có tiêu chí/, 'criteria', gadget); + fails(asA, good, /Không tìm thấy listing/, null, noListing); + fails({ role: 'anon' }, good, /đăng nhập/, null); + fails( + { sub: userA, role: 'authenticated', is_anonymous: true }, + good, + /tài khoản thật/, + null + ); + assert.equal( + sql(`SELECT count(*) FROM rate.reviews`), + '0', + 'refused calls write nothing' + ); + assert.throws( + () => asRole('anon', `SELECT ${RPC.replace(/\(.*\)/, '')}(${args(good)})`), + /permission denied/, + 'anon cannot call v2' + ); + + // A good review: the star, the 0.1 star, the verdict, the version and every line. + const result = JSON.parse(call(asA, args(good))); + const exact = rules.weightedReviewStar('smartphones', good.criteria); + assert.equal(Number(result.rating_exact), exact); + assert.equal(result.rating_star, Math.round(exact)); + assert.equal(result.criteria_version, rules.RATING_CRITERIA_VERSION); + assert.equal(result.points, 5); + assert.equal(result.include_in_rating, false); + assert.equal( + sql(`SELECT string_agg(coalesce(criterion_key, '-') || ':' || kind || ':' || position || ':' || body, ' / ' + ORDER BY criterion_key NULLS FIRST, kind DESC, position) + FROM rate.review_points WHERE review_id = '${result.id}'`), + '-:pro:0:Hiệu năng mạnh, chơi game mượt / -:pro:1:Màn hình sáng rõ ngoài nắng / -:con:0:Pin chỉ đủ một ngày dùng vừa / battery:pro:0:Sạc nhanh 30 phút được 60% / battery:con:0:Máy hơi nóng khi sạc' + ); + assert.equal( + sql(`SELECT comment = verdict FROM rate.reviews WHERE id = '${result.id}'`), + 't', + 'without free text the verdict is the comment' + ); + // One review per user and master; v1 and v2 share that rule. + fails(asA, good, /đã gửi đánh giá/, null); + // The RPC's star equals weightedReviewStar on random earbud sets. The service role has no + // per-user limit, so it submits them all to one master. + const asService = { role: 'service_role' }; + let seed = 7; + const next = () => (seed = (seed * 48271) % 2147483647) / 2147483647; + let checked = 0; + for (let n = 0; n < 30; n += 1) { + const criteria = {}; + for (const c of rules.getRatingCriteria('earbuds')) + if (c.required || next() > 0.3) + criteria[c.key] = 1 + Math.round(next() * 8) / 2; + const skipped = + rules.getRatingCriteria('earbuds').length - Object.keys(criteria).length; + if (skipped > rules.MAX_SKIPPED_CRITERIA) continue; + const r = JSON.parse( + call(asService, args({ ...good, criteria, points: {} }, buds)) + ); + const exact = rules.weightedReviewStar('earbuds', criteria); + assert.equal(Number(r.rating_exact), exact, JSON.stringify(criteria)); + assert.equal(r.rating_star, Math.round(exact), JSON.stringify(criteria)); + checked += 1; + } + assert.ok(checked >= 10, `${checked} random sets went through the RPC`); + // A stored line is counted as stored: 150 decomposed letters are 150 composed ones and fit + // the review_points length check. + const decomposed = 'ướ'.repeat(75).normalize('NFD'); + assert.ok(decomposed.length > 150); + const long = JSON.parse( + call( + asService, + args( + { + ...good, + criteria: { sound: 4, anc: 4, battery: 4, comfort: 4, value: 4 }, + pros: [decomposed], + points: {}, + }, + buds + ) + ) + ); + assert.equal( + sql( + `SELECT char_length(body) FROM rate.review_points WHERE review_id = '${long.id}' AND kind = 'pro'` + ), + '150' + ); + // A criterion star is stored at 0.1, whatever scale the caller sent. + const scaled = JSON.parse( + call( + asService, + `${q(buds)}, '{"sound": 4.50000, "anc": 3.0, "battery": 4, "comfort": 5, "value": 4}'::jsonb, ${q(good.verdict)}, ${j(good.pros)}, ${j(good.cons)}, '{}'::jsonb` + ) + ); + assert.equal(JSON.stringify(scaled.criteria_ratings.sound), '4.5'); + const budsReview = JSON.parse( + call( + asB, + args( + { + ...good, + criteria: { sound: 4.5, anc: 3, battery: 4, comfort: 5, value: 4 }, + points: {}, + }, + buds + ) + ) + ); + assert.equal( + Number(budsReview.rating_exact), + rules.weightedReviewStar('earbuds', { + sound: 4.5, + anc: 3, + battery: 4, + comfort: 5, + value: 4, + }) + ); + // Keyword rules still exclude, now over the lines too. + const spam = JSON.parse( + call( + asB, + args({ ...good, pros: ['Inbox zalo để được giá tốt hơn'] }, phone2) + ) + ); + assert.equal(spam.flag_category, 'seeding_fake'); + const criterionSpam = JSON.parse( + call( + asService, + args( + { + ...good, + points: { battery: { pros: ['Nhắn zalo để mua pin giá rẻ'] } }, + }, + phone2 + ) + ) + ); + assert.equal( + criterionSpam.flag_category, + 'seeding_fake', + 'a criterion line is checked too' + ); + checkAll('after v2 reviews'); + const linesBefore = sql(`SELECT count(*) FROM rate.review_points`); + + // 4. The rollback twice; reviews and their lines stay; the proposal applies again. + sql(rollback); + sql(rollback); + assert.equal( + sql(`SELECT count(*) FROM pg_proc WHERE pronamespace = 'rate'::regnamespace + AND proname IN ('submit_community_review_v2', 'review_point_list', 'review_point_compare')`), + '0' + ); + assert.equal(sql(`SELECT to_regclass('rate.rating_criteria') IS NULL`), 't'); + assert.equal( + sql(`SELECT count(*) FROM rate.review_points`), + linesBefore, + 'lines stay' + ); + assert.equal( + sql( + `SELECT has_function_privilege('authenticated', 'rate.submit_community_review(uuid, text, integer, text, text, text, jsonb)', 'EXECUTE')` + ), + 't', + 'v1 is untouched' + ); + sql(proposal); + checkAll('after the rollback and a re-apply'); + + console.log( + 'review submission v2 proposal: shared-data parity, refusals, a review end to end, verify and rollback replayed' + ); +} finally { + removeContainer(); +} From 546d68f93fa3a31525d870c8bea515b0fa49107c Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:05:13 +0700 Subject: [PATCH 09/12] feat(web): review form v2 with criteria, verdict and pros and cons behind a flag (#1567) * feat(web): review form v2 with criteria, verdict and pros and cons behind a flag Refs #1544. The DB-free part of P3 (design #1531, D5): per-category criteria in half stars with the weighted star beside them, a 20-300 character verdict, 1-5 pros and cons, optional pros and cons per rated criterion, and a browser draft. The form and POST /api/v1/rate/reviews check a review with the same reviewV2Errors, per field. Behind NEXT_PUBLIC_RATE_REVIEW_FORM_V2=1, off by default: with the flag off the form and the API take the current path unchanged. The API reads the product's category on the server and computes the star itself; the v2 RPC does not exist yet (DOS-Me), so submitCommunityReviewV2 is a stub that never writes and the route answers 503 REVIEWS_UNAVAILABLE. * fix(web): review findings on form v2 errors, drafts and the v2 api path Refs #1544. Errors land on the line they name (blank rows are skipped in the count), the form is keyed per product so a draft never moves, stored stars outside half steps are dropped, and the pickers carry aria-required. The API refuses v1 bodies once v2 is on, keeps pros and cons on one line, checks a criterion's pros against its cons, rejects an empty criterion entry, and builds the submission in buildReviewV2Submission so a test sees the computed star. The stub takes the user's client for the real call. * fix(web): name the duplicate-line message key so gitleaks stops flagging it Refs #1544. gitleaks read `key: 'v2ErrDuplicate'` as a generic API key: the rules now use a DUPLICATE_LINE constant, and the two findings in 2f624a1c are listed in .gitleaksignore with the reason. * fix(web): keep the duplicate-line constant clear of the secret scan Refs #1544. The constant's own comment and typed declaration still read as a generic API key. The comment drops the literal and the declaration uses `as const`; the two findings in 4c050605 join .gitleaksignore. Co-Authored-By: Claude Opus 5.5 --- .gitleaksignore | 7 + apps/web/locales/cs.json | 29 +- apps/web/locales/en.json | 29 +- apps/web/locales/vi.json | 29 +- apps/web/package.json | 2 +- .../api/v1/rate/__tests__/write-paths.test.ts | 149 ++++ apps/web/src/app/api/v1/rate/reviews/route.ts | 128 ++++ .../components/reviews/JReviewsSection.tsx | 82 ++- .../src/components/reviews/ReviewFormV2.tsx | 680 ++++++++++++++++++ .../src/lib/__tests__/review-display.test.ts | 1 + .../src/lib/__tests__/review-form-v2.test.ts | 205 ++++++ apps/web/src/lib/rate-api.ts | 30 + apps/web/src/lib/review-display.ts | 3 + apps/web/src/lib/review-form-v2.ts | 342 +++++++++ apps/web/src/server/rate/index.ts | 24 + apps/web/src/server/rate/review-v2.ts | 99 +++ 16 files changed, 1803 insertions(+), 36 deletions(-) create mode 100644 apps/web/src/components/reviews/ReviewFormV2.tsx create mode 100644 apps/web/src/lib/__tests__/review-form-v2.test.ts create mode 100644 apps/web/src/lib/review-form-v2.ts create mode 100644 apps/web/src/server/rate/review-v2.ts diff --git a/.gitleaksignore b/.gitleaksignore index d4273ba7..a74366ea 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -4,3 +4,10 @@ # Fake 'sk-123...' key in the logger redaction test. The line carries gitleaks:allow # today; this commit (#260) added it before the inline allow comment existed. 02a0187ca6b96e11476e6bf90921aabf53b1f7a5:apps/importer/src/shopee/__tests__/entity-matcher.test.ts:generic-api-key:453 + +# The message key 'v2ErrDuplicate' in `key: '...'` literals of the review form v2 +# rules (#1544), read as a generic API key. Later commits use a named constant. +2f624a1c620f68ecb658eed02aba62b05f05b04b:apps/web/src/lib/review-form-v2.ts:generic-api-key:114 +2f624a1c620f68ecb658eed02aba62b05f05b04b:apps/web/src/lib/review-form-v2.ts:generic-api-key:193 +4c0506053bbbc6668df016cdebfd3ee37fc9cea8:apps/web/src/lib/review-form-v2.ts:generic-api-key:75 +4c0506053bbbc6668df016cdebfd3ee37fc9cea8:apps/web/src/lib/review-form-v2.ts:generic-api-key:76 diff --git a/apps/web/locales/cs.json b/apps/web/locales/cs.json index c791a0c6..3131a63e 100644 --- a/apps/web/locales/cs.json +++ b/apps/web/locales/cs.json @@ -579,7 +579,34 @@ "reportTitle": "Nahlásit falešnou, spamovou nebo sabotážní recenzi (každé nahlášení sníží váhu o 0,35)", "reportBtnLabel": "Nahlásit ({count})", "filterGroupAria": "Filtrovat recenze", - "exportMenu": "Exportovat data" + "exportMenu": "Exportovat data", + "v2FormHeading": "Recenze {productName}", + "v2CriteriaLabel": "Ohodnoťte každé kritérium", + "v2Skip": "Zatím nepoužito", + "v2StarAria": "{criterion}: {star} hvězdiček", + "v2Overall": "Celkem", + "v2Verdict": "Verdikt", + "v2VerdictHint": "Kdo by si ho měl koupit a kdo ne", + "v2Pros": "Klady", + "v2Cons": "Zápory", + "v2ConsHint": "I drobnost: cena, příslušenství, hmotnost", + "v2AddLine": "Přidat řádek", + "v2RemoveLine": "Odebrat řádek", + "v2CriterionPoints": "Vlastní klady a zápory", + "v2Comment": "Podrobnosti (nepovinné)", + "v2Submit": "Odeslat recenzi", + "v2Submitting": "Odesílání...", + "v2ErrInvalid": "Recenze zatím není platná: zkontrolujte označená pole.", + "v2ErrCriterion": "Neplatné kritérium.", + "v2ErrRequired": "Toto kritérium je nutné ohodnotit.", + "v2ErrSkipped": "Vynechat lze nejvýše {max} kritéria.", + "v2ErrVerdict": "Verdikt musí mít {min} až {max} znaků.", + "v2ErrCount": "Je třeba 1 až {max} řádků.", + "v2ErrLength": "Každý řádek musí mít {min} až {max} znaků.", + "v2ErrNoContent": "Napište to konkrétněji.", + "v2ErrEvasive": "Uveďte jednu věc, která vám vadí, i drobnou.", + "v2ErrDuplicate": "Tento řádek se opakuje.", + "v2ErrBoth": "Tento řádek už je mezi klady." }, "watchlist": { "metaTitle": "Seznam hlídaných cen a upozornění na slevy | Rate.vn", diff --git a/apps/web/locales/en.json b/apps/web/locales/en.json index 16f12778..619b533d 100644 --- a/apps/web/locales/en.json +++ b/apps/web/locales/en.json @@ -579,7 +579,34 @@ "reportTitle": "Report a fake, spam or sabotage review (each report lowers its weight by 0.35)", "reportBtnLabel": "Report ({count})", "filterGroupAria": "Filter reviews", - "exportMenu": "Export data" + "exportMenu": "Export data", + "v2FormHeading": "Review {productName}", + "v2CriteriaLabel": "Rate each criterion", + "v2Skip": "Not used yet", + "v2StarAria": "{criterion}: {star} stars", + "v2Overall": "Overall", + "v2Verdict": "Verdict", + "v2VerdictHint": "Who should buy it, who should not", + "v2Pros": "Pros", + "v2Cons": "Cons", + "v2ConsHint": "Even a small one: price, accessories, weight", + "v2AddLine": "Add line", + "v2RemoveLine": "Remove line", + "v2CriterionPoints": "Own pros and cons", + "v2Comment": "Details (optional)", + "v2Submit": "Submit review", + "v2Submitting": "Submitting...", + "v2ErrInvalid": "The review is not valid yet: check the marked fields.", + "v2ErrCriterion": "Not a valid criterion.", + "v2ErrRequired": "This criterion must be rated.", + "v2ErrSkipped": "Skip at most {max} criteria.", + "v2ErrVerdict": "The verdict needs {min} to {max} characters.", + "v2ErrCount": "Needs 1 to {max} lines.", + "v2ErrLength": "Each line needs {min} to {max} characters.", + "v2ErrNoContent": "Say a bit more.", + "v2ErrEvasive": "Name one thing you dislike, however small.", + "v2ErrDuplicate": "This line is a duplicate.", + "v2ErrBoth": "This line is already a pro." }, "watchlist": { "metaTitle": "True price watchlist and drop alerts | Rate.vn", diff --git a/apps/web/locales/vi.json b/apps/web/locales/vi.json index f9b34768..df7faa26 100644 --- a/apps/web/locales/vi.json +++ b/apps/web/locales/vi.json @@ -579,7 +579,34 @@ "reportTitle": "Báo cáo đánh giá ảo, spam hoặc chơi xấu (mỗi báo cáo giảm trọng số 0.35)", "reportBtnLabel": "Báo cáo ({count})", "filterGroupAria": "Lọc đánh giá", - "exportMenu": "Xuất dữ liệu" + "exportMenu": "Xuất dữ liệu", + "v2FormHeading": "Đánh giá {productName}", + "v2CriteriaLabel": "Chấm từng tiêu chí", + "v2Skip": "Chưa dùng tới", + "v2StarAria": "{criterion}: {star} sao", + "v2Overall": "Điểm tổng", + "v2Verdict": "Kết luận", + "v2VerdictHint": "Ai nên mua, ai không nên", + "v2Pros": "Ưu điểm", + "v2Cons": "Nhược điểm", + "v2ConsHint": "Điều nhỏ nhất cũng được: giá, phụ kiện, cân nặng", + "v2AddLine": "Thêm dòng", + "v2RemoveLine": "Xoá dòng", + "v2CriterionPoints": "Ưu nhược riêng", + "v2Comment": "Chi tiết (không bắt buộc)", + "v2Submit": "Gửi đánh giá", + "v2Submitting": "Đang gửi...", + "v2ErrInvalid": "Đánh giá chưa hợp lệ, xem lại các ô được đánh dấu.", + "v2ErrCriterion": "Tiêu chí không hợp lệ.", + "v2ErrRequired": "Cần chấm tiêu chí này.", + "v2ErrSkipped": "Bỏ qua tối đa {max} tiêu chí.", + "v2ErrVerdict": "Kết luận dài {min} đến {max} ký tự.", + "v2ErrCount": "Cần 1 đến {max} dòng.", + "v2ErrLength": "Mỗi dòng {min} đến {max} ký tự.", + "v2ErrNoContent": "Viết rõ ý hơn.", + "v2ErrEvasive": "Ghi một điểm chưa ưng, dù nhỏ.", + "v2ErrDuplicate": "Dòng này bị trùng.", + "v2ErrBoth": "Dòng này đã có ở ưu điểm." }, "watchlist": { "metaTitle": "Theo dõi giá thật và cảnh báo giảm giá | Rate.vn", diff --git a/apps/web/package.json b/apps/web/package.json index 4921f950..f57430df 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -8,7 +8,7 @@ "lint": "eslint . --max-warnings 0", "preview": "next build && next start", "start": "next start", - "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-model.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/spec-scope.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/item-fields.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-profile-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-review-risk.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/watchlist-sizes.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-compare.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-variant-links.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/compare-params.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/image-url.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-market-stats.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-badge-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-criteria.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/focus-trap.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/display-truth.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/spec-template.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-spec-rows.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sale-state.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/csp.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/device-images.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/render-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/model-match.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/developers-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/plural-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/locale-format.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/paged-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/server-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/hero-products.test.ts", + "test": "pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/core-rate-architecture.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-model.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/server/rate/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/write-paths.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/spec-scope.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/app/api/v1/rate/__tests__/item-fields.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/logger.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/review-form-v2.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/concurrency.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-profile-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/merchant-review-risk.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-directory.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/watchlist-sizes.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/item-compare.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-variant-links.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/compare-params.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/catalog-listing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/discount-verdict-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-detail-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-competitors.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/analyzer-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sold-count.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/content-lang.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/image-url.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/client-messages.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/seo.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/offer-plausibility.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-market-stats.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sitemap-lastmod.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-badge-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/leaderboard-criteria.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/rating-display.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/focus-trap.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/display-truth.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/spec-template.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/product-spec-rows.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/sale-state.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/csp.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/device-images.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/render-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/model-match.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/developers-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/plural-copy.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/locale-format.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/paged-reads.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/server-timing.test.ts && pnpm --filter @repo/crawlers exec tsx --tsconfig ../web/tsconfig.test.json ../web/src/lib/__tests__/hero-products.test.ts", "typecheck": "tsc --noEmit" }, "browserslist": [ diff --git a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts index 493a2aa5..445a48b5 100644 --- a/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts +++ b/apps/web/src/app/api/v1/rate/__tests__/write-paths.test.ts @@ -360,6 +360,154 @@ async function testReviews() { ); } +// Review form v2 (DOS/Rate#1544): behind NEXT_PUBLIC_RATE_REVIEW_FORM_V2, +// checked against the product's category, and never stored until the DOS-Me +// v2 RPC exists. +async function testReviewsV2() { + const { POST } = await import('../reviews/route'); + type Req = Parameters[0]; + const v2Review = { + masterProductId: MASTER_ID, + formVersion: 2, + authorName: 'Tester', + ratingStar: 5, + criteriaRatings: { + performance: 4.5, + camera: 4, + battery: 5, + display: 4, + value: 3.5, + }, + verdict: 'Hợp người chụp ảnh nhiều, không hợp người chơi game nặng.', + pros: ['Pin dùng trọn ngày'], + cons: ['Giá hơi cao so với cấu hình'], + }; + rpcResponder = () => ({ status: 200, json: { id: 'stored' } }); + + // Flag off: the body takes the current path, which needs a comment. + delete process.env.NEXT_PUBLIC_RATE_REVIEW_FORM_V2; + calls = []; + let res = await POST( + post('/api/v1/rate/reviews', v2Review, GOOD_TOKEN) as Req + ); + assert.equal(res.status, 400); + assert.equal((await res.json()).code, 'INVALID_BODY'); + assert.equal(rpcCalls('submit_community_review').length, 0); + + process.env.NEXT_PUBLIC_RATE_REVIEW_FORM_V2 = '1'; + try { + // With v2 on, a v1 body is refused: no verdict, pros or cons. + calls = []; + res = await POST( + post('/api/v1/rate/reviews', validReview, GOOD_TOKEN) as Req + ); + assert.equal(res.status, 400); + assert.equal((await res.json()).code, 'INVALID_BODY'); + calls = []; + res = await POST( + post( + '/api/v1/rate/reviews', + { ...v2Review, formVersion: '2' }, + GOOD_TOKEN + ) as Req + ); + assert.equal(res.status, 400); + assert.equal(rpcCalls('submit_community_review').length, 0); + + // Still signed-in only. + calls = []; + res = await POST(post('/api/v1/rate/reviews', v2Review) as Req); + assert.equal(res.status, 401); + + // No verdict, no pro, no con: rejected per field. + calls = []; + res = await POST( + post( + '/api/v1/rate/reviews', + { ...v2Review, verdict: '', pros: [], cons: ['không có'] }, + GOOD_TOKEN + ) as Req + ); + assert.equal(res.status, 400); + const invalid = await res.json(); + assert.equal(invalid.code, 'INVALID_REVIEW'); + assert.deepEqual( + invalid.fieldErrors.map((e: { field: string }) => e.field), + ['verdict', 'pros', 'cons.0'] + ); + assert.equal(rpcCalls('submit_community_review').length, 0); + + // A criterion of another category is rejected, not dropped. + calls = []; + res = await POST( + post( + '/api/v1/rate/reviews', + { + ...v2Review, + criteriaRatings: { ...v2Review.criteriaRatings, anc: 5 }, + }, + GOOD_TOKEN + ) as Req + ); + assert.equal(res.status, 400); + assert.deepEqual((await res.json()).fieldErrors, [ + { field: 'criterion.anc', key: 'v2ErrCriterion' }, + ]); + + // An oversized body is refused before any check. + calls = []; + res = await POST( + post( + '/api/v1/rate/reviews', + { ...v2Review, pros: Array.from({ length: 21 }, () => 'Pin trâu') }, + GOOD_TOKEN + ) as Req + ); + assert.equal(res.status, 400); + assert.equal((await res.json()).code, 'INVALID_BODY'); + + // A draft product is not found. + calls = []; + res = await POST( + post( + '/api/v1/rate/reviews', + { ...v2Review, masterProductId: DRAFT_MASTER_ID }, + GOOD_TOKEN + ) as Req + ); + assert.equal(res.status, 404); + + // A valid review waits for the v2 RPC: 503, and nothing is written. + calls = []; + res = await POST(post('/api/v1/rate/reviews', v2Review, GOOD_TOKEN) as Req); + assert.equal(res.status, 503); + assert.equal((await res.json()).code, 'REVIEWS_UNAVAILABLE'); + assert.equal( + calls.filter((c) => c.url.includes('/rest/v1/rpc/')).length, + 0 + ); + assert.ok(calls.every((c) => c.method === 'GET')); + } finally { + delete process.env.NEXT_PUBLIC_RATE_REVIEW_FORM_V2; + } + + // The stored star is computed from the criteria (the test product is a + // phone): the body's ratingStar 5 is never read. Lines are single lines. + const { buildReviewV2Submission } = await import('@/server/rate/review-v2'); + const built = buildReviewV2Submission('smartphones', v2Review); + assert.ok(built.ok); + // (4.5*20 + 4*20 + 5*20 + 4*15 + 3.5*15) / 90 = 4.25 + assert.equal(built.submission.ratingStar, 4.3); + assert.equal(built.submission.criteriaVersion, 1); + assert.equal( + buildReviewV2Submission('smartphones', { ...v2Review, verdict: '' }).ok, + false + ); + console.warn( + ' ✅ POST review form v2 is flagged, checked per field and never stored yet (#1544)' + ); +} + // GET reviews by id serves published products only, like the slug path (DOS/Rate#1309). async function testReviewsByIdPublishedOnly() { const { GET } = await import('../reviews/route'); @@ -1688,6 +1836,7 @@ async function testMerchantCacheRevalidateAndDomainBounds() { async function main() { console.log('🧪 Public write-path security suite (#155)'); await testReviews(); + await testReviewsV2(); await testReviewsByIdPublishedOnly(); await testWatchlist(); await testCookieSessions(); diff --git a/apps/web/src/app/api/v1/rate/reviews/route.ts b/apps/web/src/app/api/v1/rate/reviews/route.ts index 02cb8577..18a7cf04 100644 --- a/apps/web/src/app/api/v1/rate/reviews/route.ts +++ b/apps/web/src/app/api/v1/rate/reviews/route.ts @@ -7,6 +7,7 @@ import { getCachedMasterProductBySlug, getCachedMasterProductReviews, getCachedMasterProductReviewSummary, + getCachedPublishedMasterProductCategory, isCachedPublishedMasterProduct, } from '@/server/rate'; import { @@ -24,6 +25,14 @@ import { type RpcRejection, } from '@/server/rate/request-auth'; import { createLogger, serializeError } from '@/lib/logger'; +import { + REVIEW_FORM_VERSION, + isReviewFormV2Enabled, +} from '@/lib/review-form-v2'; +import { + buildReviewV2Submission, + submitCommunityReviewV2, +} from '@/server/rate/review-v2'; const log = createLogger('api.reviews'); @@ -167,6 +176,51 @@ const SubmitReviewSchema = z.object({ .optional(), }); +const stripControls = (s: string) => + s + .replace(/\r\n?/g, '\n') + .replace(/[^\P{Cc}\n]+/gu, '') + .trim(); +// One line: control characters and line breaks become spaces. +const oneLine = (s: string) => + s + .replace(/\p{Cc}+/gu, ' ') + .replace(/\s+/g, ' ') + .trim(); +// Generous caps against an oversized body; the real limits are the D5 rules +// in reviewV2Errors, which name the field that breaks them. +const V2Line = z.string().max(600).transform(oneLine); +const V2Lines = z.array(V2Line).max(20); + +/** The v2 body (DOS/Rate#1544): no star, the server computes it. */ +const SubmitReviewV2Schema = z.object({ + masterProductId: z + .string() + .trim() + .uuid('Mã sản phẩm chuẩn hóa (masterProductId) không hợp lệ.'), + formVersion: z.literal(REVIEW_FORM_VERSION), + authorName: z.string().max(120).optional(), + variation: z.string().max(200).optional(), + comment: z + .string() + .transform(stripControls) + .pipe(z.string().max(3000, 'Nội dung đánh giá dài tối đa 3.000 ký tự.')) + .optional(), + criteriaRatings: z + .record(z.string().max(40), z.number().finite().nullable()) + .refine((r) => Object.keys(r).length <= 20), + verdict: z.string().max(1200).transform(oneLine), + pros: V2Lines, + cons: V2Lines, + criterionPoints: z + .record( + z.string().max(40), + z.object({ pros: V2Lines.optional(), cons: V2Lines.optional() }) + ) + .refine((r) => Object.keys(r).length <= 20) + .optional(), +}); + const FLAG_CATEGORIES = new Set([ 'authentic', 'low_value_coin_spam', @@ -376,6 +430,12 @@ export async function POST(request: NextRequest) { ); if (!auth.ok) return auth.response; + // With v2 on, every review is a v2 review: a v1 body (no verdict, pros + // or cons, a star from the browser) is refused as INVALID_BODY. + if (isReviewFormV2Enabled()) { + return await submitReviewV2(auth.rateClient, rawBody); + } + const parsed = SubmitReviewSchema.safeParse(rawBody ?? {}); if (!parsed.success) { return NextResponse.json( @@ -506,3 +566,71 @@ export async function POST(request: NextRequest) { ); } } + +/** + * The v2 path, behind NEXT_PUBLIC_RATE_REVIEW_FORM_V2 (DOS/Rate#1544): the + * review is checked against its product's category with the form's own + * rules, and its star is the weighted average of its criteria. Storing it + * waits for the DOS-Me v2 RPC, so a valid review gets a 503 for now. + */ +async function submitReviewV2( + rateClient: unknown, + rawBody: unknown +): Promise { + const parsed = SubmitReviewV2Schema.safeParse(rawBody); + if (!parsed.success) { + return NextResponse.json( + { + ok: false, + error: + parsed.error.issues[0]?.message || 'Dữ liệu đánh giá không hợp lệ.', + code: 'INVALID_BODY', + }, + { status: 400, headers: REVIEWS_CORS_HEADERS } + ); + } + const body = parsed.data; + + const categorySlug = await getCachedPublishedMasterProductCategory( + body.masterProductId + ); + if (!categorySlug) { + return NextResponse.json( + { ok: false, error: 'Product not found', code: 'NOT_FOUND' }, + { status: 404, headers: REVIEWS_CORS_HEADERS } + ); + } + + const checked = buildReviewV2Submission(categorySlug, body); + if (!checked.ok) { + return NextResponse.json( + { + ok: false, + error: 'Đánh giá chưa hợp lệ, xem lại các ô được đánh dấu.', + code: 'INVALID_REVIEW', + fieldErrors: checked.fieldErrors, + }, + { status: 400, headers: REVIEWS_CORS_HEADERS } + ); + } + + const result = await submitCommunityReviewV2(rateClient, checked.submission); + if (!result.ok) { + return NextResponse.json( + { + ok: false, + error: 'Chức năng gửi đánh giá chưa sẵn sàng trên môi trường này.', + code: result.code, + }, + { status: 503, headers: REVIEWS_CORS_HEADERS } + ); + } + return NextResponse.json( + { + ok: true, + mode: 'supabase_rpc', + review: toReviewItem(result.record, {}), + }, + { headers: REVIEWS_CORS_HEADERS } + ); +} diff --git a/apps/web/src/components/reviews/JReviewsSection.tsx b/apps/web/src/components/reviews/JReviewsSection.tsx index 0b6f21f4..3865f242 100644 --- a/apps/web/src/components/reviews/JReviewsSection.tsx +++ b/apps/web/src/components/reviews/JReviewsSection.tsx @@ -52,6 +52,8 @@ import { type ReviewBadgeKind, } from '@/lib/review-display'; import AnalyzerText, { analyzerText } from './AnalyzerText'; +import ReviewFormV2 from './ReviewFormV2'; +import { isReviewFormV2Enabled } from '@/lib/review-form-v2'; import { formatDecimal, formatPercent, @@ -628,38 +630,8 @@ export default function JReviewsSection({ return; } - const newRev = payload.review as ReviewItem; - setReviews((prev) => [newRev, ...prev]); - - try { - const cacheKey = `${LOCAL_REVIEWS_PREFIX}${masterProductId || productSlug || productName}`; - const existingLocal = JSON.parse( - window.localStorage.getItem(cacheKey) || '[]' - ); - const nextLocal = [ - newRev, - ...(Array.isArray(existingLocal) ? existingLocal : []), - ].slice(0, 20); - window.localStorage.setItem(cacheKey, JSON.stringify(nextLocal)); - } catch { - // Ignore storage quota errors - } - setComment(''); - setShowWriteForm(false); - setSubmitSuccess( - newRev.verdict === 'unanalyzed' - ? t('successPendingModeration') - : newRev.include_in_rating - ? t('successPublishedAuthentic', { - weight: formatDecimal(newRev.helpfulness_weight, locale, 2), - }) - : t('successFlaggedFiltered', { - flag: - flagMetaMap[newRev.flag_category]?.label ?? - newRev.flag_category, - }) - ); + handleReviewStored(payload.review as ReviewItem); } catch { setSubmitError(t('errNetworkFailed')); } finally { @@ -667,6 +639,40 @@ export default function JReviewsSection({ } } + /** Shows a stored review at the top and says how it was classified. */ + function handleReviewStored(newRev: ReviewItem) { + setReviews((prev) => [newRev, ...prev]); + + try { + const cacheKey = `${LOCAL_REVIEWS_PREFIX}${masterProductId || productSlug || productName}`; + const existingLocal = JSON.parse( + window.localStorage.getItem(cacheKey) || '[]' + ); + const nextLocal = [ + newRev, + ...(Array.isArray(existingLocal) ? existingLocal : []), + ].slice(0, 20); + window.localStorage.setItem(cacheKey, JSON.stringify(nextLocal)); + } catch { + // Ignore storage quota errors + } + + setShowWriteForm(false); + setSubmitSuccess( + newRev.verdict === 'unanalyzed' + ? t('successPendingModeration') + : newRev.include_in_rating + ? t('successPublishedAuthentic', { + weight: formatDecimal(newRev.helpfulness_weight, locale, 2), + }) + : t('successFlaggedFiltered', { + flag: + flagMetaMap[newRev.flag_category]?.label ?? + newRev.flag_category, + }) + ); + } + const [copiedReviewsMd, setCopiedReviewsMd] = useState(false); const handleCopyReviewsMarkdown = async () => { @@ -879,8 +885,20 @@ export default function JReviewsSection({ )} + {/* Review form v2 behind its flag (DOS/Rate#1544) */} + {showWriteForm && isReviewFormV2Enabled() && ( + + )} + {/* Interactive 5-Criteria Write Review Form */} - {showWriteForm && ( + {showWriteForm && !isReviewFormV2Enabled() && (
void; +} + +const MESSAGE_KEYS = new Set([ + 'v2ErrCriterion', + 'v2ErrRequired', + 'v2ErrSkipped', + 'v2ErrVerdict', + 'v2ErrCount', + 'v2ErrLength', + 'v2ErrNoContent', + 'v2ErrEvasive', + 'v2ErrDuplicate', + 'v2ErrBoth', +]); + +/** Field errors from the API, kept only when they have the known shape. */ +function apiFieldErrors(value: unknown): ReviewV2Error[] { + if (!Array.isArray(value)) return []; + return value + .filter( + (e): e is ReviewV2Error => + Boolean(e) && + typeof e.field === 'string' && + e.field.length <= 80 && + MESSAGE_KEYS.has(e.key) + ) + .slice(0, 40) + .map((e) => ({ + field: e.field, + key: e.key, + values: + e.values && typeof e.values === 'object' + ? Object.fromEntries( + Object.entries(e.values).filter( + ([, v]) => typeof v === 'number' && Number.isFinite(v) + ) + ) + : undefined, + })); +} + +/** + * Stars from 1 to 5 in half steps, as a radio group: one tab stop, arrows + * move by half a star (design 4.1, no default value). + */ +function HalfStarPicker({ + value, + label, + invalid, + required, + describedBy, + onChange, +}: { + value: number | null; + label: string; + invalid: boolean; + required: boolean; + describedBy?: string; + onChange: (star: number) => void; +}) { + const t = useTranslations('reviews'); + const locale = useLocale(); + const refs = useRef>({}); + const focusable = value ?? 1; + + function move(next: number) { + const star = Math.max(1, Math.min(5, next)); + onChange(star); + refs.current[String(star)]?.focus(); + } + + function handleKey(e: React.KeyboardEvent) { + const step = + e.key === 'ArrowRight' || e.key === 'ArrowUp' + ? 0.5 + : e.key === 'ArrowLeft' || e.key === 'ArrowDown' + ? -0.5 + : 0; + if (step) { + e.preventDefault(); + move((value ?? (step > 0 ? 0.5 : 1.5)) + step); + } else if (e.key === 'Home' || e.key === 'End') { + e.preventDefault(); + move(e.key === 'Home' ? 1 : 5); + } + } + + return ( +
+ {[1, 2, 3, 4, 5].map((n) => { + const fill = + value != null && value >= n + ? 100 + : value != null && value >= n - 0.5 + ? 50 + : 0; + return ( + +
+ ); +} + +function LineList({ + label, + hint, + field, + lines, + max, + errorFor, + onChange, +}: { + label: string; + hint?: string; + field: string; + lines: string[]; + max: number; + errorFor: (field: string) => string | null; + onChange: (lines: string[]) => void; +}) { + const t = useTranslations('reviews'); + const baseId = useId(); + const listError = errorFor(field); + const listErrorId = `${baseId}-err`; + const rows = lines.length > 0 ? lines : ['']; + // Blank lines are dropped before the check, so error N is the Nth line + // with text in it, not the Nth row. + let checked = -1; + const checkedIndex = rows.map((line) => + line.trim() ? (checked += 1) : null + ); + return ( +
+ {label} + {rows.map((line, index) => { + const at = checkedIndex[index]; + const error = at == null ? null : errorFor(`${field}.${at}`); + const errorId = `${baseId}-${index}-err`; + return ( +
+
+ + onChange( + rows.map((l, i) => (i === index ? e.target.value : l)) + ) + } + className={`form-control form-control-sm rounded-3 ${error ? 'is-invalid' : ''}`} + /> + {rows.length > 1 && ( + + )} +
+ {error && ( +
+ {error} +
+ )} +
+ ); + })} + {listError && ( +
+ {listError} +
+ )} + {rows.length < max && ( + + )} +
+ ); +} + +/** + * The v2 review form (DOS/Rate#1544, design #1531 section 4.1): per-category + * criteria in half stars with the weighted star beside them, a verdict, 1-5 + * pros and cons, and optional pros and cons per rated criterion. Errors show + * on the field they belong to, and the draft stays in the browser until the + * review is stored. + */ +export default function ReviewFormV2({ + masterProductId, + productName, + categorySlug, + defaultAuthorName, + onStored, +}: ReviewFormV2Props) { + const t = useTranslations('reviews'); + const locale = useLocale(); + const fieldId = useId(); + const criteria: readonly RatingCriterion[] = useMemo( + () => getRatingCriteria(categorySlug ?? null), + [categorySlug] + ); + const labelOf = (c: RatingCriterion) => + locale === 'vi' ? c.label.vi : c.label.en; + const draftKey = masterProductId + ? `${REVIEW_V2_DRAFT_PREFIX}${masterProductId}` + : null; + + const [draft, setDraft] = useState(emptyReviewV2Draft); + const [draftLoaded, setDraftLoaded] = useState(false); + const [authorName, setAuthorName] = useState(''); + const [variation, setVariation] = useState(''); + const [openPoints, setOpenPoints] = useState(null); + const [attempted, setAttempted] = useState(false); + const [serverErrors, setServerErrors] = useState([]); + const [submitError, setSubmitError] = useState(null); + const [submitting, setSubmitting] = useState(false); + + useEffect(() => { + if (!draftKey) return; + try { + const saved = parseReviewV2Draft(window.localStorage.getItem(draftKey)); + if (saved) setDraft(saved); + } catch { + // Storage blocked: the form starts empty. + } + setDraftLoaded(true); + }, [draftKey]); + + useEffect(() => { + if (!draftLoaded || !draftKey) return; + try { + window.localStorage.setItem(draftKey, JSON.stringify(draft)); + } catch { + // Storage full or blocked: the draft lives only in the page. + } + }, [draft, draftKey, draftLoaded]); + + const input = useMemo(() => reviewV2InputFromDraft(draft), [draft]); + const overall = reviewV2Star(categorySlug ?? null, input.criteriaRatings); + const liveErrors = useMemo( + () => (attempted ? reviewV2Errors(categorySlug ?? null, input) : []), + [attempted, categorySlug, input] + ); + const errors = liveErrors.length > 0 ? liveErrors : serverErrors; + const errorFor = (field: string): string | null => { + const error = errors.find((e) => e.field === field); + return error ? t(error.key, error.values) : null; + }; + + function update(next: (prev: ReviewV2Draft) => ReviewV2Draft) { + setServerErrors([]); + setDraft(next); + } + + function setRating(key: string, star: number | null) { + update((prev) => ({ ...prev, ratings: { ...prev.ratings, [key]: star } })); + if (star == null && openPoints === key) setOpenPoints(null); + } + + function setPoints(key: string, list: 'pros' | 'cons', lines: string[]) { + update((prev) => { + const entry = prev.points[key] ?? { pros: [], cons: [] }; + return { + ...prev, + points: { ...prev.points, [key]: { ...entry, [list]: lines } }, + }; + }); + } + + async function handleSubmit(e: React.FormEvent) { + e.preventDefault(); + setSubmitError(null); + setAttempted(true); + if (!masterProductId) { + setSubmitError(t('errMissingMasterId')); + return; + } + if (reviewV2Errors(categorySlug ?? null, input).length > 0) { + setSubmitError(t('v2ErrInvalid')); + return; + } + setSubmitting(true); + try { + const res = await fetch('/api/v1/rate/reviews', { + method: 'POST', + headers: await jsonWriteHeaders(), + body: JSON.stringify({ + masterProductId, + formVersion: REVIEW_FORM_VERSION, + authorName: authorName.trim() || defaultAuthorName, + variation: variation.trim() || undefined, + comment: draft.comment.trim() || undefined, + ...input, + }), + }); + if (res.status === 401) { + setSubmitError(t('errLoginRequired')); + return; + } + const payload = await res.json().catch(() => null); + if (!res.ok || !payload?.ok || !payload?.review) { + setServerErrors(apiFieldErrors(payload?.fieldErrors)); + setSubmitError(t(submitErrorKey(payload?.code))); + return; + } + try { + if (draftKey) window.localStorage.removeItem(draftKey); + } catch { + // Nothing to clear when storage is blocked. + } + setDraft(emptyReviewV2Draft()); + setAttempted(false); + onStored(payload.review as ReviewItem); + } catch { + setSubmitError(t('errNetworkFailed')); + } finally { + setSubmitting(false); + } + } + + const skippedError = errorFor('criteria'); + const verdictError = errorFor('verdict'); + const verdictLength = [...draft.verdict.trim()].length; + + return ( + +

+ {t('v2FormHeading', { productName })} +

+ + {submitError && ( +
+
+ )} + +
+
+ + setAuthorName(e.target.value)} + placeholder={t('authorNamePlaceholder')} + className="form-control form-control-sm rounded-3" + maxLength={80} + /> +
+
+ + setVariation(e.target.value)} + placeholder={t('variationPlaceholder')} + className="form-control form-control-sm rounded-3" + maxLength={100} + /> +
+
+ +
+
+ {t('v2CriteriaLabel')} + + {t('v2Overall')}{' '} + + {overall == null ? '-' : formatDecimal(overall, locale, 1)} ★ + + +
+ {criteria.map((c) => { + const value = draft.ratings[c.key] ?? null; + // Skipped on purpose: null; not rated yet: no key at all. + const skipped = + Object.prototype.hasOwnProperty.call(draft.ratings, c.key) && + value == null; + const label = labelOf(c); + const error = errorFor(`criterion.${c.key}`); + const errorId = `${fieldId}-${c.key}-err`; + const pointsOpen = openPoints === c.key && value != null; + const points = draft.points[c.key] ?? { pros: [], cons: [] }; + return ( +
+
+ + {label} + {c.required && ( + + )} + + setRating(c.key, star)} + /> + + {value == null ? '' : formatDecimal(value, locale, 1)} + + {!c.required && ( + + )} + {value != null && ( + + )} +
+ {error && ( +
+ {error} +
+ )} + {pointsOpen && ( +
+
+ setPoints(c.key, 'pros', lines)} + /> +
+
+ setPoints(c.key, 'cons', lines)} + /> +
+
+ )} +
+ ); + })} + {skippedError && ( +
+ {skippedError} +
+ )} +
+ +
+ +