diff --git a/packages/shared-data/src/__tests__/shared-data.test.ts b/packages/shared-data/src/__tests__/shared-data.test.ts index d09917e5..4e1204bc 100644 --- a/packages/shared-data/src/__tests__/shared-data.test.ts +++ b/packages/shared-data/src/__tests__/shared-data.test.ts @@ -1709,6 +1709,243 @@ function testRatingCriteriaAndReviewForm(): void { assert.ok(Date.now() - started < 1000, 'linear on an oversized con'); } +function testProductPoints(): void { + const now = new Date('2026-11-01T00:00:00Z'); + const monthsAgo = (months: number): string => + new Date(now.getTime() - months * (365.25 / 12) * 86_400_000).toISOString(); + let next = 0; + const line = ( + writerId: string, + kind: rules.PointKind, + body: string, + extra: Partial = {} + ): rules.PointLine => { + next += 1; + return { + id: `l${String(next).padStart(3, '0')}`, + reviewId: `r-${writerId}`, + writerId, + kind, + criterionKey: null, + body, + weight: 1, + isExpert: false, + writtenAt: now.toISOString(), + ...extra, + }; + }; + const run = (lines: rules.PointLine[], slug = 'smartphones') => + rules.aggregateProductPoints(slug, lines, now); + + // Normalising: filler words go, diacritics stay, a negation joins the next + // word, so "pin không trâu" never matches "pin trâu". + assert.deepEqual(rules.pointTokens('Pin KHÁ trâu!!'), ['pin', 'trâu']); + assert.deepEqual(rules.pointTokens('pin không trâu'), ['pin', 'không_trâu']); + assert.deepEqual(rules.pointTokens('ko trâu'), ['không_trâu']); + assert.notDeepEqual( + rules.pointTokens('sạc nhanh'), + rules.pointTokens('sắc nhanh') + ); + assert.equal(rules.wordOverlap(['a', 'b'], ['a', 'b']), 1); + assert.equal(rules.wordOverlap(['a', 'b'], ['c']), 0); + assert.equal(rules.wordOverlap([], []), 0); + assert.ok( + rules.wordOverlap( + rules.pointTokens('pin trâu'), + rules.pointTokens('pin không trâu') + ) < rules.POINT_CLUSTER_OVERLAP + ); + + // Criteria: the line's own key when the category has it, else keywords, + // matched with or without diacritics but never with other ones. + const assign = rules.assignPointCriterion; + assert.equal(assign('smartphones', 'camera', 'pin trâu'), 'camera'); + assert.equal(assign('smartphones', 'anc', 'pin trâu'), 'battery'); + assert.equal(assign('smartphones', null, 'Pin trâu dùng cả ngày'), 'battery'); + assert.equal(assign('smartphones', null, 'pin trau, sac nhanh'), 'battery'); + assert.equal(assign('smartphones', null, 'cảm ứng mượt'), 'performance'); + assert.equal(assign('smartphones', null, 'Màn hình sáng rõ'), 'display'); + assert.equal(assign('earbuds', null, 'chống ồn tốt'), 'anc'); + assert.equal(assign('earbuds', null, 'Màn hình sáng rõ'), null); + assert.equal( + assign('smartphones', null, 'hộp đựng đẹp, giao đúng hẹn'), + null + ); + assert.equal(assign('smartphones', 'toString', 'abc def'), null); + + // A plain word that is a keyword without its diacritics ("do", "cho", + // "em", "cam") does not route a line, and "màn" inside "màn hình" is one hit. + assert.equal( + assign('smartphones', null, 'Pin tụt nhanh do cập nhật'), + 'battery' + ); + assert.equal( + assign('smartphones', null, 'Camera mờ do chống rung kém'), + 'camera' + ); + assert.equal( + assign('smartwatches', null, 'Pin tụt nhanh do bật GPS'), + 'battery' + ); + assert.equal(assign('smartphones', null, 'Màu cam đẹp lắm'), null); + assert.equal(assign('earbuds', null, 'Mua cho em gái, mic rõ'), 'calls'); + assert.equal( + assign('smartphones', null, 'màn hình đẹp, pin trâu'), + 'battery' + ); + + // Clustering: alike lines of one criterion and kind join; a pro and a con, + // or a negated line, do not. + const negated = run([ + line('a', 'con', 'Màn hình bị ám vàng nhẹ'), + line('b', 'con', 'Màn hình không bị ám vàng'), + ]); + assert.equal(negated.clusters.length, 2, 'a negation splits one kind'); + const alike = run([ + line('a', 'pro', 'Pin trâu, dùng cả ngày'), + line('b', 'pro', 'pin rất trâu dùng cả ngày'), + line('c', 'pro', 'Pin trâu dùng cả ngày luôn'), + line('d', 'con', 'pin không trâu dùng cả ngày'), + ]); + const batteryPro = alike.clusters.find((cl) => cl.kind === 'pro')!; + assert.equal(batteryPro.criterionKey, 'battery'); + assert.equal(batteryPro.reviewerCount, 3); + assert.deepEqual(batteryPro.lineIds.length, 3); + assert.equal(alike.clusters.length, 2); + assert.equal(alike.byCriterion.battery!.pros.length, 1); + assert.equal(alike.byCriterion.battery!.cons.length, 0, 'one writer: no con'); + assert.equal(alike.reviewCount, 4); + // The label is the form most writers used, in the first one's words. + const label = run([ + line('a', 'pro', 'Màn hình sáng, rõ nét', { writtenAt: monthsAgo(3) }), + line('b', 'pro', 'màn hình rất sáng rõ nét'), + line('c', 'pro', 'Màn hình sáng rõ nét!'), + ]); + assert.equal(label.clusters.length, 1); + assert.equal(label.clusters[0]!.label, 'Màn hình sáng, rõ nét'); + + // D9: two writers and a weight sum of 1.5, or one expert. + const two = (weight: number) => + run( + [ + line('a', 'pro', 'camera chụp đêm đẹp', { weight }), + line('b', 'pro', 'camera chụp đêm rất đẹp', { weight }), + ], + 'smartphones' + ).clusters[0]!; + assert.equal(two(0.7).visible, false, '1.4 is below 1.5'); + assert.equal(two(0.75).support, 1.5); + assert.equal(two(0.75).visible, true); + // The threshold reads the exact sum: 1.4952 is stored as 1.50 but hidden. + assert.equal(two(0.7476).support, 1.5); + assert.equal(two(0.7476).visible, false); + const expert = run([ + line('x', 'con', 'Camera tele hơi mờ', { isExpert: true }), + ]); + assert.equal(expert.clusters[0]!.visible, true); + assert.equal(expert.summary.cons.length, 1); + + // A hostile writer, even at full weight over several reviews, shows no con. + const hostile = run([ + line('h', 'con', 'Pin tụt nhanh kinh khủng', { weight: 3, reviewId: 'r1' }), + line('h', 'con', 'pin tụt rất nhanh kinh khủng', { + weight: 3, + reviewId: 'r2', + }), + line('h', 'con', 'Màn hình ám vàng', { weight: 3, reviewId: 'r1' }), + line('ok1', 'pro', 'Camera đẹp'), + line('ok2', 'pro', 'camera rất đẹp'), + ]); + assert.equal(hostile.summary.cons.length, 0); + assert.equal(hostile.summary.pros.length, 1); + const hostileBattery = hostile.clusters.find( + (cl) => cl.criterionKey === 'battery' + )!; + assert.equal(hostileBattery.reviewerCount, 1); + assert.equal( + hostileBattery.support, + 3, + 'one writer counts once, capped at 3' + ); + // Gated reviews (weight 0) and dodging cons are not counted at all. + const gated = run([ + line('a', 'con', 'loa nhỏ, rè', { weight: 0 }), + line('b', 'con', 'loa nhỏ rè', { weight: 0 }), + line('c', 'con', 'Không có nhược điểm nào cả'), + line('d', 'con', 'không có gì để chê'), + ]); + assert.equal(gated.clusters.length, 0); + assert.equal(gated.reviewCount, 0); + // Lines the review_points table would refuse, or with no writer, no date or + // a weight that is not a number, are not counted either. + const malformed = run([ + line('a', 'pro', 'ok'), + line('b', 'pro', 'ok'), + line('c', 'pro', 'aaaaaaa'), + line('d', 'pro', `pin trâu ${'x'.repeat(150)}`), + line('', 'pro', 'pin trâu cả ngày'), + { ...line('e', 'pro', 'pin trâu cả ngày'), writerId: undefined as never }, + { ...line('f', 'pro', 'pin trâu cả ngày'), writtenAt: null as never }, + line('g', 'pro', 'pin trâu cả ngày', { weight: Infinity }), + line('h', 'pro', 'pin trâu cả ngày', { weight: Number.NaN }), + ]); + assert.equal(malformed.clusters.length, 0); + + // Freshness: a weight halves every 12 months; 18 silent months hide a point. + assert.ok( + Math.abs(rules.decayedWeight(1, new Date(monthsAgo(12)), now) - 0.5) < 1e-9 + ); + assert.equal(rules.decayedWeight(5, now, now), rules.POINT_WEIGHT_CAP); + assert.equal(rules.decayedWeight(-1, now, now), 0); + const aged = (months: number) => + run([ + line('a', 'pro', 'loa to rõ', { + isExpert: true, + writtenAt: monthsAgo(months), + }), + ]).clusters[0]!.visible; + assert.equal(aged(17), true); + assert.equal(aged(19), false); + // Two writers at weight 1, a year old: 0.5 + 0.5 is below 1.5. + const old = run([ + line('a', 'pro', 'sạc nhanh', { writtenAt: monthsAgo(12) }), + line('b', 'pro', 'sạc rất nhanh', { writtenAt: monthsAgo(12) }), + ]).clusters[0]!; + assert.equal(old.support, 1); + assert.equal(old.visible, false); + + // Limits: three per criterion and five per side in the summary; a passing + // con is kept however many pros pass. + const pros = [ + 'pin trâu', + 'sạc nhanh', + 'pin bền lâu chai', + 'mah lớn', + 'camera đẹp', + 'màn hình sáng', + 'hiệu năng mạnh', + 'thiết kế sang', + ]; + const many: rules.PointLine[] = []; + for (const body of pros) + for (const writer of ['a', 'b', 'c']) + many.push(line(writer, 'pro', body, { weight: 3 })); + many.push(line('d', 'con', 'giá hơi cao', { weight: 0.8 })); + many.push(line('e', 'con', 'giá cao', { weight: 0.8 })); + const crowded = run(many); + assert.equal(crowded.byCriterion.battery!.pros.length, 3); + assert.equal(crowded.summary.pros.length, rules.PRODUCT_POINTS_MAX); + assert.equal(crowded.summary.cons.length, 1); + assert.equal(crowded.summary.cons[0]!.criterionKey, 'value'); + assert.equal(crowded.updatedAt, now); + // The same lines in another order give the same clusters. + const reversed = run([...many].reverse()); + assert.deepEqual( + reversed.clusters.map((cl) => [cl.key, cl.support, cl.lineIds]), + crowded.clusters.map((cl) => [cl.key, cl.support, cl.lineIds]) + ); +} + function main(): void { console.log('🧪 Running @repo/shared-data unit tests...'); testVersionAndZeroStrapiExports(); @@ -1745,6 +1982,8 @@ function main(): void { console.log( ' ✅ [11/11] Rating criteria and review form rules (DOS/Rate#1542) verified' ); + testProductPoints(); + console.log(' ✅ [12/12] Product pros and cons (DOS/Rate#1546) verified'); console.log('🎉 All @repo/shared-data tests passed!'); } diff --git a/packages/shared-data/src/rules/index.ts b/packages/shared-data/src/rules/index.ts index 3abdc626..042f81d1 100644 --- a/packages/shared-data/src/rules/index.ts +++ b/packages/shared-data/src/rules/index.ts @@ -13,3 +13,4 @@ export * from './model-key.js'; export * from './review-summary.js'; export * from './rating-criteria.js'; export * from './review-form.js'; +export * from './product-points.js'; diff --git a/packages/shared-data/src/rules/product-points.ts b/packages/shared-data/src/rules/product-points.ts new file mode 100644 index 00000000..e4fbfb1d --- /dev/null +++ b/packages/shared-data/src/rules/product-points.ts @@ -0,0 +1,552 @@ +/** + * A product's pros and cons, built from its reviews' pro and con lines + * (DOS/Rate#1546, design #1531 section 5.1, D9). Code only, no AI: lines are + * normalised, given a criterion, clustered by shared words, counted once per + * writer by trust weight, decayed by age and filtered for display. AI later + * rewords a cluster's label; it never decides what is shown. + * + * Plain data and functions only, like the rest of `rules`, and no database: + * the importer job reads `rate.review_points` into PointLine values and + * writes the clusters to `rate.product_points`. + */ +import { getRatingCriteria } from './rating-criteria.js'; +import { + REVIEW_POINT_MAX, + REVIEW_POINT_MIN, + comparablePoint, + hasContent, + isEvasiveCon, +} from './review-form.js'; + +/** Two lines join a cluster when this share of their words is shared. */ +export const POINT_CLUSTER_OVERLAP = 0.6; +/** D9: a point shows with this many distinct writers... */ +export const POINT_MIN_WRITERS = 2; +/** ...whose decayed weights sum to at least this, or with one expert. */ +export const POINT_MIN_SUPPORT = 1.5; +/** A writer's weight counts at most this, as in the product rating. */ +export const POINT_WEIGHT_CAP = 3; +/** Pros and cons shown per criterion, each side. */ +export const PRODUCT_CRITERION_POINTS_MAX = 3; +/** Pros and cons shown in the product summary, each side. */ +export const PRODUCT_POINTS_MAX = 5; + +const DAY_MS = 24 * 60 * 60 * 1000; +const YEAR_MS = 365.25 * DAY_MS; +/** A line's weight halves every 12 months (design 5.3). */ +export const POINT_HALF_LIFE_MS = YEAR_MS; +/** A cluster nobody mentioned for 18 months is hidden (design 5.3). */ +export const POINT_SILENT_MS = 1.5 * YEAR_MS; + +export type PointKind = 'pro' | 'con'; + +/** One pro or con line of a review, as the importer reads it. */ +export interface PointLine { + /** `rate.review_points.id`. */ + id: string; + reviewId: string; + /** Who wrote it: a user id, or a marketplace reviewer key. */ + writerId: string; + kind: PointKind; + /** The line's own criterion, or null for a pro or con of the whole review. */ + criterionKey: string | null; + body: string; + /** The review's trust weight; 0 means the gate excluded it. */ + weight: number; + /** A review Rate published as an expert's (reviewer_type = 'expert'). */ + isExpert: boolean; + /** When the review was written (ISO string or Date). */ + writtenAt: string | Date; +} + +/** A cluster of lines, as `rate.product_points` stores it. */ +export interface PointCluster { + /** Stable within one run: criterion, kind and the first line's id. */ + key: string; + criterionKey: string | null; + kind: PointKind; + /** The line most writers wrote alike, in its writer's words. */ + label: string; + /** Distinct writers' decayed, capped weights, at two decimals. */ + support: number; + reviewerCount: number; + hasExpert: boolean; + firstSeen: Date; + lastSeen: Date; + /** Every counted line in the cluster, for `review_points.cluster_id`. */ + lineIds: string[]; + /** Whether it passes D9 and is not silent (design 5.3). */ + visible: boolean; +} + +export interface PointSides { + pros: PointCluster[]; + cons: PointCluster[]; +} + +export interface ProductPoints { + /** Every cluster, shown or not, in display order. */ + clusters: PointCluster[]; + /** Shown points per criterion key; '' holds the whole-review ("Chung") ones. */ + byCriterion: Record; + /** The product summary: up to five each side, a passing con always kept. */ + summary: PointSides; + /** Distinct reviews whose lines were counted ("Tóm tắt từ N review"). */ + reviewCount: number; + /** The run's clock, shown as "Cập nhật ngày ...". */ + updatedAt: Date; +} + +// The rules' import check reads quotes on an export line as a module path. +const EMPTY = ''; +const SPACE = ' '; +/** The bucket of a whole-review point ("Chung"). */ +export const GENERAL_POINTS = EMPTY; + +// Intensifiers and particles that do not change what a line says +// ("pin khá trâu" is "pin trâu"). +const FILLER = new Set([ + 'khá', + 'rất', + 'hơi', + 'quá', + 'lắm', + 'cực', + 'siêu', + 'cũng', + 'thì', + 'là', + 'ạ', + 'nhé', + 'nha', +]); +const NEGATED = 'không_'; +// A negation joins the next word, so "pin không trâu" never matches +// "pin trâu" by sharing "pin" and "trâu". +const NEGATION = new Set([ + 'không', + 'ko', + 'k', + 'kh', + 'khum', + 'hok', + 'chưa', + 'chẳng', + 'chả', +]); + +/** A word without diacritics, "đ" as "d". */ +function plainWord(word: string): string { + return word + .normalize('NFKD') + .replace(/\p{M}+/gu, '') + .replace(/đ/g, 'd'); +} + +/** + * The words two lines are compared by: the comparable line (diacritics + * kept, design 5.1) without filler words, each negation joined to the word + * after it ("không_trâu"). + */ +export function pointTokens(text: string): string[] { + const words = comparablePoint(text) + .split(SPACE) + .filter((word) => word && !FILLER.has(word)); + const tokens: string[] = []; + for (let i = 0; i < words.length; i += 1) { + const word = words[i]!; + if (NEGATION.has(word) && i + 1 < words.length) { + tokens.push(`${NEGATED}${words[i + 1]}`); + i += 1; + } else tokens.push(NEGATION.has(word) ? 'không' : word); + } + return tokens; +} + +/** + * The share of words two lines have in common: twice the shared distinct + * words over both lines' distinct words (the Dice coefficient), from 0 to 1. + */ +export function wordOverlap( + a: readonly string[], + b: readonly string[] +): number { + const left = new Set(a); + const right = new Set(b); + if (left.size + right.size === 0) return 0; + let shared = 0; + for (const word of left) if (right.has(word)) shared += 1; + return (2 * shared) / (left.size + right.size); +} + +// Keywords per criterion, written with diacritics. The analyzer's aspect +// words (battery, display, camera, audio, performance) are the start. A word +// whose plain form is a common word is left out: "đơ" and "đo" read "do" +// (because), "chờ" reads "cho", "êm" reads "em", "cầm" and "cam" read +// the colour, "mắc" reads "Mac". +const CRITERION_KEYWORDS: Readonly> = { + battery: ['pin', 'sạc', 'mah', 'sạc nhanh', 'thời lượng', 'củ sạc'], + display: [ + 'màn hình', + 'màn', + 'hiển thị', + 'độ sáng', + 'tần số quét', + 'nit', + 'ám màu', + 'chói', + ], + camera: [ + 'camera', + 'chụp', + 'chụp ảnh', + 'quay', + 'quay phim', + 'selfie', + 'zoom', + 'góc rộng', + ], + performance: [ + 'hiệu năng', + 'mượt', + 'lag', + 'giật', + 'chip', + 'game', + 'chơi game', + 'cấu hình', + 'ram', + 'xử lý', + ], + design: [ + 'thiết kế', + 'bền', + 'nhẹ', + 'nặng', + 'vỏ', + 'khung', + 'mỏng', + 'chắc chắn', + 'chống nước', + 'ngoại hình', + ], + value: ['giá', 'tiền', 'đáng tiền', 'rẻ', 'đắt', 'hời'], + sound: ['âm thanh', 'chất âm', 'loa', 'bass', 'treble', 'âm lượng'], + software: ['phần mềm', 'ứng dụng', 'cập nhật', 'ios', 'android', 'giao diện'], + keyboard: ['bàn phím', 'phím', 'touchpad', 'trackpad'], + thermals: ['nóng', 'nhiệt', 'quạt', 'ồn', 'tản nhiệt'], + health: [ + 'sức khoẻ', + 'sức khỏe', + 'nhịp tim', + 'spo2', + 'giấc ngủ', + 'ngủ', + 'bước chân', + ], + apps: [ + 'ứng dụng', + 'app', + 'kết nối', + 'thông báo', + 'bluetooth', + 'wifi', + 'gps', + 'nfc', + ], + anc: ['chống ồn', 'anc', 'xuyên âm', 'cách âm'], + comfort: ['đeo', 'thoải mái', 'đau tai', 'vừa tai', 'cấn'], + calls: ['mic', 'micro', 'gọi', 'cuộc gọi', 'đàm thoại', 'nghe gọi'], + service_quality: ['chất lượng', 'dịch vụ', 'sạch', 'ngon'], + price: ['giá', 'tiền', 'rẻ', 'đắt'], + attitude: ['thái độ', 'nhân viên', 'phục vụ', 'nhiệt tình', 'lịch sự'], + speed: ['nhanh', 'chậm', 'đợi', 'lâu'], +}; + +/** + * Whether a keyword's words appear in a line's words, in order. A line typed + * without any diacritics ("pin trau, sac nhanh") matches the plain keyword; + * a line with diacritics matches only as written, so "cảm" (cảm ứng) is not + * "cam" and "do" (because) is not "đo". + */ +function containsKeyword( + words: readonly string[], + keyword: string, + plainLine: boolean +): boolean { + const wanted = (plainLine ? plainWord(keyword) : keyword).split(SPACE); + for (let start = 0; start + wanted.length <= words.length; start += 1) { + if (wanted.every((want, offset) => words[start + offset] === want)) + return true; + } + return false; +} + +/** Matched keywords, not counting one inside a longer match ("màn" in "màn hình"). */ +function keywordHits( + words: readonly string[], + keywords: readonly string[], + plainLine: boolean +): number { + const matched = keywords.filter((kw) => + containsKeyword(words, kw, plainLine) + ); + return matched.filter( + (kw) => + !matched.some((other) => other !== kw && ` ${other} `.includes(` ${kw} `)) + ).length; +} + +/** + * The criterion a line belongs to in a category: its own key when the + * category has it, otherwise the criterion whose keywords it mentions most + * (ties go to the category's order), otherwise null ("Chung"). + */ +export function assignPointCriterion( + categorySlug: string | null | undefined, + criterionKey: string | null, + body: string +): string | null { + const criteria = getRatingCriteria(categorySlug); + if (criterionKey && criteria.some((cr) => cr.key === criterionKey)) + return criterionKey; + const words = comparablePoint(body).split(SPACE); + const plainLine = words.every((word) => word === plainWord(word)); + let best: string | null = null; + let bestHits = 0; + for (const criterion of criteria) { + const keywords = Object.prototype.hasOwnProperty.call( + CRITERION_KEYWORDS, + criterion.key + ) + ? CRITERION_KEYWORDS[criterion.key]! + : []; + const hits = keywordHits(words, keywords, plainLine); + if (hits > bestHits) { + best = criterion.key; + bestHits = hits; + } + } + return best; +} + +/** A weight after decay: half per POINT_HALF_LIFE_MS of age, capped. */ +export function decayedWeight( + weight: number, + writtenAt: Date, + now: Date +): number { + if (!Number.isFinite(weight) || weight <= 0) return 0; + const age = Math.max(0, now.getTime() - writtenAt.getTime()); + return Math.min(weight, POINT_WEIGHT_CAP) * 0.5 ** (age / POINT_HALF_LIFE_MS); +} + +interface Line { + source: PointLine; + criterionKey: string | null; + tokens: string[]; + form: string; + writtenAt: Date; +} + +function asDate(value: string | Date | null | undefined): Date { + if (value instanceof Date) return value; + // new Date(null) is 1970, not an unknown date. + return typeof value === 'string' ? new Date(value) : new Date(Number.NaN); +} + +/** + * Two lines that say opposite things: one negates a word the other states + * ("không_bị" against "bị"), however many other words they share. + */ +function contradicts(a: readonly string[], b: readonly string[]): boolean { + const negates = (from: readonly string[], to: Set): boolean => + from.some( + (token) => + token.startsWith(NEGATED) && to.has(token.slice(NEGATED.length)) + ); + return negates(a, new Set(b)) || negates(b, new Set(a)); +} + +/** + * The lines that may count: a writer, a finite positive weight, a date, a + * length rate.review_points allows, words in it, and not a dodge. + */ +function countedLines( + categorySlug: string | null | undefined, + lines: readonly PointLine[] +): Line[] { + const counted: Line[] = []; + for (const source of lines) { + if (source.kind !== 'pro' && source.kind !== 'con') continue; + if (!Number.isFinite(source.weight) || source.weight <= 0) continue; + if (typeof source.writerId !== 'string' || !source.writerId.trim()) + continue; + const writtenAt = asDate(source.writtenAt); + if (Number.isNaN(writtenAt.getTime())) continue; + const body = String(source.body ?? EMPTY).trim(); + const length = [...body.normalize('NFKC')].length; + if (length < REVIEW_POINT_MIN || length > REVIEW_POINT_MAX) continue; + if (!hasContent(body)) continue; + if (source.kind === 'con' && isEvasiveCon(body)) continue; + const tokens = pointTokens(body); + if (tokens.length === 0) continue; + counted.push({ + source, + criterionKey: assignPointCriterion( + categorySlug, + source.criterionKey, + source.body + ), + tokens, + form: tokens.join(SPACE), + writtenAt, + }); + } + // Oldest first, so a cluster's key and leader do not depend on read order. + return counted.sort( + (a, b) => + a.writtenAt.getTime() - b.writtenAt.getTime() || + (a.source.id < b.source.id ? -1 : a.source.id > b.source.id ? 1 : 0) + ); +} + +/** + * Greedy clustering within one criterion and kind: a line joins the cluster + * whose first line it overlaps most, at POINT_CLUSTER_OVERLAP or more, or + * starts a new one. Comparing with the first line, not any member, keeps a + * chain of small steps from joining two different points. + */ +function clusterLines(lines: readonly Line[]): Line[][] { + const clusters: Line[][] = []; + for (const line of lines) { + let best: Line[] | undefined; + let bestOverlap = POINT_CLUSTER_OVERLAP; + for (const cluster of clusters) { + const leader = cluster[0]!; + if ( + leader.criterionKey !== line.criterionKey || + leader.source.kind !== line.source.kind + ) + continue; + const overlap = contradicts(leader.tokens, line.tokens) + ? 0 + : wordOverlap(leader.tokens, line.tokens); + if (overlap >= bestOverlap && (!best || overlap > bestOverlap)) { + best = cluster; + bestOverlap = overlap; + } + } + if (best) best.push(line); + else clusters.push([line]); + } + return clusters; +} + +/** The label: the form most writers used, in its first writer's words. */ +function clusterLabel(lines: readonly Line[]): string { + const forms = new Map; first: Line }>(); + for (const line of lines) { + const entry = forms.get(line.form); + if (entry) entry.writers.add(line.source.writerId); + else + forms.set(line.form, { + writers: new Set([line.source.writerId]), + first: line, + }); + } + let best: { writers: Set; first: Line } | undefined; + for (const entry of forms.values()) + if (!best || entry.writers.size > best.writers.size) best = entry; + return best!.first.source.body.trim().replace(/\s+/g, ' '); +} + +function summarise(lines: readonly Line[], now: Date): PointCluster { + const leader = lines[0]!; + // Each writer counts once, with their strongest line in the cluster. + const writers = new Map(); + let hasExpert = false; + for (const line of lines) { + const weight = decayedWeight(line.source.weight, line.writtenAt, now); + const writer = line.source.writerId; + writers.set(writer, Math.max(writers.get(writer) ?? 0, weight)); + if (line.source.isExpert) hasExpert = true; + } + let rawSupport = 0; + for (const weight of writers.values()) rawSupport += weight; + const support = Math.round(rawSupport * 100) / 100; + const firstSeen = leader.writtenAt; + const lastSeen = lines[lines.length - 1]!.writtenAt; + const fresh = now.getTime() - lastSeen.getTime() <= POINT_SILENT_MS; + const passes = + (writers.size >= POINT_MIN_WRITERS && rawSupport >= POINT_MIN_SUPPORT) || + hasExpert; + return { + key: `${leader.criterionKey ?? GENERAL_POINTS}:${leader.source.kind}:${leader.source.id}`, + criterionKey: leader.criterionKey, + kind: leader.source.kind, + label: clusterLabel(lines), + support, + reviewerCount: writers.size, + hasExpert, + firstSeen, + lastSeen, + lineIds: lines.map((line) => line.source.id), + visible: passes && fresh, + }; +} + +/** Display order: more support, more writers, an expert, then newer. */ +function byRank(a: PointCluster, b: PointCluster): number { + return ( + b.support - a.support || + b.reviewerCount - a.reviewerCount || + Number(b.hasExpert) - Number(a.hasExpert) || + b.lastSeen.getTime() - a.lastSeen.getTime() || + (a.key < b.key ? -1 : a.key > b.key ? 1 : 0) + ); +} + +function sides(clusters: readonly PointCluster[], max: number): PointSides { + return { + pros: clusters.filter((cl) => cl.kind === 'pro').slice(0, max), + cons: clusters.filter((cl) => cl.kind === 'con').slice(0, max), + }; +} + +/** + * A product's pros and cons from its review lines (design 5.1 and 5.3): + * lines of gated reviews (weight 0), dodging cons and lines with no words + * are left out; the rest are clustered per criterion and kind, counted once + * per writer and shown under D9. `now` is the run's clock, so a run is + * repeatable. + */ +export function aggregateProductPoints( + categorySlug: string | null | undefined, + lines: readonly PointLine[], + now: Date +): ProductPoints { + const counted = countedLines(categorySlug, lines); + const clusters = clusterLines(counted) + .map((cluster) => summarise(cluster, now)) + .sort(byRank); + const shown = clusters.filter((cl) => cl.visible); + const byCriterion: Record = {}; + for (const key of new Set( + shown.map((cl) => cl.criterionKey ?? GENERAL_POINTS) + )) + byCriterion[key] = sides( + shown.filter((cl) => (cl.criterionKey ?? GENERAL_POINTS) === key), + PRODUCT_CRITERION_POINTS_MAX + ); + // Each side has its own five places, so a passing con is always kept in + // the summary however many pros pass (design 5.1, "luôn giữ nhược điểm"). + const summary = sides(shown, PRODUCT_POINTS_MAX); + return { + clusters, + byCriterion, + summary, + reviewCount: new Set(counted.map((line) => line.source.reviewId)).size, + updatedAt: now, + }; +} diff --git a/packages/shared-data/src/rules/review-form.ts b/packages/shared-data/src/rules/review-form.ts index 3a4453c3..9576d9ef 100644 --- a/packages/shared-data/src/rules/review-form.ts +++ b/packages/shared-data/src/rules/review-form.ts @@ -68,7 +68,7 @@ function visibleLength(text: string): number { * Whether a line has words in it: at least three letters or digits, and * not one character repeated ("aaaaa", "....", "🔥🔥🔥🔥🔥"). */ -function hasContent(text: string): boolean { +export function hasContent(text: string): boolean { const letters = comparablePoint(text).match(/[\p{L}\p{N}]/gu) ?? []; return letters.length >= 3 && new Set(letters).size >= 2; }