diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8a171cae..81ceff8a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,9 @@ jobs: - name: Replay the DOS-Me Item RPC and M3/M4 proposals (Docker) run: node scripts/test-m3-m4-item-rpcs-proposal.mjs + - name: Replay the DOS-Me criteria ratings proposal (Docker) + run: node scripts/test-criteria-ratings-proposal.mjs + - name: Check generated rate schema types match the migrations run: pnpm check:db-types diff --git a/.gitignore b/.gitignore index 48f5c2c3e..eb9fd6efb 100644 --- a/.gitignore +++ b/.gitignore @@ -149,6 +149,9 @@ playwright-report-item-model/ !Docs/Proposals/2026-10-29-dos-me-m3-m4.sql !Docs/Proposals/2026-10-29-dos-me-m3-m4-rollback.sql !Docs/Proposals/2026-10-29-dos-me-m3-m4-verify.sql +!Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql +!Docs/Proposals/2026-10-31-dos-me-criteria-ratings-rollback.sql +!Docs/Proposals/2026-10-31-dos-me-criteria-ratings-verify.sql * - Copy* * - Bản sao* *.dump diff --git a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings-rollback.sql b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings-rollback.sql new file mode 100644 index 000000000..2e84748f6 --- /dev/null +++ b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings-rollback.sql @@ -0,0 +1,46 @@ +-- PROPOSAL, NOT APPLIED. Rollback of 2026-10-31-dos-me-criteria-ratings.sql (DOS/Rate#1543). +-- One transaction, re-runnable, safe before or after M1+M2 or its rollback: the migration +-- replaces no existing function, so this only drops what it added. Data written only into the +-- new columns and tables (exact stars, verdicts, pro and con lines, product points, +-- rating_score, criteria_scores) is lost; nothing older depends on it. Run it only while no +-- phase reads or writes them: before P3 (#1544), P4 (#1545) and P5 (#1546), or after they are +-- reverted. + +BEGIN; + +-- 1. Triggers first, so the drops below fire nothing. +DROP TRIGGER IF EXISTS trg_reviews_rating_extras_ins ON rate.reviews; +DROP TRIGGER IF EXISTS trg_reviews_rating_extras_upd ON rate.reviews; +DROP TRIGGER IF EXISTS trg_reviews_rating_extras_del ON rate.reviews; +DROP TRIGGER IF EXISTS trg_review_flags_rating_extras_ins ON rate.review_flags; +DROP TRIGGER IF EXISTS trg_review_flags_rating_extras_upd ON rate.review_flags; +DROP TRIGGER IF EXISTS trg_review_flags_rating_extras_del ON rate.review_flags; +DROP TRIGGER IF EXISTS trg_master_products_rating_extras ON rate.master_products; + +-- 2. Functions. +DROP FUNCTION IF EXISTS rate.master_products_refresh_rating_extras(); +DROP FUNCTION IF EXISTS rate.review_flags_refresh_rating_extras(); +DROP FUNCTION IF EXISTS rate.reviews_refresh_rating_extras(); +DROP FUNCTION IF EXISTS rate.refresh_rating_extras_of_products(uuid[]); +DROP FUNCTION IF EXISTS rate.refresh_master_rating_extras(uuid); +DROP FUNCTION IF EXISTS rate.master_rating_extras(uuid); +DROP FUNCTION IF EXISTS rate.master_rating_item(uuid); + +-- 3. Tables (their policies and indexes go with them). +DROP TABLE IF EXISTS rate.product_points; +DROP TABLE IF EXISTS rate.review_points; + +-- 4. Columns (their CHECK constraints go with them). +ALTER TABLE rate.master_products + DROP COLUMN IF EXISTS criteria_scores, + DROP COLUMN IF EXISTS rating_weight_sum, + DROP COLUMN IF EXISTS rating_score; + +ALTER TABLE rate.reviews + DROP COLUMN IF EXISTS value_breakdown, + DROP COLUMN IF EXISTS value_score, + DROP COLUMN IF EXISTS criteria_version, + DROP COLUMN IF EXISTS verdict, + DROP COLUMN IF EXISTS rating_exact; + +COMMIT; diff --git a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings-verify.sql b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings-verify.sql new file mode 100644 index 000000000..9fe8ff517 --- /dev/null +++ b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings-verify.sql @@ -0,0 +1,68 @@ +-- PROPOSAL. Read-only checks for 2026-10-31-dos-me-criteria-ratings.sql (DOS/Rate#1543). Run +-- after it. `scripts/test-criteria-ratings-proposal.mjs` runs the expect-0 ones on the replay +-- stack. + +-- V1 Every new column exists. expect 0 +SELECT 10 - count(*) AS missing_columns +FROM information_schema.columns +WHERE table_schema = 'rate' + AND ((table_name = 'reviews' AND column_name IN ('rating_exact', 'verdict', 'criteria_version', + 'value_score', 'value_breakdown')) + OR (table_name = 'master_products' AND column_name IN ('rating_score', 'rating_weight_sum', + 'criteria_scores')) + OR (table_name = 'review_points' AND column_name = 'body') + OR (table_name = 'product_points' AND column_name = 'label')); + +-- V2 Both new tables have row level security on. expect 0 +SELECT 2 - count(*) AS tables_without_rls +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname = 'rate' AND c.relname IN ('review_points', 'product_points') + AND c.relrowsecurity; + +-- V3 Each new function exists once, and no client role can run it. expect 0 +-- Client roles: PUBLIC, anon, authenticated. +SELECT count(*) FILTER (WHERE to_regprocedure(w.signature) IS NULL) + + count(*) FILTER (WHERE to_regprocedure(w.signature) IS NOT NULL AND ( + has_function_privilege('anon', to_regprocedure(w.signature), 'EXECUTE') + OR has_function_privilege('authenticated', to_regprocedure(w.signature), 'EXECUTE') + OR EXISTS (SELECT 1 FROM pg_proc p, aclexplode(p.proacl) a + WHERE p.oid = to_regprocedure(w.signature) AND a.grantee = 0))) + AS missing_or_open_functions +FROM (VALUES ('rate.master_rating_item(uuid)'), ('rate.master_rating_extras(uuid)'), + ('rate.refresh_master_rating_extras(uuid)'), + ('rate.refresh_rating_extras_of_products(uuid[])'), + ('rate.reviews_refresh_rating_extras()'), + ('rate.review_flags_refresh_rating_extras()'), + ('rate.master_products_refresh_rating_extras()')) w(signature); + +-- V4 All seven triggers exist and are enabled. expect 0 +SELECT 7 - count(*) AS missing_triggers +FROM pg_trigger t JOIN pg_class c ON c.oid = t.tgrelid JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname = 'rate' AND NOT t.tgisinternal AND t.tgenabled <> 'D' + AND t.tgname IN ('trg_reviews_rating_extras_ins', 'trg_reviews_rating_extras_upd', + 'trg_reviews_rating_extras_del', 'trg_review_flags_rating_extras_ins', + 'trg_review_flags_rating_extras_upd', 'trg_review_flags_rating_extras_del', + 'trg_master_products_rating_extras'); + +-- V5 Every criterion score meets the display thresholds of D9. expect 0 +SELECT count(*) AS criteria_below_threshold +FROM rate.master_products m, jsonb_each(m.criteria_scores) c(key, v) +WHERE (c.v ->> 'writers')::int < 3 + OR (c.v ->> 'weight_sum')::numeric < 1.5 + OR (c.v ->> 'score')::numeric NOT BETWEEN 1 AND 5; + +-- V6 Every master holds what its reviews give today. expect 0 +-- No stale, missing or wrong rating or criteria: catches a skipped backfill and a missed trigger. +SELECT count(*) AS masters_out_of_date +FROM rate.master_products m +CROSS JOIN LATERAL rate.master_rating_extras(m.id) x +WHERE (m.rating_score, m.rating_weight_sum, m.criteria_scores) + IS DISTINCT FROM (x.rating_score, x.rating_weight_sum, x.criteria_scores); + +-- V7 Information: the rating each page will read, next to today's weighted mean. +SELECT m.slug, m.user_rating_weighted, m.rating_score, m.rating_weight_sum, + (SELECT count(*) FROM jsonb_object_keys(m.criteria_scores)) AS criteria_with_scores +FROM rate.master_products m +WHERE m.authentic_review_count > 0 +ORDER BY m.authentic_review_count DESC +LIMIT 20; diff --git a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.md b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.md new file mode 100644 index 000000000..b8c8bbe12 --- /dev/null +++ b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.md @@ -0,0 +1,86 @@ +# Proposal: DOS-Me migration for criteria ratings, pros and cons (DOS/Rate#1543) + +**Status:** proposal for JOY, **not applied**. A schema change on the shared Supabase: JOY approves and applies it through DOS-Me (preview, then beta, then prod). Nothing runs from DOS/Rate. + +Design: [`2026-10-31-criteria-stars-pros-cons.html`](./2026-10-31-criteria-stars-pros-cons.html), section 3; phase P2 of #1531, decisions D2, D3, D9 and D10 approved by JOY on 2026-10-31. + +## Files + +- [`2026-10-31-dos-me-criteria-ratings.sql`](./2026-10-31-dos-me-criteria-ratings.sql): the migration. Idempotent, in two transactions: the schema change first, then a backfill of the reviewed masters. +- [`2026-10-31-dos-me-criteria-ratings-verify.sql`](./2026-10-31-dos-me-criteria-ratings-verify.sql): read-only checks. V1 to V6 must return 0 after applying: + - V3: function grants for PUBLIC, anon and authenticated. + - V4: the triggers. + - V6: recomputes every master and compares it with what is stored. + - V7: lists `rating_score` next to today's `user_rating_weighted` for the 20 most-reviewed masters. +- [`2026-10-31-dos-me-criteria-ratings-rollback.sql`](./2026-10-31-dos-me-criteria-ratings-rollback.sql): drops everything the migration added. One transaction, re-runnable. +- [`scripts/test-criteria-ratings-proposal.mjs`](../../scripts/test-criteria-ratings-proposal.mjs): the CI replay in Docker, in four steps: + 1. **Reference stack:** apply twice; check the ratings, the triggers (exact star, a flag that leaves the rating, the weight cap) and the grants; roll back twice. + 2. **Before M1+M2:** apply, then run M0 and M1+M2. Two sizes become one Item, both variants show the pooled rating, and a vote on the Item path updates it. + 3. **M1+M2 rollback** with this migration in place. + 4. **After M1+M2:** M1+M2 again, then this migration on top, then the rollback. + +## What it adds + +- **`rate.reviews`:** + - `rating_exact numeric(2,1)`: the 0.1-step star a criteria review computes. `rating_star` stays the rounded integer that existing code reads. + - `verdict` (300 characters at most). + - `criteria_version`. + - `value_score` (0 to 100) and `value_breakdown`, for P6. +- **`rate.review_points`:** one pro or con line of a review, for the whole review or one criterion. + - Each line is 5 to 150 characters. + - `method` is `writer` or `ai`. + - `cluster_id` is filled by P5. + - Public read; only the service role writes. +- **`rate.product_points`:** the aggregated lines of a master, with support, writer count, an expert flag, and first and last seen dates. Public read; the service role and the worker (P5) write. +- **`rate.master_products`:** + - `rating_score`: the one rating of D3. It is a Bayesian average with k = 3 and mu = 4.2 (the same formula as the web's `computeBayesianRatingSummary`). Each review weight is capped at 3, and `rating_exact` is used when present. It is stored as `numeric(4,3)`, and criterion scores at 3 decimals too, so a page rounds once, to 1 decimal (4.249 shows 4.2). + - `rating_weight_sum`. + - `criteria_scores`: `{key: {score, weight_sum, writers}}`. A criterion appears only when all of these hold (D9): + - 3 or more reviews rated it; + - each rating is a number from 1 to 5, under a key shaped like the shared-data keys; + - the reviews' weight sum is 1.5 or more. + + Readers apply the ranking threshold of 5 from `writers`. +- **Which reviews count:** the same reviews `refresh_master_product_rating` counts. + - Before M1+M2: the master's own listings. + - Once the master belongs to an Item: every listing of the Item, and the result is written to every variant, as `refresh_item_rating` does. + - The `item_id` columns are read dynamically, so the same functions work with or without M1+M2. +- **Functions:** + - `rate.master_rating_extras(master)` computes the values. It is read-only; the verify file uses it. + - `rate.refresh_master_rating_extras(master)` writes them, only where they changed. + - Only the service role can run them. +- **Triggers keep the values current.** + - On `rate.reviews` (a change of a rating input: star, exact star, criteria, weight, type, origin or listing) and `rate.review_flags` (`include_in_rating`), for every insert, update and delete. They run once per statement; see Known limits for bulk flag imports. + - On `rate.master_products`: a change of `user_rating_weighted` or `authentic_review_count`, for example after a listing moves. + - **No existing function is replaced or renamed.** A later `CREATE OR REPLACE` of a refresh function (M1+M2, its rollback, a hotfix) cannot stop these updates. + +## Order of operations + +1. **Apply order:** before or after M0 and M1+M2. The replay covers both orders and the M1+M2 rollback. +2. **Apply on preview:** + - The schema transaction takes ACCESS EXCLUSIVE locks on `rate.reviews` and `rate.master_products` only for the `ALTER TABLE` statements. The new CHECKs are added NOT VALID, so nothing is scanned under that lock. + - The backfill transaction validates the CHECKs (a scan that lets reads and writes go on) and refreshes the reviewed masters, locking only those rows. + - Run the verify file: V1 to V6 must return 0. + - Compare V7: `rating_score` sits a little below `user_rating_weighted` for masters with few reviews, because of the prior. That is intended (D3). +3. **Then beta, then prod**, running the verify file after each. +4. **If M1+M2 is ever rolled back while this is in place:** run the backfill again (section 8 of the migration, its last transaction), then V6. Variants whose reviews did not change keep the Item's pooled values until then; V6 lists them. +5. **Next phases:** + - P4 (#1545) switches the web to `rating_score`. + - P3 (#1544) writes the new review columns and lines. + - P5 (#1546) writes `product_points`. + +## Rollback + +`2026-10-31-dos-me-criteria-ratings-rollback.sql` drops the triggers, the functions, both tables and the new columns. It is safe before or after M1+M2, because the migration replaced nothing. Data written only into these objects is lost; nothing older depends on it. + +Roll back only while no phase reads or writes the new columns: before P3, P4 and P5, or after reverting them. + +## Known limits + +- **Bulk flag imports.** The existing `trg_review_flags_sync_weight` updates `rate.reviews` one row at a time. A bulk flag import therefore refreshes a master once per flag row, not once per import. Re-crawls that rewrite reviews unchanged refresh nothing: only changes to rating inputs count. +- **Concurrent writers.** A variant locked by another transaction is skipped, as `refresh_item_rating` does. This avoids deadlocks between sibling variants. The other transaction writes its own values when it reaches that variant. Like the live refresh, two writers on the same master can leave the older values until the next write; V6 shows any master out of date. +- **Deleting a listing with its reviews** leaves the master's values in place until its next review write or refresh, the same as `user_rating_weighted` today. V6 shows it. Re-run the backfill to fix it. + +- **`rate_worker` can write the new columns directly.** It keeps its table-level UPDATE on `master_products` and on crawled reviews (20261018183000). The next trigger run overwrites such a write, and V6 shows it. Column-level grants are a separate hardening step. +- **Per-category prior means:** version 1 uses 4.2 for every criterion. +- **Not in this migration:** `submit_community_review` v2 (P3) and the value score weight (P6). Each comes with its own phase. diff --git a/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql new file mode 100644 index 000000000..c783187c9 --- /dev/null +++ b/Docs/Proposals/2026-10-31-dos-me-criteria-ratings.sql @@ -0,0 +1,469 @@ +-- PROPOSAL, NOT APPLIED. DOS-Me migration for phase P2 of the criteria ratings design +-- (DOS/Rate#1543; design Docs/Proposals/2026-10-31-criteria-stars-pros-cons.html, section 3; +-- decisions D2, D3, D9, D10 approved by JOY on 2026-10-31). JOY reviews and applies it through +-- DOS-Me (preview, then beta, then prod). Zero DDL on prod from DOS/Rate. +-- Rollback: 2026-10-31-dos-me-criteria-ratings-rollback.sql. +-- Verification: 2026-10-31-dos-me-criteria-ratings-verify.sql. +-- Replay test: scripts/test-criteria-ratings-proposal.mjs (Docker, never a real database). +-- +-- What it adds: +-- rate.reviews rating_exact (the 0.1 star a criteria review computes), verdict, +-- criteria_version, value_score, value_breakdown +-- rate.review_points one pro or con line of a review, optionally for one criterion +-- rate.product_points the aggregated pro and con lines of a master (written by P5) +-- rate.master_products rating_score (the one rating of D3), rating_weight_sum, +-- criteria_scores (per criterion, only when D9's thresholds are met) +-- rate.master_rating_extras(p_master_id) computes the three master columns (read-only); +-- rate.refresh_master_rating_extras(p_master_id) writes them, to the master or, once M1+M2 +-- is applied, to every variant of its Item (the pooled rating, like refresh_item_rating). +-- Triggers on rate.reviews, rate.review_flags and rate.master_products keep them current. +-- No existing function is replaced or renamed. +-- +-- Order: before or after M0 and M1+M2; the replay test applies it both ways, and through the +-- M1+M2 rollback. +-- +-- Idempotent: IF NOT EXISTS everywhere, constraints guarded by pg_constraint, CREATE OR +-- REPLACE for functions, DROP ... IF EXISTS before each policy and trigger. Two transactions: +-- the schema change, then the backfill (section 8), which writes only the three new master +-- columns of reviewed masters. + +BEGIN; + +-- 1. rate.reviews ---------------------------------------------------------------------------- + +ALTER TABLE rate.reviews + ADD COLUMN IF NOT EXISTS rating_exact numeric(2, 1), + ADD COLUMN IF NOT EXISTS verdict text, + ADD COLUMN IF NOT EXISTS criteria_version integer, + ADD COLUMN IF NOT EXISTS value_score numeric(5, 2), + ADD COLUMN IF NOT EXISTS value_breakdown jsonb; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'reviews_rating_exact_range') THEN + ALTER TABLE rate.reviews ADD CONSTRAINT reviews_rating_exact_range + CHECK (rating_exact IS NULL OR (rating_exact >= 1 AND rating_exact <= 5)) NOT VALID; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'reviews_verdict_length') THEN + ALTER TABLE rate.reviews ADD CONSTRAINT reviews_verdict_length + CHECK (verdict IS NULL OR char_length(verdict) <= 300) NOT VALID; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'reviews_criteria_version_positive') THEN + ALTER TABLE rate.reviews ADD CONSTRAINT reviews_criteria_version_positive + CHECK (criteria_version IS NULL OR criteria_version >= 1) NOT VALID; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'reviews_value_score_range') THEN + ALTER TABLE rate.reviews ADD CONSTRAINT reviews_value_score_range + CHECK (value_score IS NULL OR (value_score >= 0 AND value_score <= 100)) NOT VALID; + END IF; +END $$; + +-- 2. rate.review_points ------------------------------------------------------------------------ + +CREATE TABLE IF NOT EXISTS rate.review_points ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + review_id uuid NOT NULL REFERENCES rate.reviews(id) ON DELETE CASCADE, + -- NULL: a pro or con of the whole review; otherwise a key of + -- packages/shared-data/src/rules/rating-criteria.ts. + criterion_key text CHECK (criterion_key IS NULL OR criterion_key ~ '^[a-z_]{2,32}$'), + kind text NOT NULL CHECK (kind IN ('pro', 'con')), + body text NOT NULL CHECK (char_length(btrim(body)) BETWEEN 5 AND 150), + position smallint NOT NULL DEFAULT 0 CHECK (position BETWEEN 0 AND 19), + -- Set by the P5 aggregation; a line belongs to at most one cluster. + cluster_id uuid, + -- 'writer' for the review's own text, 'ai' for a line P5 extracts from a marketplace review. + method text NOT NULL DEFAULT 'writer' CHECK (method IN ('writer', 'ai')), + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS review_points_review_idx ON rate.review_points (review_id); +CREATE INDEX IF NOT EXISTS review_points_cluster_idx ON rate.review_points (cluster_id) + WHERE cluster_id IS NOT NULL; + +-- 3. rate.product_points ----------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS rate.product_points ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + master_id uuid NOT NULL REFERENCES rate.master_products(id) ON DELETE CASCADE, + criterion_key text CHECK (criterion_key IS NULL OR criterion_key ~ '^[a-z_]{2,32}$'), + kind text NOT NULL CHECK (kind IN ('pro', 'con')), + label text NOT NULL CHECK (char_length(btrim(label)) BETWEEN 3 AND 200), + -- Sum of the trust weights of the distinct writers behind the line, decayed by age. + support numeric(8, 2) NOT NULL DEFAULT 0 CHECK (support >= 0), + reviewer_count integer NOT NULL DEFAULT 0 CHECK (reviewer_count >= 0), + has_expert boolean NOT NULL DEFAULT false, + first_seen timestamptz NOT NULL DEFAULT now(), + last_seen timestamptz NOT NULL DEFAULT now(), + method text NOT NULL DEFAULT 'code' CHECK (method IN ('code', 'ai')), + updated_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS product_points_master_idx + ON rate.product_points (master_id, kind, support DESC); + +-- 4. RLS: public read; service role writes both, the worker writes the aggregate -------------- + +ALTER TABLE rate.review_points ENABLE ROW LEVEL SECURITY; +ALTER TABLE rate.product_points ENABLE ROW LEVEL SECURITY; + +DROP POLICY IF EXISTS "Public can view review_points" ON rate.review_points; +CREATE POLICY "Public can view review_points" ON rate.review_points FOR SELECT USING (true); +DROP POLICY IF EXISTS "Service role write review_points" ON rate.review_points; +CREATE POLICY "Service role write review_points" ON rate.review_points + FOR ALL TO service_role USING (true) WITH CHECK (true); + +DROP POLICY IF EXISTS "Public can view product_points" ON rate.product_points; +CREATE POLICY "Public can view product_points" ON rate.product_points FOR SELECT USING (true); +DROP POLICY IF EXISTS "Service role write product_points" ON rate.product_points; +CREATE POLICY "Service role write product_points" ON rate.product_points + FOR ALL TO service_role USING (true) WITH CHECK (true); +DROP POLICY IF EXISTS "Worker writes product_points" ON rate.product_points; +CREATE POLICY "Worker writes product_points" ON rate.product_points + FOR ALL TO rate_worker USING (true) WITH CHECK (true); + +GRANT SELECT ON rate.review_points, rate.product_points TO anon, authenticated; +GRANT ALL ON rate.review_points, rate.product_points TO service_role; +-- The importer's P5 job writes the aggregate; review lines come only through the review RPC. +GRANT SELECT, INSERT, UPDATE, DELETE ON rate.product_points TO rate_worker; +GRANT SELECT ON rate.review_points TO rate_worker; + +-- 5. rate.master_products ---------------------------------------------------------------------- + +ALTER TABLE rate.master_products + ADD COLUMN IF NOT EXISTS rating_score numeric(4, 3), + ADD COLUMN IF NOT EXISTS rating_weight_sum numeric(10, 2) NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS criteria_scores jsonb NOT NULL DEFAULT '{}'::jsonb; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'master_products_rating_score_range') THEN + ALTER TABLE rate.master_products ADD CONSTRAINT master_products_rating_score_range + CHECK (rating_score IS NULL OR (rating_score >= 0 AND rating_score <= 5)) NOT VALID; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'master_products_rating_weight_sum_nonneg') THEN + ALTER TABLE rate.master_products ADD CONSTRAINT master_products_rating_weight_sum_nonneg + CHECK (rating_weight_sum >= 0) NOT VALID; + END IF; +END $$; + +-- 6. The rating and criteria scores of a master (D2, D3, D9) ----------------------------------- +-- +-- The reviews rate.refresh_master_product_rating counts: user reviews of crawled listings, +-- crawl or community, included in the rating, with a weight above 0. Their scope follows the +-- live refresh: the master's own listings (cluster_id) before M1+M2, the whole Item's listings +-- (item_id) once the master belongs to an Item, so every variant shows the Item's pooled +-- rating, as refresh_item_rating does for user_rating_weighted. The item_id columns are read +-- dynamically, so the same functions work with and without M1+M2. +-- +-- rating_score = (k * mu + sum(w * star)) / (k + sum(w)), k = 3, mu = 4.2, each weight capped +-- at 3 and the star = rating_exact when a criteria review set it, else +-- rating_star (apps/web/src/lib/rate-domain-core.ts computeBayesianRatingSummary). +-- NULL with no weight, so a page shows "no rating yet" instead of the prior. +-- Stored at 3 decimals (scores too), so a page rounds once, to 1 decimal. +-- criteria = the same estimate per criterion of criteria_ratings, over the reviews that +-- rated it with a number from 1 to 5 under a key shaped like the shared-data +-- keys (^[a-z_]{2,32}$); kept only with 3 or more reviews and a weight sum of +-- 1.5 or more (D9). "writers" lets a reader apply the ranking threshold (5). A +-- Rate user writes one review per product, so a review is a writer. The prior +-- mean is 4.2 for every criterion until per-category means exist. + +-- The Item a master belongs to, or NULL (always NULL before M1+M2). +CREATE OR REPLACE FUNCTION rate.master_rating_item(p_master_id uuid) +RETURNS uuid +LANGUAGE plpgsql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_item uuid; +BEGIN + IF EXISTS (SELECT 1 FROM pg_catalog.pg_attribute + WHERE attrelid = 'rate.master_products'::pg_catalog.regclass + AND attname = 'item_id' AND NOT attisdropped) THEN + EXECUTE 'SELECT item_id FROM rate.master_products WHERE id = $1' INTO v_item USING p_master_id; + END IF; + RETURN v_item; +END $$; + +-- What the master's rating columns should hold. Read-only: the verify file compares it with +-- the stored columns. +CREATE OR REPLACE FUNCTION rate.master_rating_extras(p_master_id uuid) +RETURNS TABLE (rating_score numeric, rating_weight_sum numeric, criteria_scores jsonb) +LANGUAGE plpgsql +STABLE +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_item uuid := rate.master_rating_item(p_master_id); +BEGIN + RETURN QUERY EXECUTE format($q$ + WITH eligible AS ( + SELECT LEAST(3.0, r.helpfulness_weight)::numeric AS w, + COALESCE(r.rating_exact, r.rating_star)::numeric AS star, + r.criteria_ratings + FROM rate.products p + JOIN rate.reviews r ON r.product_id = p.id + JOIN rate.review_flags f ON f.review_id = r.id + WHERE %s = $1 + AND p.origin = 'crawl' + AND r.origin IN ('crawl', 'community') + AND r.reviewer_type = 'user' + AND COALESCE(f.include_in_rating, true) + AND r.helpfulness_weight > 0 + ), + overall AS ( + SELECT round(COALESCE(sum(w), 0), 2) AS weight_sum, + CASE WHEN sum(w) > 0 THEN round((3 * 4.2 + sum(w * star)) / (3 + sum(w)), 3) END AS score + FROM eligible + WHERE star BETWEEN 1 AND 5 + ), + per_criterion AS ( + SELECT c.key, + count(*) AS writers, + round(sum(e.w), 2) AS weight_sum, + round((3 * 4.2 + sum(e.w * (c.value #>> '{}')::numeric)) / (3 + sum(e.w)), 3) AS score + FROM eligible e + CROSS JOIN LATERAL jsonb_each( + CASE WHEN jsonb_typeof(e.criteria_ratings) = 'object' THEN e.criteria_ratings + ELSE '{}'::jsonb END + ) AS c(key, value) + WHERE c.key ~ '^[a-z_]{2,32}$' + AND jsonb_typeof(c.value) = 'number' + AND (c.value #>> '{}')::numeric BETWEEN 1 AND 5 + GROUP BY c.key + ) + SELECT o.score, + o.weight_sum, + COALESCE( + (SELECT jsonb_object_agg(pc.key, jsonb_build_object( + 'score', pc.score, 'weight_sum', pc.weight_sum, 'writers', pc.writers)) + FROM per_criterion pc + WHERE pc.writers >= 3 AND pc.weight_sum >= 1.5), + '{}'::jsonb) + FROM overall o + $q$, CASE WHEN v_item IS NULL THEN 'p.cluster_id' ELSE 'p.item_id' END) + USING COALESCE(v_item, p_master_id); +END $$; + +-- Writes them, to the master or to every variant of its Item, only where they changed. +CREATE OR REPLACE FUNCTION rate.refresh_master_rating_extras(p_master_id uuid) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_item uuid := rate.master_rating_item(p_master_id); + v_score numeric; + v_weight numeric; + v_criteria jsonb; +BEGIN + SELECT x.rating_score, x.rating_weight_sum, x.criteria_scores + INTO v_score, v_weight, v_criteria + FROM rate.master_rating_extras(p_master_id) x; + -- Variants in id order, skipping one another transaction holds, as refresh_item_rating + -- does: that transaction is refreshing the same Item and writes when it gets there, so two + -- refreshes of sibling variants never wait on each other (no deadlock). + EXECUTE format($q$ + UPDATE rate.master_products m + SET rating_score = $2, rating_weight_sum = $3, criteria_scores = $4 + WHERE m.id IN ( + SELECT mp.id FROM rate.master_products mp + WHERE %s = $1 + AND (mp.rating_score, mp.rating_weight_sum, mp.criteria_scores) + IS DISTINCT FROM ($2, $3, $4) + ORDER BY mp.id + FOR UPDATE SKIP LOCKED) + $q$, CASE WHEN v_item IS NULL THEN 'mp.id' ELSE 'mp.item_id' END) + USING COALESCE(v_item, p_master_id), v_score, v_weight, v_criteria; +END $$; + +REVOKE EXECUTE ON FUNCTION rate.master_rating_item(uuid) FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.master_rating_extras(uuid) FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.refresh_master_rating_extras(uuid) FROM PUBLIC, anon, authenticated; +GRANT EXECUTE ON FUNCTION rate.master_rating_item(uuid) TO service_role; +GRANT EXECUTE ON FUNCTION rate.master_rating_extras(uuid) TO service_role; +GRANT EXECUTE ON FUNCTION rate.refresh_master_rating_extras(uuid) TO service_role; + +-- 7. Keep them current: triggers on what they read -------------------------------------------- +-- +-- No refresh function is replaced or renamed, so a later CREATE OR REPLACE of +-- refresh_master_product_rating or refresh_item_rating (M1+M2, its rollback, a hotfix) cannot +-- drop these columns' updates. The triggers fire on: +-- rate.reviews every insert, update and delete (star, exact star, criteria, weight) +-- rate.review_flags every insert, update and delete (include_in_rating) +-- rate.master_products a change of user_rating_weighted or authentic_review_count, which the +-- live refresh writes after a listing moves between masters or Items +-- Statement-level with transition tables for the two review tables, and only rows whose rating +-- inputs changed count, so a re-crawl that rewrites reviews unchanged refreshes nothing. A bulk +-- flag import still refreshes a master once per flag row: the existing trg_review_flags_sync_weight +-- updates rate.reviews one row at a time (20261014033000). + +-- The masters whose ratings a set of listings feeds: each listing's master, and once M1+M2 is +-- in, one variant of the Item of a listing with no variant (cluster_id NULL, item_id set). +-- Each Item is refreshed once, whichever of its variants comes first. +CREATE OR REPLACE FUNCTION rate.refresh_rating_extras_of_products(p_products uuid[]) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + v_masters uuid[]; + v_extra uuid[]; + v_master uuid; + v_item uuid; + v_done uuid[] := '{}'; +BEGIN + IF p_products IS NULL OR cardinality(p_products) = 0 THEN + RETURN; + END IF; + v_masters := ARRAY(SELECT DISTINCT p.cluster_id FROM rate.products p + WHERE p.id = ANY (p_products) AND p.cluster_id IS NOT NULL); + IF EXISTS (SELECT 1 FROM pg_catalog.pg_attribute + WHERE attrelid = 'rate.products'::pg_catalog.regclass + AND attname = 'item_id' AND NOT attisdropped) THEN + EXECUTE $q$ + SELECT ARRAY( + SELECT DISTINCT ON (p.item_id) m.id + FROM rate.products p + JOIN rate.master_products m ON m.item_id = p.item_id + WHERE p.id = ANY ($1) AND p.cluster_id IS NULL AND p.item_id IS NOT NULL + ORDER BY p.item_id, m.id) + $q$ INTO v_extra USING p_products; + v_masters := v_masters || v_extra; + END IF; + FOR v_master IN SELECT DISTINCT unnest(v_masters) ORDER BY 1 LOOP + v_item := rate.master_rating_item(v_master); + IF v_item IS NOT NULL THEN + CONTINUE WHEN v_item = ANY (v_done); + v_done := v_done || v_item; + END IF; + PERFORM rate.refresh_master_rating_extras(v_master); + END LOOP; +END $$; + +-- Only rows whose rating inputs changed count, so a re-crawl that rewrites a review unchanged +-- (ON CONFLICT DO UPDATE) refreshes nothing. +CREATE OR REPLACE FUNCTION rate.reviews_refresh_rating_extras() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + PERFORM rate.refresh_rating_extras_of_products(ARRAY(SELECT product_id FROM changed_new)); + ELSIF TG_OP = 'UPDATE' THEN + PERFORM rate.refresh_rating_extras_of_products(ARRAY( + SELECT x.product_id + FROM changed_new n JOIN changed_old o ON o.id = n.id, + LATERAL (VALUES (n.product_id), (o.product_id)) x(product_id) + WHERE (n.product_id, n.rating_star, n.rating_exact, n.criteria_ratings, + n.helpfulness_weight, n.reviewer_type, n.origin) + IS DISTINCT FROM + (o.product_id, o.rating_star, o.rating_exact, o.criteria_ratings, + o.helpfulness_weight, o.reviewer_type, o.origin))); + ELSE + PERFORM rate.refresh_rating_extras_of_products(ARRAY(SELECT product_id FROM changed_old)); + END IF; + RETURN NULL; +END $$; + +CREATE OR REPLACE FUNCTION rate.review_flags_refresh_rating_extras() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + PERFORM rate.refresh_rating_extras_of_products(ARRAY( + SELECT r.product_id FROM changed_new f JOIN rate.reviews r ON r.id = f.review_id)); + ELSIF TG_OP = 'UPDATE' THEN + PERFORM rate.refresh_rating_extras_of_products(ARRAY( + SELECT r.product_id + FROM changed_new n JOIN changed_old o ON o.review_id = n.review_id + JOIN rate.reviews r ON r.id = n.review_id + WHERE n.include_in_rating IS DISTINCT FROM o.include_in_rating)); + ELSE + -- A flag deleted with its review (ON DELETE CASCADE) finds no review here; the reviews + -- trigger refreshes that master. + PERFORM rate.refresh_rating_extras_of_products(ARRAY( + SELECT r.product_id FROM changed_old f JOIN rate.reviews r ON r.id = f.review_id)); + END IF; + RETURN NULL; +END $$; + +CREATE OR REPLACE FUNCTION rate.master_products_refresh_rating_extras() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +BEGIN + PERFORM rate.refresh_master_rating_extras(NEW.id); + RETURN NULL; +END $$; + +REVOKE EXECUTE ON FUNCTION rate.refresh_rating_extras_of_products(uuid[]) FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.reviews_refresh_rating_extras() FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.review_flags_refresh_rating_extras() FROM PUBLIC, anon, authenticated; +REVOKE EXECUTE ON FUNCTION rate.master_products_refresh_rating_extras() FROM PUBLIC, anon, authenticated; + +DROP TRIGGER IF EXISTS trg_reviews_rating_extras_ins ON rate.reviews; +CREATE TRIGGER trg_reviews_rating_extras_ins AFTER INSERT ON rate.reviews + REFERENCING NEW TABLE AS changed_new + FOR EACH STATEMENT EXECUTE FUNCTION rate.reviews_refresh_rating_extras(); +DROP TRIGGER IF EXISTS trg_reviews_rating_extras_upd ON rate.reviews; +CREATE TRIGGER trg_reviews_rating_extras_upd AFTER UPDATE ON rate.reviews + REFERENCING OLD TABLE AS changed_old NEW TABLE AS changed_new + FOR EACH STATEMENT EXECUTE FUNCTION rate.reviews_refresh_rating_extras(); +DROP TRIGGER IF EXISTS trg_reviews_rating_extras_del ON rate.reviews; +CREATE TRIGGER trg_reviews_rating_extras_del AFTER DELETE ON rate.reviews + REFERENCING OLD TABLE AS changed_old + FOR EACH STATEMENT EXECUTE FUNCTION rate.reviews_refresh_rating_extras(); + +DROP TRIGGER IF EXISTS trg_review_flags_rating_extras_ins ON rate.review_flags; +CREATE TRIGGER trg_review_flags_rating_extras_ins AFTER INSERT ON rate.review_flags + REFERENCING NEW TABLE AS changed_new + FOR EACH STATEMENT EXECUTE FUNCTION rate.review_flags_refresh_rating_extras(); +DROP TRIGGER IF EXISTS trg_review_flags_rating_extras_upd ON rate.review_flags; +CREATE TRIGGER trg_review_flags_rating_extras_upd AFTER UPDATE ON rate.review_flags + REFERENCING OLD TABLE AS changed_old NEW TABLE AS changed_new + FOR EACH STATEMENT EXECUTE FUNCTION rate.review_flags_refresh_rating_extras(); +DROP TRIGGER IF EXISTS trg_review_flags_rating_extras_del ON rate.review_flags; +CREATE TRIGGER trg_review_flags_rating_extras_del AFTER DELETE ON rate.review_flags + REFERENCING OLD TABLE AS changed_old + FOR EACH STATEMENT EXECUTE FUNCTION rate.review_flags_refresh_rating_extras(); + +DROP TRIGGER IF EXISTS trg_master_products_rating_extras ON rate.master_products; +CREATE TRIGGER trg_master_products_rating_extras + AFTER UPDATE OF user_rating_weighted, authentic_review_count ON rate.master_products + FOR EACH ROW + WHEN ((OLD.user_rating_weighted, OLD.authentic_review_count) + IS DISTINCT FROM (NEW.user_rating_weighted, NEW.authentic_review_count)) + EXECUTE FUNCTION rate.master_products_refresh_rating_extras(); + +COMMIT; + +-- 8. Backfill, in its own transaction after the schema change commits, so the ALTER TABLE +-- locks are already released and only the touched master rows are locked. Masters with no +-- counted review already hold the right values (NULL, 0, {}) from the column defaults. +-- Re-runnable; it writes only rows whose values differ. A catalog of many thousand reviewed +-- masters can run it in id ranges instead. + +-- The CHECKs above were added NOT VALID (no scan under the ACCESS EXCLUSIVE lock); validating +-- them here scans each table under a lock that lets reads and writes go on. +BEGIN; +ALTER TABLE rate.reviews VALIDATE CONSTRAINT reviews_rating_exact_range; +ALTER TABLE rate.reviews VALIDATE CONSTRAINT reviews_verdict_length; +ALTER TABLE rate.reviews VALIDATE CONSTRAINT reviews_criteria_version_positive; +ALTER TABLE rate.reviews VALIDATE CONSTRAINT reviews_value_score_range; +ALTER TABLE rate.master_products VALIDATE CONSTRAINT master_products_rating_score_range; +ALTER TABLE rate.master_products VALIDATE CONSTRAINT master_products_rating_weight_sum_nonneg; +SELECT rate.refresh_master_rating_extras(m.id) +FROM rate.master_products m +WHERE m.authentic_review_count > 0 OR m.user_rating_weighted > 0 +ORDER BY m.id; +COMMIT; diff --git a/scripts/test-criteria-ratings-proposal.mjs b/scripts/test-criteria-ratings-proposal.mjs new file mode 100644 index 000000000..6288df75a --- /dev/null +++ b/scripts/test-criteria-ratings-proposal.mjs @@ -0,0 +1,353 @@ +#!/usr/bin/env node +// Replays the DOS-Me criteria ratings PROPOSAL (DOS/Rate#1543, phase P2 of #1531) on the +// migration replay stack: a small catalog with criteria reviews, the proposal twice, the rating +// and criteria scores its triggers keep current (review writes, a weight change, a flag that +// leaves the rating), every verification query, the grants, the rollback twice; then the +// proposal BEFORE M0 and M1+M2 (an Item pools its variants' reviews), the M1+M2 rollback under +// it, and the proposal applied again AFTER M1+M2. Nothing touches a real database. +// +// Usage: node scripts/test-criteria-ratings-proposal.mjs (needs Docker) + +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +import { repoRoot, startRateDb } from './lib/rate-db.mjs'; + +const read = (name) => + readFileSync(path.join(repoRoot, 'Docs', 'Proposals', name), 'utf8'); +const proposal = read('2026-10-31-dos-me-criteria-ratings.sql'); +const rollback = read('2026-10-31-dos-me-criteria-ratings-rollback.sql'); +const verify = read('2026-10-31-dos-me-criteria-ratings-verify.sql'); +const m0 = read('2026-10-29-dos-me-m0-seed-rekey.sql'); +const m1m2 = read('2026-10-28-dos-me-m1-m2-items.sql'); +const m1m2Rollback = read('2026-10-28-dos-me-m1-m2-items-rollback.sql'); + +const container = `rate-criteria-proposal-${process.pid}`; +const removeContainer = () => + spawnSync('docker', ['rm', '--force', container], { timeout: 30_000 }); +for (const signal of ['SIGINT', 'SIGTERM']) { + process.on(signal, () => { + removeContainer(); + process.exit(130); + }); +} + +const expectZero = (file) => + [ + ...file.matchAll( + /^-- (V\d+) [^\n]*expect 0\n(?:--[^\n]*\n)*(SELECT[\s\S]*?;)/gm + ), + ].map(([, id, query]) => ({ id, query })); +const checks = expectZero(verify); +assert.equal(checks.length, 6, 'the verification lists its expect-0 queries'); + +const PHONES = 'c0000000-0000-4000-8000-000000000001'; +const MERCHANT = 'e0000000-0000-4000-8000-000000000001'; +const x128 = 'a0000000-0000-4000-8000-000000000128'; +const x256 = 'a0000000-0000-4000-8000-000000000256'; +const empty = 'a0000000-0000-4000-8000-0000000000ee'; +const listing128 = 'b0000000-0000-4000-8000-000000000128'; +const listing256 = 'b0000000-0000-4000-8000-000000000256'; +const review = (n) => `d0000000-0000-4000-8000-00000000000${n}`; + +try { + const { sql, asRole } = startRateDb({ name: container }); + const checkAll = (when) => { + for (const { id, query } of checks) + assert.equal(sql(query), '0', `${id} ${when}`); + }; + const row = (id) => + sql( + `SELECT coalesce(rating_score::text, '-') || '|' || rating_weight_sum || '|' || criteria_scores::text + FROM rate.master_products WHERE id = '${id}'` + ); + const weighted = (id) => + sql( + `SELECT user_rating_weighted FROM rate.master_products WHERE id = '${id}'` + ); + + sql(` + INSERT INTO rate.categories (id, name, slug) VALUES ('${PHONES}', 'Điện thoại', 'smartphones'); + INSERT INTO rate.merchants (id, platform, platform_shop_id, name, slug) + VALUES ('${MERCHANT}', 'shopee', 's1', 'Shop 1', 'shop-1'); + INSERT INTO rate.master_products (id, brand, model, model_key, name, slug, category_id, status, + storage_gb, min_price, max_price) + VALUES + ('${x128}', 'Acme', 'Phone X', 'acme:phone-x:128gb', 'Acme Phone X 128GB', + 'acme-phone-x-128gb', '${PHONES}', 'published', 128, 10000000, 10000000), + ('${x256}', 'Acme', 'Phone X', 'acme:phone-x:256gb', 'Acme Phone X 256GB', + 'acme-phone-x-256gb', '${PHONES}', 'published', 256, 12000000, 12000000), + ('${empty}', 'Acme', 'Phone Y', 'acme:phone-y:128gb', 'Acme Phone Y 128GB', + 'acme-phone-y-128gb', '${PHONES}', 'published', 128, 9000000, 9000000); + INSERT INTO rate.products (id, merchant_id, platform, platform_item_id, title, price_current, url, cluster_id) + VALUES + ('${listing128}', '${MERCHANT}', 'shopee', 'x128', 'Acme Phone X 128GB', 10000000, + 'https://shopee.vn/x128', '${x128}'), + ('${listing256}', '${MERCHANT}', 'shopee', 'x256', 'Acme Phone X 256GB', 12000000, + 'https://shopee.vn/x256', '${x256}'); + -- On the 128GB: three criteria reviews, one excluded from the rating, one marketplace + -- review with no criteria and weight 2. On the 256GB: one criteria review. + INSERT INTO rate.reviews (id, product_id, rating_star, comment, reviewer_type, criteria_ratings) + VALUES + ('${review(1)}', '${listing128}', 5, 'Màn hình đẹp, pin ổn.', 'user', + '{"display": 5, "value": 4, "Bad Key!": 5, "camera": "5"}'), + ('${review(2)}', '${listing128}', 4, 'Dùng tốt trong tầm giá.', 'user', '{"display": 4, "value": 4}'), + ('${review(3)}', '${listing128}', 3, 'Màn hơi tối ngoài nắng.', 'user', '{"display": 3}'), + ('${review(4)}', '${listing128}', 1, 'Giao sai màu.', 'user', '{"display": 1, "value": 1}'), + ('${review(5)}', '${listing128}', 5, 'Máy chạy mượt.', 'user', '{}'), + ('${review(6)}', '${listing256}', 2, 'Màn ám vàng.', 'user', '{"display": 2}'); + INSERT INTO rate.review_flags (review_id, verdict, include_in_rating, detector, detector_version, input_hash) + VALUES + ('${review(1)}', 'authentic', true, 'replay', 'v1', 'h1'), + ('${review(2)}', 'authentic', true, 'replay', 'v1', 'h2'), + ('${review(3)}', 'authentic', true, 'replay', 'v1', 'h3'), + ('${review(4)}', 'suspicious', false, 'replay', 'v1', 'h4'), + ('${review(5)}', 'authentic', true, 'replay', 'v1', 'h5'), + ('${review(6)}', 'authentic', true, 'replay', 'v1', 'h6'); + UPDATE rate.reviews SET helpfulness_weight = 1 WHERE id <> '${review(5)}'; + UPDATE rate.reviews SET helpfulness_weight = 2 WHERE id = '${review(5)}'; + SELECT rate.refresh_master_product_rating('${x128}'); + SELECT rate.refresh_master_product_rating('${x256}'); + `); + + // Per master (no Items yet). 128GB: weights 1 + 1 + 1 + 2 = 5 (review 4 is excluded), stars + // 5, 4.3 (exact), 3, 5: (3 * 4.2 + 22.3) / 8 = 4.3625, stored 4.363. display: 3 writers, weight 3: + // (12.6 + 12) / 6 = 4.100. value has 2 writers (hidden); a bad key and a string star are not + // criteria. 256GB: (12.6 + 2) / 4 = 3.650, one writer, no criteria. + const per128 = + '4.363|5.00|{"display": {"score": 4.100, "writers": 3, "weight_sum": 3.00}}'; + const per256 = '3.650|1.00|{}'; + + const applyAndCheckPerMaster = (when) => { + sql(proposal); + sql(proposal); + // A criteria review gets its exact star: the reviews trigger refreshes the master. + asRole( + 'rate_worker', + `UPDATE rate.reviews SET rating_exact = 4.3 WHERE id = '${review(2)}'` + ); + assert.equal(row(x128), per128, `128GB ${when}`); + assert.equal(row(x256), per256, `256GB ${when}`); + assert.equal(row(empty), '-|0.00|{}', `a master with no reviews ${when}`); + assert.equal( + weighted(x128), + '4.40', + `user_rating_weighted untouched ${when}` + ); + checkAll(`after applying ${when}`); + + // A flag that takes a review out of the rating refreshes it, and back. + sql( + `UPDATE rate.review_flags SET include_in_rating = false WHERE review_id = '${review(3)}'` + ); + assert.equal( + row(x128), + '4.557|4.00|{}', + `review 3 out of the rating ${when}` + ); + sql( + `UPDATE rate.review_flags SET include_in_rating = true WHERE review_id = '${review(3)}'` + ); + assert.equal(row(x128), per128, `review 3 back ${when}`); + + for (const fn of [ + `rate.refresh_master_rating_extras('${x128}')`, + `rate.master_rating_extras('${x128}')`, + ]) { + assert.throws( + () => asRole('anon', `SELECT * FROM ${fn}`), + /permission denied/, + `anon ${fn}` + ); + assert.throws( + () => asRole('rate_worker', `SELECT * FROM ${fn}`), + /permission denied/, + `worker ${fn}` + ); + } + assert.equal( + asRole('anon', 'SELECT count(*) FROM rate.review_points'), + '0', + `anon reads review_points ${when}` + ); + for (const role of ['anon', 'authenticated', 'rate_worker']) + assert.throws( + () => + asRole( + role, + `INSERT INTO rate.review_points (review_id, kind, body) VALUES ('${review(1)}', 'pro', 'Pin trâu')` + ), + /permission denied|row-level security/, + `${role} cannot write review_points ${when}` + ); + assert.throws( + () => + asRole( + 'anon', + `INSERT INTO rate.product_points (master_id, kind, label) VALUES ('${x128}', 'pro', 'Pin tốt')` + ), + /permission denied|row-level security/, + `anon cannot write product_points ${when}` + ); + asRole( + 'rate_worker', + `INSERT INTO rate.product_points (master_id, kind, label, support, reviewer_count) + VALUES ('${x128}', 'pro', 'Màn hình đẹp', 2, 2)` + ); + assert.throws( + () => + sql( + `INSERT INTO rate.review_points (review_id, kind, body) VALUES ('${review(1)}', 'pro', 'ok')` + ), + /review_points_body_check/, + `a pro line under 5 characters is refused ${when}` + ); + // A weight above 3 counts as 3: review 5 at weight 9 equals review 5 at weight 3. + sql( + `UPDATE rate.reviews SET helpfulness_weight = 9 WHERE id = '${review(5)}'` + ); + const capped = row(x128); + sql( + `UPDATE rate.reviews SET helpfulness_weight = 3 WHERE id = '${review(5)}'` + ); + assert.equal(row(x128), capped, `weight capped at 3 ${when}`); + sql( + `UPDATE rate.reviews SET helpfulness_weight = 2 WHERE id = '${review(5)}'` + ); + assert.equal(row(x128), per128, `weight restored ${when}`); + }; + + const rollBackAndCheck = (when) => { + sql(rollback); + sql(rollback); + assert.equal( + sql(`SELECT count(*) FROM information_schema.columns + WHERE table_schema = 'rate' + AND column_name IN ('rating_exact', 'rating_score', 'criteria_scores')`), + '0', + `columns dropped ${when}` + ); + assert.equal( + sql( + `SELECT count(*) FROM pg_trigger WHERE tgname LIKE 'trg_%rating_extras%'` + ), + '0', + `triggers dropped ${when}` + ); + assert.equal( + sql(`SELECT to_regclass('rate.review_points') IS NULL AND to_regclass('rate.product_points') IS NULL + AND to_regprocedure('rate.refresh_master_rating_extras(uuid)') IS NULL`), + 't', + `tables and functions dropped ${when}` + ); + asRole( + 'rate_worker', + `SELECT rate.refresh_master_product_rating('${x128}')` + ); + sql(`INSERT INTO rate.reviews (id, product_id, rating_star, comment, reviewer_type) + VALUES ('d0000000-0000-4000-8000-0000000000ff', '${listing256}', 4, 'Ổn.', 'user'); + DELETE FROM rate.reviews WHERE id = 'd0000000-0000-4000-8000-0000000000ff';`); + }; + + // 1. On the reference stack. + applyAndCheckPerMaster('on the reference stack'); + rollBackAndCheck('on the reference stack'); + + // 2. The proposal first, then M0 and M1+M2: the two sizes become one Item, and both + // variants show the Item's pooled rating. Weights 5 + 1 = 6, stars 22.3 + 2: + // (12.6 + 24.3) / 9 = 4.100. display: 4 writers, weight 4: (12.6 + 14) / 7 = 3.800. + applyAndCheckPerMaster('before M1+M2'); + sql(m0); + sql(m1m2); + const pooled = + '4.100|6.00|{"display": {"score": 3.800, "writers": 4, "weight_sum": 4.00}}'; + assert.equal( + sql( + `SELECT count(DISTINCT item_id) FROM rate.master_products WHERE id IN ('${x128}', '${x256}')` + ), + '1', + 'the two sizes share one Item' + ); + assert.equal(row(x128), pooled, '128GB shows the Item rating'); + assert.equal(row(x256), pooled, '256GB shows the Item rating'); + checkAll('after M1+M2 on top'); + // A vote's weight change on the Item path (refresh_item_rating, not the master refresh): + // review 5 at weight 1: weights 5, stars 5 + 4.3 + 3 + 5 + 2 = 19.3, (12.6 + 19.3) / 8 = 3.9875, stored 3.988. + sql( + `UPDATE rate.reviews SET helpfulness_weight = 1 WHERE id = '${review(5)}'` + ); + asRole('rate_worker', `SELECT rate.refresh_master_product_rating('${x256}')`); + const pooledAfterVote = + '3.988|5.00|{"display": {"score": 3.800, "writers": 4, "weight_sum": 4.00}}'; + assert.equal(row(x128), pooledAfterVote, '128GB after the vote'); + assert.equal(row(x256), pooledAfterVote, '256GB after the vote'); + checkAll('after a vote on the Item path'); + // A review on an Item listing with no variant (cluster_id NULL) refreshes the Item too: + // weights 5 + 1 = 6, stars 19.3 + 4: (12.6 + 23.3) / 9 = 3.989. + const noVariant = 'b0000000-0000-4000-8000-0000000000a0'; + const noVariantReview = 'd0000000-0000-4000-8000-0000000000a0'; + sql(` + INSERT INTO rate.products (id, merchant_id, platform, platform_item_id, title, price_current, url, + cluster_id, item_id) + SELECT '${noVariant}', '${MERCHANT}', 'shopee', 'xnv', 'Acme Phone X', 11000000, + 'https://shopee.vn/xnv', NULL, item_id + FROM rate.master_products WHERE id = '${x128}'; + INSERT INTO rate.reviews (id, product_id, rating_star, comment, reviewer_type) + VALUES ('${noVariantReview}', '${noVariant}', 4, 'Máy ổn.', 'user'); + INSERT INTO rate.review_flags (review_id, verdict, include_in_rating, detector, detector_version, input_hash) + VALUES ('${noVariantReview}', 'authentic', true, 'replay', 'v1', 'hnv'); + UPDATE rate.reviews SET helpfulness_weight = 1 WHERE id = '${noVariantReview}'; + `); + assert.equal( + sql( + `SELECT cluster_id IS NULL FROM rate.products WHERE id = '${noVariant}'` + ), + 't', + 'the listing has no variant' + ); + const withNoVariant = + '3.989|6.00|{"display": {"score": 3.800, "writers": 4, "weight_sum": 4.00}}'; + assert.equal(row(x128), withNoVariant, '128GB counts the no-variant review'); + assert.equal(row(x256), withNoVariant, '256GB counts the no-variant review'); + // A re-crawl that rewrites the review unchanged refreshes nothing; V6 stays 0 either way. + sql( + `UPDATE rate.reviews SET comment = comment WHERE id = '${noVariantReview}'` + ); + checkAll('with a no-variant review'); + sql(`DELETE FROM rate.reviews WHERE id = '${noVariantReview}'; + DELETE FROM rate.products WHERE id = '${noVariant}';`); + assert.equal(row(x128), pooledAfterVote, '128GB after the review is deleted'); + + // 3. The M1+M2 rollback under the proposal: the triggers stay and the 128GB, whose review + // changes, goes back to its own rating. The 256GB keeps the pooled values until something + // touches it: V6 shows it, and the runbook's step after an M1+M2 rollback (the backfill of + // section 8 again) brings it back. + sql(m1m2Rollback); + sql( + `UPDATE rate.reviews SET helpfulness_weight = 2 WHERE id = '${review(5)}'` + ); + assert.equal(row(x128), per128, '128GB after the M1+M2 rollback'); + const stale = checks.find(({ id }) => id === 'V6'); + assert.equal(sql(stale.query), '1', 'V6 shows the 256GB as out of date'); + const backfill = proposal.slice(proposal.lastIndexOf('BEGIN;')); + assert.match(backfill, /refresh_master_rating_extras/, 'the backfill step'); + sql(backfill); + assert.equal(row(x256), per256, '256GB after the M1+M2 rollback'); + checkAll('after the M1+M2 rollback'); + + // 4. M1+M2 again, then the proposal applied (again) after it: same pooled values. + sql(m1m2); + sql(proposal); + assert.equal(row(x128), pooled, '128GB, proposal after M1+M2'); + assert.equal(row(x256), pooled, '256GB, proposal after M1+M2'); + checkAll('with the proposal applied after M1+M2'); + rollBackAndCheck('after M1+M2'); + + console.log( + 'criteria ratings proposal: apply, triggers, Items, verify and rollback replayed' + ); +} finally { + removeContainer(); +}