diff --git a/apps/importer/src/shopee/entity-matcher.ts b/apps/importer/src/shopee/entity-matcher.ts index 2cde1198..cd4588a5 100644 --- a/apps/importer/src/shopee/entity-matcher.ts +++ b/apps/importer/src/shopee/entity-matcher.ts @@ -18,8 +18,10 @@ export class EntityMatcher { .eq('model_key', candidate.model_key) .maybeSingle(); + // Infra errors must abort ingestion (plan v7 §5.1): a failed lookup must + // never fall through to Tier 3 and silently split the canonical cluster. if (exactErr) { - console.warn(`[EntityMatcher] Tier 1 query warning: ${exactErr.message}`); + throw new Error(`[EntityMatcher] Tier 1 exact lookup failed: ${exactErr.message}`); } if (exactMatch) { @@ -36,7 +38,13 @@ export class EntityMatcher { p_similarity_threshold: 0.65, }); - if (!fuzzyErr && Array.isArray(fuzzyMatches) && fuzzyMatches.length > 0) { + // Same rule for Tier 2: RPC/infra failure aborts; only a genuine + // "no match above threshold" reaches Tier 3 (plan v7 §5.1). + if (fuzzyErr) { + throw new Error(`[EntityMatcher] Tier 2 fuzzy match failed: ${fuzzyErr.message}`); + } + let ambiguousFuzzy = false; + if (Array.isArray(fuzzyMatches) && fuzzyMatches.length > 0) { const topMatch = fuzzyMatches[0]; if (topMatch.similarity >= 0.70) { console.log( @@ -44,51 +52,72 @@ export class EntityMatcher { ); return { masterId: topMatch.master_id, isNew: false }; } + // Candidates exist but all below the accept threshold: identity is + // uncertain -> seed as draft (plan v7 §5.1), never auto-publish. + ambiguousFuzzy = true; + console.log(`[EntityMatcher] Tier 2 best similarity ${(topMatch.similarity * 100).toFixed(1)}% < 70%: seeding as draft if new.`); } - // Tier 3: No existing master product matches -> Seed a new Master Product - console.log(`[EntityMatcher] Tier 3: Creating new canonical Master Product: ${candidate.name}...`); - const { data: createdMaster, error: createErr } = await this.supabase - .from('master_products') - .insert({ - category_id: categoryId || null, - brand: candidate.brand, - model: candidate.model, - variant_name: candidate.variant_name, - model_key: candidate.model_key, - name: candidate.name, - slug: candidate.slug, - description: candidate.description || '', - ram_gb: candidate.ram_gb, - storage_gb: candidate.storage_gb, - screen_size_inch: candidate.screen_size_inch, - battery_mah: candidate.battery_mah, - has_5g: candidate.has_5g, - specifications: candidate.specifications, - key_specs: candidate.key_specs, - images: candidate.images, - thumbnail: candidate.images[0] || null, - colors: candidate.colors, - status: 'published', - }) - .select('id') - .single(); + // Tier 3: No existing master product matches -> Seed a new Master Product. + // - Upsert on model_key (unique) so a concurrent crawler inserting the same + // canonical entity resolves instead of crashing (plan v7 §5.1 / Gemini B3). + // - Low-confidence identities (unparseable storage etc.) are seeded as + // 'draft', never auto-published; the live status CHECK allows only + // draft/published/archived, so 'draft' is the needs-review state. + const seedStatus = candidate.needs_review || ambiguousFuzzy ? 'draft' : 'published'; + console.log(`[EntityMatcher] Tier 3: Creating new canonical Master Product: ${candidate.name} (status: ${seedStatus})...`); - if (createErr || !createdMaster) { - // In case of unique slug / model_key race condition, retry fetch - const { data: fallback } = await this.supabase - .from('master_products') - .select('id') - .eq('model_key', candidate.model_key) - .maybeSingle(); + // Insert-only with ignoreDuplicates: the LOSING worker must refetch the + // winner WITHOUT overwriting its canonical metadata or status (Codex + // blocker: a re-asserting upsert could re-publish a draft/archived master). + const { data: createdRows, error: createErr } = await this.supabase + .from('master_products') + .upsert( + { + category_id: categoryId || null, + brand: candidate.brand, + model: candidate.model, + variant_name: candidate.variant_name, + model_key: candidate.model_key, + name: candidate.name, + slug: candidate.slug, + description: candidate.description || '', + ram_gb: candidate.ram_gb, + storage_gb: candidate.storage_gb, + screen_size_inch: candidate.screen_size_inch, + battery_mah: candidate.battery_mah, + has_5g: candidate.has_5g, + specifications: candidate.specifications, + key_specs: candidate.key_specs, + images: candidate.images, + thumbnail: candidate.images[0] || null, + colors: candidate.colors, + status: seedStatus, + }, + { onConflict: 'model_key', ignoreDuplicates: true } + ) + .select('id'); - if (fallback) { - return { masterId: fallback.id, isNew: false }; - } - throw new Error(`Failed to create master product: ${createErr?.message}`); + if (createErr) { + throw new Error(`Failed to create master product: ${createErr.message}`); + } + if (Array.isArray(createdRows) && createdRows.length > 0) { + const createdMaster = createdRows[0]; + console.log(`[EntityMatcher] Successfully created Master Product ID: ${createdMaster.id}`); + return { masterId: createdMaster.id, isNew: true }; } - console.log(`[EntityMatcher] Successfully created Master Product ID: ${createdMaster.id}`); - return { masterId: createdMaster.id, isNew: true }; + // Lost the race: another worker inserted this model_key first. Adopt the + // winner as-is (its status/identity decisions stay authoritative). + const { data: winner, error: winnerErr } = await this.supabase + .from('master_products') + .select('id') + .eq('model_key', candidate.model_key) + .maybeSingle(); + if (winnerErr || !winner) { + throw new Error(`Master product race lost and winner fetch failed: ${winnerErr?.message}`); + } + console.log(`[EntityMatcher] Master product already existed (race): ${winner.id}`); + return { masterId: winner.id, isNew: false }; } } diff --git a/apps/importer/src/shopee/product-normalizer.ts b/apps/importer/src/shopee/product-normalizer.ts index 3b1e317c..c65d35d8 100644 --- a/apps/importer/src/shopee/product-normalizer.ts +++ b/apps/importer/src/shopee/product-normalizer.ts @@ -8,7 +8,8 @@ export interface NormalizedProductCandidate { model_key: string; name: string; slug: string; - storage_gb: number; + storage_gb: number | null; // null = unparseable from the title; identity carries ':unknown' + needs_review: boolean; // true when identity could not be confidently derived (plan v7 §5.2) ram_gb: number | null; screen_size_inch: number | null; battery_mah: number | null; @@ -149,15 +150,66 @@ export class ProductNormalizer { model = tokens.slice(0, 3).join(' '); } - // 3. Extract Storage - let storageGb = 128; // Default - const tbMatch = title.match(/(\d+)\s*TB/i); - const gbMatch = title.match(/(\d+)\s*GB/i); - if (tbMatch && tbMatch[1]) { - storageGb = parseInt(tbMatch[1], 10) * 1024; - } else if (gbMatch && gbMatch[1]) { - storageGb = parseInt(gbMatch[1], 10); + // 3. Extract Storage - RAM-aware (plan v7 §5.2, Codex-reproduced bug: + // "Samsung Galaxy A55 8GB RAM 256GB" must be 256GB, not the first GB token). + // Decimal separator: dot or comma (Vietnamese titles use "1,5TB"). + const sizeTokens = [...title.matchAll(/(\d+(?:[.,]\d+)?)\s*(TB|GB)/gi)].map((m) => ({ + value: parseFloat(m[1].replace(',', '.')) * (m[2].toUpperCase() === 'TB' ? 1024 : 1), + index: m.index ?? 0, + raw: m[0], + })); + // RAM-word matching (replaces per-pattern heuristics after two review + // rounds found holes: "8GB RAM 12GB RAM", "RAM 8GB RAM 12GB", the shared + // "RAM" in "8GB RAM 256GB", "dung luong 8GB bo nho RAM 256GB"). Rule: + // every literal "RAM" word claims the nearest UNCLAIMED size token - + // preferring the one BEFORE it (suffix " RAM"), else the one AFTER + // it ("RAM "). Any token left unclaimed is a storage candidate. + // A prefix claim requires adjacency: only whitespace between the token + // and the RAM phrase. "256GB - RAM 8GB" (Codex round-4) separates the ROM + // size with a dash - claiming 256 as RAM there would publish a wrong + // identity, so non-adjacent tokens are left for the storage side. Phrases + // like "bộ nhớ RAM" span their full length (Codex round-4 regression: + // "dung lượng 8GB bộ nhớ RAM 256GB" - the phrase's own words sit between + // 8GB and the literal RAM word, so adjacency is measured to the PHRASE + // start and 8GB claims correctly). + const ramPhrases = [...title.matchAll(/bộ\s*nhớ\s*RAM|dung\s*lượng\s*RAM|RAM\b/gi)].map((m) => m.index ?? 0); + const claimable = sizeTokens.map((t) => ({ ...t, claimed: false })); + for (const p of ramPhrases) { + const before = [...claimable] + .filter((t) => !t.claimed && t.index + t.raw.length <= p && /^\s*$/.test(title.slice(t.index + t.raw.length, p))) + .sort((a, b) => b.index - a.index)[0]; + const after = [...claimable].filter((t) => !t.claimed && t.index > p).sort((a, b) => a.index - b.index)[0]; + const pick = before ?? after; + if (pick) pick.claimed = true; } + const storageCandidates = claimable.filter((t) => !t.claimed); + + // ROM/storage markers, two confidence tiers (Codex round-6): an EXPLICIT + // "ROM"/"bộ nhớ trong" marker always wins - even when a separate RAM + // phrase follows ("ROM 256GB RAM 8GB" = storage 256, RAM 8). The LOOSE + // "dung lượng" marker is only trusted when not immediately followed by a + // RAM phrase ("dung lượng 8GB RAM 256GB" / "dung lượng 8GB dung lượng RAM + // 256GB" mean 8 is RAM, not storage). + const romExplicit = title.match(/(?:ROM|bộ nhớ trong)\s*[:\-]?\s*(\d+(?:[.,]\d+)?)\s*(TB|GB)/i); + const romLoose = title.match( + /dung lượng\s*[:\-]?\s*(\d+(?:[.,]\d+)?)\s*(TB|GB)(?!\s*(?:RAM|bộ\s*nhớ\s*RAM|dung\s*lượng\s*RAM))/i + ); + + let storageGb: number | null = null; + let storageConfident = false; + const marker = romExplicit ?? romLoose; + if (marker) { + // An explicit ROM marker always wins; the loose "dung lượng" marker has + // already been rejected when a RAM phrase follows its size. + storageGb = parseFloat(marker[1].replace(',', '.')) * (marker[2].toUpperCase() === 'TB' ? 1024 : 1); + storageConfident = true; + } else if (storageCandidates.length === 1) { + storageGb = storageCandidates[0].value; + storageConfident = true; + } + // Ambiguous (multiple unclaimed tokens, no ROM anchor) or nothing found: + // leave storageGb null -> ':unknown' identity + needs_review (never guess). + const storageKnown = storageGb !== null; // 4. Extract Colors from models / variations const colors: string[] = []; @@ -169,17 +221,22 @@ export class ProductNormalizer { } } - // 5. Construct keys & slugs + // 5. Construct keys & slugs. Unknown storage keeps identity space explicit + // (':unknown') instead of guessing a default (plan v7 §5.2). const baseKey = `${slugify(brand)}:${slugify(model)}`; - const modelKey = `${baseKey}:${storageGb}gb`; - const storageDisplay = storageGb >= 1024 ? `${storageGb / 1024}TB` : `${storageGb}GB`; - const canonicalName = `${brand} ${model} ${storageDisplay}`; + const modelKey = storageKnown ? `${baseKey}:${storageGb}gb` : `${baseKey}:unknown`; + const storageDisplay = storageGb === null + ? 'Không rõ dung lượng' + : storageGb >= 1024 + ? `${storageGb / 1024}TB` + : `${storageGb}GB`; + const canonicalName = storageKnown ? `${brand} ${model} ${storageDisplay}` : `${brand} ${model}`; const slug = slugify(canonicalName); // 6. Enrich with 10 Field Groups const benchmark = CANONICAL_PHONE_REGISTRY[baseKey]; - const ramGb = benchmark ? benchmark.ram_gb : null; + const ramGb = benchmark ? benchmark.ram_gb : claimable.find((t) => t.claimed)?.value ?? null; const screenSize = benchmark ? benchmark.screen_size_inch : null; const batteryMah = benchmark ? benchmark.battery_mah : null; const has5g = benchmark ? benchmark.has_5g : true; @@ -213,7 +270,7 @@ export class ProductNormalizer { cpu: benchmark?.cpu || undefined, gpu: benchmark?.gpu || undefined, ram_gb: ramGb || undefined, - storage_gb: storageGb, + storage_gb: storageGb !== null ? storageGb : undefined, storage_expansion: 'Không hỗ trợ thẻ nhớ', os: benchmark?.os || 'iOS 18', }, @@ -277,7 +334,8 @@ export class ProductNormalizer { model_key: modelKey, name: canonicalName, slug, - storage_gb: storageGb, + storage_gb: storageKnown ? storageGb : null, + needs_review: !storageConfident, ram_gb: ramGb, screen_size_inch: screenSize, battery_mah: batteryMah, diff --git a/apps/importer/src/shopee/shopee-to-supabase.ts b/apps/importer/src/shopee/shopee-to-supabase.ts index 1775beed..8e8e5d08 100644 --- a/apps/importer/src/shopee/shopee-to-supabase.ts +++ b/apps/importer/src/shopee/shopee-to-supabase.ts @@ -149,20 +149,59 @@ export class ShopeeIngestionPipeline { const productId = savedProduct.id; console.log(`[ShopeeIngest] Listing upserted -> ID: ${productId}`); - // 4. Insert price history entry - await this.supabase.from('price_history').insert({ - product_id: productId, - price: product.priceMin, - price_original: product.priceBeforeDiscount || null, - recorded_at: new Date().toISOString(), - source: 'shopee_crawler', - }); + // 4. Price history entry with a stable observation id (plan v7 §4.1, + // Codex #8): re-crawls of the same price on the same day upsert instead of + // duplicating, and insert errors are SURFACED (never silent). + const dayBucket = new Date().toISOString().slice(0, 10); + const observationId = 'px-' + crypto + .createHash('sha256') + // Scoped to the listing UUID (Codex: itemId alone is not cross-platform safe). + .update(`${productId}|${product.priceMin}|${product.priceBeforeDiscount || ''}|${dayBucket}`) + .digest('hex'); + // The unique index on observation_id is PARTIAL, which ON CONFLICT cannot + // target (42P10, Codex blocker): dedupe client-side - fetch the id, insert + // only when absent. Errors are SURFACED (never silent). + const { data: existingPrice, error: priceLookupErr } = await this.supabase + .from('price_history') + .select('observation_id') + .eq('observation_id', observationId) + .limit(1); + if (priceLookupErr) { + throw new Error(`Failed to check existing price observation: ${priceLookupErr.message}`); + } + if (Array.isArray(existingPrice) && existingPrice.length > 0) { + console.log(`[ShopeeIngest] Price observation already recorded (${observationId}), skipping.`); + } else { + const { error: priceErr } = await this.supabase.from('price_history').insert({ + product_id: productId, + observation_id: observationId, + price: product.priceMin, + price_original: product.priceBeforeDiscount || null, + recorded_at: new Date().toISOString(), + source: 'shopee_crawler', + }); + if (priceErr) { + throw new Error(`Failed to record price history (observation ${observationId}): ${priceErr.message}`); + } + } // 5. Batch insert reviews if available if (product.reviews && product.reviews.length > 0) { const reviewRows = product.reviews.map((r) => ({ product_id: productId, - platform_review_id: r.reviewId || undefined, + // Stable observation id fallback (plan v7 §4.1): reviews without a + // platform id must not be re-inserted on every crawl. createdAt is + // deliberately excluded from the hash - the scraper falls back to + // now() when Shopee gives no ctime, minting a fresh id every recrawl + // (Codex #7). author+comment is stable; two distinct reviews with + // identical author+comment dedupe to one (accepted). + platform_review_id: + r.reviewId || + 'rv-' + + crypto + .createHash('sha256') + .update(`${r.author || ''}|${r.comment || ''}`) + .digest('hex'), author_name: r.author, author_avatar: r.authorAvatar || null, rating_star: r.ratingStar, @@ -172,14 +211,38 @@ export class ShopeeIngestionPipeline { platform_created_at: r.createdAt, })); - const { error: reviewErr } = await this.supabase + // The unique index on (product_id, platform_review_id) is PARTIAL + // (WHERE platform_review_id IS NOT NULL) so ON CONFLICT cannot target it + // (42P10): dedupe client-side against existing ids, plain-insert new rows. + const { data: existingReviews, error: existingErr } = await this.supabase .from('reviews') - .upsert(reviewRows, { onConflict: 'platform_review_id', ignoreDuplicates: true }); - - if (reviewErr) { - console.warn(`[ShopeeIngest] Warning: Reviews insert partial error: ${reviewErr.message}`); + .select('platform_review_id') + .eq('product_id', productId) + .not('platform_review_id', 'is', null); + if (existingErr) { + throw new Error(`Failed to fetch existing reviews: ${existingErr.message}`); + } + const seen = new Set((existingReviews || []).map((r: { platform_review_id: string }) => r.platform_review_id)); + const newRows = reviewRows.filter((r) => { + if (!r.platform_review_id) return true; + if (seen.has(r.platform_review_id)) return false; + // In-batch dedupe (Codex round-2): two id-less reviews with the same + // author+comment hash to the same id - inserting both violates the + // partial unique index (23505). Keep the first, skip the rest. + seen.add(r.platform_review_id); + return true; + }); + + if (newRows.length === 0) { + console.log(`[ShopeeIngest] No new reviews to ingest (${seen.size} duplicates skipped).`); } else { - console.log(`[ShopeeIngest] Successfully ingested ${reviewRows.length} reviews.`); + const { error: reviewErr } = await this.supabase.from('reviews').insert(newRows); + if (reviewErr) { + // Fail the whole ingestion (plan v7 §4.1, Codex blocker): a review + // write failure must NOT let the caller ack the job as done. + throw new Error(`Failed to insert reviews: ${reviewErr.message}`); + } + console.log(`[ShopeeIngest] Successfully ingested ${newRows.length} reviews (${reviewRows.length - newRows.length} duplicates skipped).`); } } diff --git a/apps/web/src/lib/rate-api.ts b/apps/web/src/lib/rate-api.ts index f2662b78..f90e133b 100644 --- a/apps/web/src/lib/rate-api.ts +++ b/apps/web/src/lib/rate-api.ts @@ -1,26 +1,27 @@ import { createClient, SupabaseClient } from '@supabase/supabase-js'; import type { MasterProduct, ProductSpecifications, KeySpecs } from '@repo/shared-data'; -const SUPABASE_URL = - process.env.NEXT_PUBLIC_SUPABASE_URL || - process.env.SUPABASE_URL || - 'https://gulptwduchsjcsbndmua.supabase.co'; +// Fail hard on missing env (plan v7 §5.4): a silent fallback to the PROD +// project makes preview verification read prod and look healthy. +const SUPABASE_URL = process.env.NEXT_PUBLIC_SUPABASE_URL || process.env.SUPABASE_URL; +const SUPABASE_ANON_KEY = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || process.env.SUPABASE_ANON_KEY; -const SUPABASE_ANON_KEY = - process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY || - process.env.SUPABASE_ANON_KEY || - 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Imd1bHB0d2R1Y2hzamNzYm5kbXVhIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NjY1OTI4NzQsImV4cCI6MjA4MjE2ODg3NH0.rRf1P8DhC_iK9KM2TSOU0XnjwoXmlBgZymGuhUdPazs'; - -let _client: SupabaseClient | null = null; +let _client: any = null; export function getRateClient(): SupabaseClient { - if (!_client) { - const raw = createClient(SUPABASE_URL, SUPABASE_ANON_KEY, { - auth: { persistSession: false }, - }); - _client = (raw as any).schema('rate'); + if (_client) { + return _client as SupabaseClient; + } + if (!SUPABASE_URL || !SUPABASE_ANON_KEY) { + throw new Error( + 'rate-api: NEXT_PUBLIC_SUPABASE_URL / NEXT_PUBLIC_SUPABASE_ANON_KEY are not configured; refusing to fall back to a hardcoded project' + ); } - return _client; + const raw = createClient(SUPABASE_URL, SUPABASE_ANON_KEY, { + auth: { persistSession: false }, + }); + _client = (raw as any).schema('rate'); + return _client as SupabaseClient; } export interface MerchantOffer { @@ -52,6 +53,8 @@ export interface ReviewItem { comment: string; variation: string | null; created_at: string; + verdict: 'authentic' | 'suspicious' | 'fake' | 'unanalyzed' | null; + fake_probability: number | null; } /** @@ -83,8 +86,7 @@ export async function getMasterProducts(params?: { const { data, error } = await query; if (error) { - console.error('Error fetching master products:', error.message); - return []; + throw new Error(`rate-api: failed to fetch master products: ${error.message}`); } return (data as MasterProduct[]) || []; } @@ -98,6 +100,7 @@ export async function getMasterProductBySlug(slug: string): Promise { const supabase = getRateClient(); // First find all product IDs for this master - const { data: prods } = await supabase + const { data: prods, error: prodsErr } = await supabase .from('products') .select('id') .eq('cluster_id', masterId); - + if (prodsErr) { + throw new Error(`rate-api: failed to resolve listings for master ${masterId}: ${prodsErr.message}`); + } if (!prods || prods.length === 0) return []; const productIds = prods.map((p: any) => p.id); + // review_flags is embedded via review_flags.review_id -> reviews.id + // (migration 20261005130001); absent rows mean "not yet analyzed". const { data: reviews, error } = await supabase .from('reviews') - .select('id, author_name, rating_star, comment, variation, created_at') + .select('id, author_name, rating_star, comment, variation, created_at, review_flags(verdict, fake_probability)') .in('product_id', productIds) .order('created_at', { ascending: false }) .limit(30); - if (error || !reviews) { + if (error) { + throw new Error(`rate-api: failed to fetch reviews for master ${masterId}: ${error.message}`); + } + if (!reviews) { return []; } - return reviews.map((r: any) => ({ - id: r.id, - author_name: r.author_name || 'Khách hàng', - rating_star: r.rating_star || 5, - comment: r.comment || '', - variation: r.variation || null, - created_at: r.created_at, - })); + return reviews.map((r: any) => { + // PostgREST returns an OBJECT for a 1:1 embed (review_flags.review_id is + // the PK), not an array - handle both defensively (Codex SHOULD). + const vf = Array.isArray(r.review_flags) ? r.review_flags[0] : r.review_flags; + return { + id: r.id, + author_name: r.author_name || 'Khách hàng', + rating_star: r.rating_star || 5, + comment: r.comment || '', + variation: r.variation || null, + created_at: r.created_at, + verdict: vf?.verdict ?? null, + fake_probability: vf?.fake_probability != null ? Number(vf.fake_probability) : null, + }; + }); } /** @@ -210,7 +227,8 @@ export async function compareMasterProducts(slugs: string[]): Promise