Skip to content

fix(importer,web): plan v7 section 5 code fixes #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JOY (JOY) merged 1 commit into from
Oct 6, 2026
Merged

fix(importer,web): plan v7 section 5 code fixes #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 71 additions & 42 deletions apps/importer/src/shopee/entity-matcher.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@ export class EntityMatcher {
.eq('model_key', candidate.model_key)
.maybeSingle();

// Infra errors must abort ingestion (plan v7 §5.1): a failed lookup must
// never fall through to Tier 3 and silently split the canonical cluster.
if (exactErr) {
console.warn(`[EntityMatcher] Tier 1 query warning: ${exactErr.message}`);
throw new Error(`[EntityMatcher] Tier 1 exact lookup failed: ${exactErr.message}`);
}

if (exactMatch) {
Expand All @@ -36,59 +38,86 @@ export class EntityMatcher {
p_similarity_threshold: 0.65,
});

if (!fuzzyErr && Array.isArray(fuzzyMatches) && fuzzyMatches.length > 0) {
// Same rule for Tier 2: RPC/infra failure aborts; only a genuine
// "no match above threshold" reaches Tier 3 (plan v7 §5.1).
if (fuzzyErr) {
throw new Error(`[EntityMatcher] Tier 2 fuzzy match failed: ${fuzzyErr.message}`);
}
let ambiguousFuzzy = false;
if (Array.isArray(fuzzyMatches) && fuzzyMatches.length > 0) {
const topMatch = fuzzyMatches[0];
if (topMatch.similarity >= 0.70) {
console.log(
`[EntityMatcher] Tier 2 Fuzzy Match found: "${candidate.name}" ~= "${topMatch.name}" (similarity: ${(topMatch.similarity * 100).toFixed(1)}%) -> ID: ${topMatch.master_id}`
);
return { masterId: topMatch.master_id, isNew: false };
}
// Candidates exist but all below the accept threshold: identity is
// uncertain -> seed as draft (plan v7 §5.1), never auto-publish.
ambiguousFuzzy = true;
console.log(`[EntityMatcher] Tier 2 best similarity ${(topMatch.similarity * 100).toFixed(1)}% < 70%: seeding as draft if new.`);
}

// Tier 3: No existing master product matches -> Seed a new Master Product
console.log(`[EntityMatcher] Tier 3: Creating new canonical Master Product: ${candidate.name}...`);
const { data: createdMaster, error: createErr } = await this.supabase
.from('master_products')
.insert({
category_id: categoryId || null,
brand: candidate.brand,
model: candidate.model,
variant_name: candidate.variant_name,
model_key: candidate.model_key,
name: candidate.name,
slug: candidate.slug,
description: candidate.description || '',
ram_gb: candidate.ram_gb,
storage_gb: candidate.storage_gb,
screen_size_inch: candidate.screen_size_inch,
battery_mah: candidate.battery_mah,
has_5g: candidate.has_5g,
specifications: candidate.specifications,
key_specs: candidate.key_specs,
images: candidate.images,
thumbnail: candidate.images[0] || null,
colors: candidate.colors,
status: 'published',
})
.select('id')
.single();
// Tier 3: No existing master product matches -> Seed a new Master Product.
// - Upsert on model_key (unique) so a concurrent crawler inserting the same
// canonical entity resolves instead of crashing (plan v7 §5.1 / Gemini B3).
// - Low-confidence identities (unparseable storage etc.) are seeded as
// 'draft', never auto-published; the live status CHECK allows only
// draft/published/archived, so 'draft' is the needs-review state.
const seedStatus = candidate.needs_review || ambiguousFuzzy ? 'draft' : 'published';
console.log(`[EntityMatcher] Tier 3: Creating new canonical Master Product: ${candidate.name} (status: ${seedStatus})...`);

if (createErr || !createdMaster) {
// In case of unique slug / model_key race condition, retry fetch
const { data: fallback } = await this.supabase
.from('master_products')
.select('id')
.eq('model_key', candidate.model_key)
.maybeSingle();
// Insert-only with ignoreDuplicates: the LOSING worker must refetch the
// winner WITHOUT overwriting its canonical metadata or status (Codex
// blocker: a re-asserting upsert could re-publish a draft/archived master).
const { data: createdRows, error: createErr } = await this.supabase
.from('master_products')
.upsert(
{
category_id: categoryId || null,
brand: candidate.brand,
model: candidate.model,
variant_name: candidate.variant_name,
model_key: candidate.model_key,
name: candidate.name,
slug: candidate.slug,
description: candidate.description || '',
ram_gb: candidate.ram_gb,
storage_gb: candidate.storage_gb,
screen_size_inch: candidate.screen_size_inch,
battery_mah: candidate.battery_mah,
has_5g: candidate.has_5g,
specifications: candidate.specifications,
key_specs: candidate.key_specs,
images: candidate.images,
thumbnail: candidate.images[0] || null,
colors: candidate.colors,
status: seedStatus,
},
{ onConflict: 'model_key', ignoreDuplicates: true }
)
.select('id');

if (fallback) {
return { masterId: fallback.id, isNew: false };
}
throw new Error(`Failed to create master product: ${createErr?.message}`);
if (createErr) {
throw new Error(`Failed to create master product: ${createErr.message}`);
}
if (Array.isArray(createdRows) && createdRows.length > 0) {
const createdMaster = createdRows[0];
console.log(`[EntityMatcher] Successfully created Master Product ID: ${createdMaster.id}`);
return { masterId: createdMaster.id, isNew: true };
}

console.log(`[EntityMatcher] Successfully created Master Product ID: ${createdMaster.id}`);
return { masterId: createdMaster.id, isNew: true };
// Lost the race: another worker inserted this model_key first. Adopt the
// winner as-is (its status/identity decisions stay authoritative).
const { data: winner, error: winnerErr } = await this.supabase
.from('master_products')
.select('id')
.eq('model_key', candidate.model_key)
.maybeSingle();
if (winnerErr || !winner) {
throw new Error(`Master product race lost and winner fetch failed: ${winnerErr?.message}`);
}
console.log(`[EntityMatcher] Master product already existed (race): ${winner.id}`);
return { masterId: winner.id, isNew: false };
}
}
90 changes: 74 additions & 16 deletions apps/importer/src/shopee/product-normalizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ export interface NormalizedProductCandidate {
model_key: string;
name: string;
slug: string;
storage_gb: number;
storage_gb: number | null; // null = unparseable from the title; identity carries ':unknown'
needs_review: boolean; // true when identity could not be confidently derived (plan v7 §5.2)
ram_gb: number | null;
screen_size_inch: number | null;
battery_mah: number | null;
Expand Down Expand Up @@ -149,15 +150,66 @@ export class ProductNormalizer {
model = tokens.slice(0, 3).join(' ');
}

// 3. Extract Storage
let storageGb = 128; // Default
const tbMatch = title.match(/(\d+)\s*TB/i);
const gbMatch = title.match(/(\d+)\s*GB/i);
if (tbMatch && tbMatch[1]) {
storageGb = parseInt(tbMatch[1], 10) * 1024;
} else if (gbMatch && gbMatch[1]) {
storageGb = parseInt(gbMatch[1], 10);
// 3. Extract Storage - RAM-aware (plan v7 §5.2, Codex-reproduced bug:
// "Samsung Galaxy A55 8GB RAM 256GB" must be 256GB, not the first GB token).
// Decimal separator: dot or comma (Vietnamese titles use "1,5TB").
const sizeTokens = [...title.matchAll(/(\d+(?:[.,]\d+)?)\s*(TB|GB)/gi)].map((m) => ({
value: parseFloat(m[1].replace(',', '.')) * (m[2].toUpperCase() === 'TB' ? 1024 : 1),
index: m.index ?? 0,
raw: m[0],
}));
// RAM-word matching (replaces per-pattern heuristics after two review
// rounds found holes: "8GB RAM 12GB RAM", "RAM 8GB RAM 12GB", the shared
// "RAM" in "8GB RAM 256GB", "dung luong 8GB bo nho RAM 256GB"). Rule:
// every literal "RAM" word claims the nearest UNCLAIMED size token -
// preferring the one BEFORE it (suffix "<size> RAM"), else the one AFTER
// it ("RAM <size>"). Any token left unclaimed is a storage candidate.
// A prefix claim requires adjacency: only whitespace between the token
// and the RAM phrase. "256GB - RAM 8GB" (Codex round-4) separates the ROM
// size with a dash - claiming 256 as RAM there would publish a wrong
// identity, so non-adjacent tokens are left for the storage side. Phrases
// like "bộ nhớ RAM" span their full length (Codex round-4 regression:
// "dung lượng 8GB bộ nhớ RAM 256GB" - the phrase's own words sit between
// 8GB and the literal RAM word, so adjacency is measured to the PHRASE
// start and 8GB claims correctly).
const ramPhrases = [...title.matchAll(/bộ\s*nhớ\s*RAM|dung\s*lượng\s*RAM|RAM\b/gi)].map((m) => m.index ?? 0);
const claimable = sizeTokens.map((t) => ({ ...t, claimed: false }));
for (const p of ramPhrases) {
const before = [...claimable]
.filter((t) => !t.claimed && t.index + t.raw.length <= p && /^\s*$/.test(title.slice(t.index + t.raw.length, p)))
.sort((a, b) => b.index - a.index)[0];
const after = [...claimable].filter((t) => !t.claimed && t.index > p).sort((a, b) => a.index - b.index)[0];
const pick = before ?? after;
if (pick) pick.claimed = true;
}
const storageCandidates = claimable.filter((t) => !t.claimed);

// ROM/storage markers, two confidence tiers (Codex round-6): an EXPLICIT
// "ROM"/"bộ nhớ trong" marker always wins - even when a separate RAM
// phrase follows ("ROM 256GB RAM 8GB" = storage 256, RAM 8). The LOOSE
// "dung lượng" marker is only trusted when not immediately followed by a
// RAM phrase ("dung lượng 8GB RAM 256GB" / "dung lượng 8GB dung lượng RAM
// 256GB" mean 8 is RAM, not storage).
const romExplicit = title.match(/(?:ROM|bộ nhớ trong)\s*[:\-]?\s*(\d+(?:[.,]\d+)?)\s*(TB|GB)/i);
const romLoose = title.match(
/dung lượng\s*[:\-]?\s*(\d+(?:[.,]\d+)?)\s*(TB|GB)(?!\s*(?:RAM|bộ\s*nhớ\s*RAM|dung\s*lượng\s*RAM))/i
);

let storageGb: number | null = null;
let storageConfident = false;
const marker = romExplicit ?? romLoose;
if (marker) {
// An explicit ROM marker always wins; the loose "dung lượng" marker has
// already been rejected when a RAM phrase follows its size.
storageGb = parseFloat(marker[1].replace(',', '.')) * (marker[2].toUpperCase() === 'TB' ? 1024 : 1);
storageConfident = true;
} else if (storageCandidates.length === 1) {
storageGb = storageCandidates[0].value;
storageConfident = true;
}
// Ambiguous (multiple unclaimed tokens, no ROM anchor) or nothing found:
// leave storageGb null -> ':unknown' identity + needs_review (never guess).
const storageKnown = storageGb !== null;

// 4. Extract Colors from models / variations
const colors: string[] = [];
Expand All @@ -169,17 +221,22 @@ export class ProductNormalizer {
}
}

// 5. Construct keys & slugs
// 5. Construct keys & slugs. Unknown storage keeps identity space explicit
// (':unknown') instead of guessing a default (plan v7 §5.2).
const baseKey = `${slugify(brand)}:${slugify(model)}`;
const modelKey = `${baseKey}:${storageGb}gb`;
const storageDisplay = storageGb >= 1024 ? `${storageGb / 1024}TB` : `${storageGb}GB`;
const canonicalName = `${brand} ${model} ${storageDisplay}`;
const modelKey = storageKnown ? `${baseKey}:${storageGb}gb` : `${baseKey}:unknown`;
const storageDisplay = storageGb === null
? 'Không rõ dung lượng'
: storageGb >= 1024
? `${storageGb / 1024}TB`
: `${storageGb}GB`;
const canonicalName = storageKnown ? `${brand} ${model} ${storageDisplay}` : `${brand} ${model}`;
const slug = slugify(canonicalName);

// 6. Enrich with 10 Field Groups
const benchmark = CANONICAL_PHONE_REGISTRY[baseKey];

const ramGb = benchmark ? benchmark.ram_gb : null;
const ramGb = benchmark ? benchmark.ram_gb : claimable.find((t) => t.claimed)?.value ?? null;
const screenSize = benchmark ? benchmark.screen_size_inch : null;
const batteryMah = benchmark ? benchmark.battery_mah : null;
const has5g = benchmark ? benchmark.has_5g : true;
Expand Down Expand Up @@ -213,7 +270,7 @@ export class ProductNormalizer {
cpu: benchmark?.cpu || undefined,
gpu: benchmark?.gpu || undefined,
ram_gb: ramGb || undefined,
storage_gb: storageGb,
storage_gb: storageGb !== null ? storageGb : undefined,
storage_expansion: 'Không hỗ trợ thẻ nhớ',
os: benchmark?.os || 'iOS 18',
},
Expand Down Expand Up @@ -277,7 +334,8 @@ export class ProductNormalizer {
model_key: modelKey,
name: canonicalName,
slug,
storage_gb: storageGb,
storage_gb: storageKnown ? storageGb : null,
needs_review: !storageConfident,
ram_gb: ramGb,
screen_size_inch: screenSize,
battery_mah: batteryMah,
Expand Down
93 changes: 78 additions & 15 deletions apps/importer/src/shopee/shopee-to-supabase.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,20 +149,59 @@ export class ShopeeIngestionPipeline {
const productId = savedProduct.id;
console.log(`[ShopeeIngest] Listing upserted -> ID: ${productId}`);

// 4. Insert price history entry
await this.supabase.from('price_history').insert({
product_id: productId,
price: product.priceMin,
price_original: product.priceBeforeDiscount || null,
recorded_at: new Date().toISOString(),
source: 'shopee_crawler',
});
// 4. Price history entry with a stable observation id (plan v7 §4.1,
// Codex #8): re-crawls of the same price on the same day upsert instead of
// duplicating, and insert errors are SURFACED (never silent).
const dayBucket = new Date().toISOString().slice(0, 10);
const observationId = 'px-' + crypto
.createHash('sha256')
// Scoped to the listing UUID (Codex: itemId alone is not cross-platform safe).
.update(`${productId}|${product.priceMin}|${product.priceBeforeDiscount || ''}|${dayBucket}`)
.digest('hex');
// The unique index on observation_id is PARTIAL, which ON CONFLICT cannot
// target (42P10, Codex blocker): dedupe client-side - fetch the id, insert
// only when absent. Errors are SURFACED (never silent).
const { data: existingPrice, error: priceLookupErr } = await this.supabase
.from('price_history')
.select('observation_id')
.eq('observation_id', observationId)
.limit(1);
if (priceLookupErr) {
throw new Error(`Failed to check existing price observation: ${priceLookupErr.message}`);
}
if (Array.isArray(existingPrice) && existingPrice.length > 0) {
console.log(`[ShopeeIngest] Price observation already recorded (${observationId}), skipping.`);
} else {
const { error: priceErr } = await this.supabase.from('price_history').insert({
product_id: productId,
observation_id: observationId,
price: product.priceMin,
price_original: product.priceBeforeDiscount || null,
recorded_at: new Date().toISOString(),
source: 'shopee_crawler',
});
if (priceErr) {
throw new Error(`Failed to record price history (observation ${observationId}): ${priceErr.message}`);
}
}

// 5. Batch insert reviews if available
if (product.reviews && product.reviews.length > 0) {
const reviewRows = product.reviews.map((r) => ({
product_id: productId,
platform_review_id: r.reviewId || undefined,
// Stable observation id fallback (plan v7 §4.1): reviews without a
// platform id must not be re-inserted on every crawl. createdAt is
// deliberately excluded from the hash - the scraper falls back to
// now() when Shopee gives no ctime, minting a fresh id every recrawl
// (Codex #7). author+comment is stable; two distinct reviews with
// identical author+comment dedupe to one (accepted).
platform_review_id:
r.reviewId ||
'rv-' +
crypto
.createHash('sha256')
.update(`${r.author || ''}|${r.comment || ''}`)
.digest('hex'),
author_name: r.author,
author_avatar: r.authorAvatar || null,
rating_star: r.ratingStar,
Expand All @@ -172,14 +211,38 @@ export class ShopeeIngestionPipeline {
platform_created_at: r.createdAt,
}));

const { error: reviewErr } = await this.supabase
// The unique index on (product_id, platform_review_id) is PARTIAL
// (WHERE platform_review_id IS NOT NULL) so ON CONFLICT cannot target it
// (42P10): dedupe client-side against existing ids, plain-insert new rows.
const { data: existingReviews, error: existingErr } = await this.supabase
.from('reviews')
.upsert(reviewRows, { onConflict: 'platform_review_id', ignoreDuplicates: true });

if (reviewErr) {
console.warn(`[ShopeeIngest] Warning: Reviews insert partial error: ${reviewErr.message}`);
.select('platform_review_id')
.eq('product_id', productId)
.not('platform_review_id', 'is', null);
if (existingErr) {
throw new Error(`Failed to fetch existing reviews: ${existingErr.message}`);
}
const seen = new Set<string>((existingReviews || []).map((r: { platform_review_id: string }) => r.platform_review_id));
const newRows = reviewRows.filter((r) => {
if (!r.platform_review_id) return true;
if (seen.has(r.platform_review_id)) return false;
// In-batch dedupe (Codex round-2): two id-less reviews with the same
// author+comment hash to the same id - inserting both violates the
// partial unique index (23505). Keep the first, skip the rest.
seen.add(r.platform_review_id);
return true;
});

if (newRows.length === 0) {
console.log(`[ShopeeIngest] No new reviews to ingest (${seen.size} duplicates skipped).`);
} else {
console.log(`[ShopeeIngest] Successfully ingested ${reviewRows.length} reviews.`);
const { error: reviewErr } = await this.supabase.from('reviews').insert(newRows);
if (reviewErr) {
// Fail the whole ingestion (plan v7 §4.1, Codex blocker): a review
// write failure must NOT let the caller ack the job as done.
throw new Error(`Failed to insert reviews: ${reviewErr.message}`);
}
console.log(`[ShopeeIngest] Successfully ingested ${newRows.length} reviews (${reviewRows.length - newRows.length} duplicates skipped).`);
}
}

Expand Down
Loading
Loading