// dedup.jsx — catalog dedup screen // // Sweeps WORKS, RECORDINGS, RELEASES, PARTIES looking for likely duplicate // pairs using fuzzy-engine.jsx. Three tabs: // 01 · Candidates — ranked pair list w/ algorithm breakdown + merge action // 02 · Blocking — explains the blocking strategy and lets user tune it // 03 · Merge log — every merge is reversible, recorded, replayable // // EXPORT: window.ScreenDedup // ============================================================================ (() => { const { useState, useMemo, useEffect } = React; const F = () => window.FuzzyEngine; const KINDS = [ { v: 'works', l: 'Works', src: () => window.WORKS || [], get: (w) => w.title || w.name }, { v: 'recordings', l: 'Recordings', src: () => window.RECORDINGS || [], get: (r) => r.title || r.name }, { v: 'releases', l: 'Releases', src: () => window.RELEASES || [], get: (r) => r.title || r.name }, { v: 'parties', l: 'Parties', src: () => window.PARTIES || [], get: (p) => p.name || p.legalName }, ]; // ── merge log persistence ──────────────────────────────────────── const MERGE_KEY = 'astro.dedup.merges'; function loadMerges() { try { return JSON.parse(localStorage.getItem(MERGE_KEY) || '[]'); } catch (_) { return []; } } function saveMerges(arr) { try { localStorage.setItem(MERGE_KEY, JSON.stringify(arr.slice(0, 500))); } catch (_) {} } function ScreenDedup({ go, payload }) { const [tab, setTab] = useState(payload?.tab || 'candidates'); const [kind, setKind] = useState(payload?.kind || 'works'); const [threshold, setThreshold] = useState(0.78); const [pairs, setPairs] = useState(null); const [scanning, setScanning] = useState(false); const [scanMs, setScanMs] = useState(0); const [inspect, setInspect] = useState(null); const [merges, setMerges] = useState(loadMerges); const [dismissed, setDismissed] = useState(new Set()); function runScan() { if (!F()) return; setScanning(true); setPairs(null); // Defer to next frame so the UI flips before the (potentially heavy) // scoring loop runs. setTimeout(() => { const k = KINDS.find(x => x.v === kind); if (!k) { setScanning(false); return; } const items = k.src(); const t0 = performance.now(); // build corpus DF for soft-tfidf signal try { F().buildCorpusDFFromCatalog(); } catch (_) {} const found = F().findDuplicates(items, k.get, { threshold, maxPairs: 300 }); setScanMs(Math.round(performance.now() - t0)); setPairs(found); setScanning(false); }, 30); } // Auto-scan when entering candidates tab useEffect(() => { if (tab === 'candidates' && !pairs && !scanning) runScan(); }, [tab, kind]); function dismiss(pair) { const id = pairId(pair); setDismissed(s => { const n = new Set(s); n.add(id); return n; }); } function recordMerge(pair, keep) { const k = KINDS.find(x => x.v === kind); const drop = pair.a === keep ? pair.b : pair.a; const entry = { id: 'mrg_' + Date.now().toString(36) + Math.random().toString(36).slice(2, 6), ts: Date.now(), kind, keepId: keep.id, keepLabel: k.get(keep), dropId: drop.id, dropLabel: k.get(drop), score: pair.score, }; const next = [entry, ...merges]; setMerges(next); saveMerges(next); setInspect(null); dismiss(pair); } function undoMerge(entry) { if (!confirm(`Undo merge — restore "${entry.dropLabel}" as a separate ${entry.kind.slice(0, -1)}?`)) return; const next = merges.filter(m => m.id !== entry.id); setMerges(next); saveMerges(next); } const visiblePairs = useMemo(() => { if (!pairs) return []; return pairs.filter(p => !dismissed.has(pairId(p))); }, [pairs, dismissed]); const TABS = [ { id: 'candidates', l: '01 · Candidates' }, { id: 'blocking', l: '02 · Blocking' }, { id: 'log', l: '03 · Merge log' }, ]; return (
{/* breadcrumb */}
/ CATALOG DEDUP
{/* hero */}
TOOLS · QUALITY

Catalog dedup

Find likely-duplicate works, recordings, releases, and parties across the catalog. Six fuzzy algorithms run in parallel, blocked by phonetic key + leading trigram so even a 50k-row sweep stays cheap. Every merge is reversible.
{/* tabs */}
{TABS.map(t => ( ))} {tab === 'candidates' && (
SCAN ≥ setThreshold(parseFloat(e.target.value) || 0.78)} className="ff-mono num" style={{ width: 60, fontSize: 11, padding: '4px 6px', background: 'var(--paper)', border: '1px solid var(--rule)', color: 'var(--ink)', textAlign: 'right' }} />
)}
{tab === 'candidates' && ( )} {tab === 'blocking' && } {tab === 'log' && } {inspect && ( setInspect(null)} onMerge={recordMerge} onDismiss={(p) => { dismiss(p); setInspect(null); }} /> )}
); } function pairId(p) { return (p.a.id || '') + '::' + (p.b.id || ''); } // ── 01 · Candidates ─────────────────────────────────────────────── function Candidates({ pairs, totalRaw, dismissedCount, scanning, kind, scanMs, onInspect, onDismiss, go }) { const k = KINDS.find(x => x.v === kind); if (scanning) { return
Scanning catalog…
; } if (!pairs || pairs.length === 0) { return (
NO DUPLICATES FOUND
{totalRaw === 0 ? `No likely-duplicate ${k?.l.toLowerCase()} at this threshold. Lower the score floor to widen the search.` : `All ${dismissedCount} candidate${dismissedCount === 1 ? '' : 's'} dismissed.`}
); } return (
{/* KPI strip */}
p.score >= 0.92).length} tone="warn" />
SCORE RECORD A RECORD B Δ SIGNAL ACTION
{pairs.map((p, i) => ( ))}
); } function PairRow({ pair, kind, k, onInspect, onDismiss }) { const sa = k.get(pair.a), sb = k.get(pair.b); const conf = Math.round(pair.score * 100); const tone = pair.score >= 0.92 ? '#a04432' : pair.score >= 0.85 ? '#c79538' : 'var(--ink-3)'; // dominant signal const br = useMemo(() => { const F_ = window.FuzzyEngine; if (!F_) return null; return F_.scoreBreakdown(sa, sb); }, [sa, sb]); const dominant = br ? Object.entries(br).filter(([k_]) => k_ !== 'composite' && br[k_] != null).sort((a, b) => b[1] - a[1])[0]?.[0] : '?'; const subA = subtitle(pair.a, kind); const subB = subtitle(pair.b, kind); return (
onInspect(pair)} onMouseEnter={(e) => e.currentTarget.style.background = 'var(--surface-2)'} onMouseLeave={(e) => e.currentTarget.style.background = 'transparent'}> {conf}
{sa}
{subA &&
{subA}
}
{sb}
{subB &&
{subB}
}
{sa.length === sb.length ? '0' : (sa.length > sb.length ? '+' : '−') + Math.abs(sa.length - sb.length)} {dominant}
e.stopPropagation()}>
); } function subtitle(item, kind) { if (kind === 'works') return item.iswc || (item.writers && item.writers[0]?.name) || ''; if (kind === 'recordings') return item.isrc || item.primaryArtistName || item.artist || ''; if (kind === 'releases') return item.upc || item.primaryArtistName || item.artist || ''; if (kind === 'parties') return item.ipi || item.role || item.kind || ''; return ''; } // ── Inspect drawer ─────────────────────────────────────────────── function InspectDrawer({ pair, kind, onClose, onMerge, onDismiss }) { const k = KINDS.find(x => x.v === kind); const sa = k.get(pair.a), sb = k.get(pair.b); const F_ = window.FuzzyEngine; const br = F_ ? F_.scoreBreakdown(sa, sb) : null; return ( <>
REVIEW DUPLICATE PAIR
Composite score · {Math.round(pair.score * 100)}
{/* Side-by-side */}
onMerge(pair, pair.a)} /> onMerge(pair, pair.b)} />
{/* Algorithm breakdown */} {br && (
ALGORITHM BREAKDOWN
{ALG_ROWS.map(row => { const v = br[row.k]; if (v == null) return (
{row.l}
n/a · corpus DF unavailable
); const pct = Math.round(v * 100); return (
{row.l}
= 85 ? '#3a8a52' : pct >= 60 ? '#c79538' : 'var(--ink-3)' }} />
{pct} ); })}
)} {/* Actions */}
); } const ALG_ROWS = [ { k: 'jaccard', l: 'Token Jaccard' }, { k: 'trigram', l: 'Trigram cosine' }, { k: 'metaphone', l: 'Double Metaphone' }, { k: 'damerau', l: 'Damerau-Levenshtein' }, { k: 'smithWaterman', l: 'Smith-Waterman' }, { k: 'softTfidf', l: 'Soft-TFIDF' }, ]; function RecordCard({ item, kind, k, side, onKeep }) { const sub = subtitle(item, kind); return (
RECORD · {side}
{k.get(item)}
{sub &&
{sub}
}
id · {item.id || '—'}
{item.createdAt &&
created · {fmtDate(item.createdAt)}
}
); } function fmtDate(ts) { try { return new Date(ts).toISOString().slice(0, 10); } catch (_) { return String(ts).slice(0, 10); } } // ── 02 · Blocking ──────────────────────────────────────────────── function BlockingExplainer() { const [demo, setDemo] = useState('Despacito'); const F_ = window.FuzzyEngine; const keys = F_ ? [...F_.blockingKeys(demo)] : []; return (
HOW BLOCKING KEEPS THIS CHEAP

A naive sweep of 50,000 records is 1.25 billion pairs — far too many for browser-side scoring. Instead we generate a small set of blocking keys per record and only score pairs that share a key.

Each record gets up to three keys: a 4-char metaphone prefix, the leading 3 normalized characters, and the first 6 chars of its first ≥4-char token. Two truly-similar records share at least one with overwhelming probability; two unrelated records almost never do.

A 50k catalog scan typically lands at ~2M scored pairs — under a second on commodity hardware.

TRY IT
setDemo(e.target.value)} className="ff-mono" style={{ width: '100%', padding: '8px 10px', background: 'var(--paper)', border: '1px solid var(--rule)', color: 'var(--ink)', fontSize: 13, marginBottom: 12 }} />
BLOCKING KEYS
{keys.map(k => ( {k} ))} {keys.length === 0 && (empty)}
METAPHONE
{F_ ? F_.metaphone(demo) || '(none)' : '—'}
NORMALIZED
{F_ ? F_.normalize(demo) || '(empty)' : '—'}
); } // ── 03 · Merge log ─────────────────────────────────────────────── function MergeLog({ merges, onUndo, go }) { if (!merges.length) { return (
NO MERGES YET
Approved merges from the Candidates tab show up here, with one-click undo.
); } return (
WHEN KIND KEPT MERGED IN SCORE UNDO
{merges.map(m => (
{fmtTs(m.ts)} {m.kind} {m.keepLabel} {m.dropLabel} {Math.round(m.score * 100)}
))}
); } function fmtTs(ts) { try { const d = new Date(ts); return d.toISOString().slice(0, 10) + ' ' + d.toTimeString().slice(0, 5); } catch (_) { return ''; } } // ── KPI cell ───────────────────────────────────────────────────── function Kpi({ label, value, tone }) { const c = tone === 'warn' ? '#a04432' : tone === 'ok' ? '#3a8a52' : 'var(--ink)'; return (
{label}
{value}
); } Object.assign(window, { ScreenDedup }); })();