Commit 93b03ea9 93b03ea91557e1bebbe201d31f48271e2d68830e by 沈秋雨

feat(dedup): 优化歌词去重逻辑并引入作者字段增量合并

- 新增 L2 候选加载缓存机制,避免重复下载歌词内容
- L2 去重返回 top-K 召回候选信息,丰富判定依据
- 分类函数新增跳过导入的规则,如空歌词且作者不详、原声类歌曲名等
- 实现作者字段(作词、作曲)增量合并功能,避免重复覆盖
- 对合并冲突添加异常处理并记录日志,确保稳定执行
- 引入后台线程异步写入数据库,提升批量处理性能
- 优化 L1 索引更新,避免去重误差,提高去重准确率
- 扩展统计字段,新增作者合并和跳过导入计数
- 前端增加决策过滤选项,支持根据决策筛选展示记录
- 修改界面显示逻辑,支持删除已审核样本,完善用户操作
- 调整页面显示候选数目为 Top5,优化视觉体验
- 参数新增 L2 精确哈希自动合并的最小歌词长度配置
- 添加数据库表结构自动迁移,新增 recalled_candidates 字段支持新功能
- 增强程序中断时数据库写入的安全关闭与日志提示
1 parent 25cb261f
......@@ -460,6 +460,13 @@
<select id="runSelect"></select>
<label id="topKLabel" for="topKSelect">topK</label>
<select id="topKSelect"></select>
<label for="decisionFilter">决策</label>
<select id="decisionFilter">
<option value="">全部</option>
<option value="new">new</option>
<option value="merge">merge</option>
<option value="review">review</option>
</select>
<input id="searchInput" type="search" placeholder="搜索歌名 / ID">
<label for="pageSizeSelect">每页</label>
<select id="pageSizeSelect">
......@@ -520,6 +527,7 @@
pageSize: 50,
mode: 'retrieval',
topK: '',
decisionFilter: '',
queryId: '',
candidateId: '',
lyricsCache: new Map()
......@@ -529,6 +537,7 @@
runSelect: document.getElementById('runSelect'),
topKSelect: document.getElementById('topKSelect'),
topKLabel: document.getElementById('topKLabel'),
decisionFilter: document.getElementById('decisionFilter'),
searchInput: document.getElementById('searchInput'),
pageSizeSelect: document.getElementById('pageSizeSelect'),
exportBtn: document.getElementById('exportBtn'),
......@@ -576,7 +585,7 @@
}
function rowDecision(row) {
return row.decision || row.candidate_decision || '';
return row.candidate_decision || row.decision || row.action || '';
}
function isConflict(row) {
......@@ -706,7 +715,8 @@
function selectedCandidate(rows) {
if (!rows.length) return null;
if (!state.candidateId || !rows.some(row => row.candidate_id === state.candidateId)) {
state.candidateId = rows[0].candidate_id || '';
const firstReal = rows.find(row => row.candidate_id);
state.candidateId = firstReal ? firstReal.candidate_id : (rows[0].candidate_id || '');
}
return rows.find(row => row.candidate_id === state.candidateId) || rows[0];
}
......@@ -726,6 +736,7 @@
const conflict = isConflict(row);
const l1 = l1Match(row);
const extraClass = `${conflict ? ' conflict' : ''}${l1 ? ' l1hint' : ''}`;
const l2Decision = row.candidate_decision || '';
return `<div class="candidate ${esc(decision)}${active}${extraClass}" data-candidate-id="${esc(row.candidate_id)}">
<div class="candidate-head">
<span class="rank">${esc(row.rank || '')}</span>
......@@ -737,6 +748,7 @@
<div class="query-meta">词 ${esc(row.candidate_lyricist || '-')} · 曲 ${esc(row.candidate_composer || '-')}</div>
<div style="margin-top:7px;display:flex;gap:5px;flex-wrap:wrap">
${l1 ? '<span class="pill l1">L1 元数据命中</span>' : '<span class="pill">L1 未命中</span>'}
${l2Decision ? `<span class="pill ${esc(l2Decision)}">L2 ${esc(l2Decision)}</span>` : ''}
${conflict ? '<span class="pill conflict">冲突</span>' : ''}
</div>
<div class="score-row">
......@@ -764,6 +776,7 @@
approved_import: '确认不重复(已入库)',
rejected_duplicate: '确认重复',
unsure: '待确认',
deleted: '已删除',
};
async function renderDetail() {
......@@ -824,7 +837,7 @@
<div class="section-head">
<h2 class="section-title">人工标注</h2>
${query.review_status && query.review_status !== 'pending' && query.review_status !== 'not_required'
? `<span class="pill ${query.review_status === 'approved_import' ? 'new' : query.review_status === 'rejected_duplicate' ? 'duplicate' : ''}">已审核</span>`
? `<span class="pill ${query.review_status === 'approved_import' ? 'new' : query.review_status === 'rejected_duplicate' ? 'duplicate' : query.review_status === 'deleted' ? 'duplicate' : ''}">${query.review_status === 'deleted' ? '已删除' : '已审核'}</span>`
: ''}
</div>
<div class="section-body">
......@@ -846,14 +859,15 @@
</button>`).join('')}
<input id="reviewNote" value="${esc(selectedReview.note || '')}" placeholder="人工备注">
<button id="saveReviewBtn" class="secondary">保存</button>
<button id="deleteReviewBtn" class="secondary" style="color:#c0392b;border-color:#c0392b">删除</button>
</div>
`}
</div>
</div>
<div class="section">
<div class="section-head"><h2 class="section-title">Top10 候选</h2></div>
<div class="section-body"><div class="candidates">${rows.map(candidateCard).join('') || '<div class="empty">无召回候选</div>'}</div></div>
<div class="section-head"><h2 class="section-title">Top5 候选</h2></div>
<div class="section-body"><div class="candidates">${rows.filter(r => r.candidate_id).map(candidateCard).join('') || '<div class="empty">无召回候选</div>'}</div></div>
</div>
<div class="section">
......@@ -900,6 +914,21 @@
renderDetail();
});
}
const deleteBtn = document.getElementById('deleteReviewBtn');
if (deleteBtn) {
deleteBtn.addEventListener('click', async () => {
if (!confirm('确认删除此样本?删除后将不再导入。')) return;
deleteBtn.disabled = true;
try {
await updateStagingReview(query.query_source_id, 'deleted',
document.getElementById('reviewNote')?.value || '');
await loadQueryRows(query.query_source_id);
} catch (e) {
alert(`删除失败: ${e.message}`);
}
renderDetail();
});
}
const undoBtn = document.getElementById('undoReviewBtn');
if (undoBtn) {
undoBtn.addEventListener('click', async () => {
......@@ -1024,6 +1053,7 @@
path: dataPath(),
top_k: state.topK,
mode: state.mode,
decision: state.decisionFilter,
q: els.searchInput.value.trim(),
page: String(state.page),
page_size: String(state.pageSize)
......@@ -1100,6 +1130,14 @@
await loadGroups();
renderAll();
});
els.decisionFilter.addEventListener('change', async () => {
state.decisionFilter = els.decisionFilter.value;
state.page = 1;
state.queryId = '';
state.candidateId = '';
await loadGroups();
renderAll();
});
els.reloadBtn.addEventListener('click', loadRuns);
els.exportBtn.addEventListener('click', exportAnnotations);
els.importReviewedBtn.addEventListener('click', importReviewed);
......
......@@ -91,6 +91,7 @@ class DuplicateChecker:
metadata_duplicate_line_coverage_threshold: float = 0.65,
metadata_duplicate_min_primary_lines: int = 8,
auto_duplicate_min_primary_lines: int = 4,
exact_duplicate_min_primary_chars: int = 40,
) -> None:
self.duplicate_jaccard_threshold = duplicate_jaccard_threshold
self.duplicate_line_coverage_threshold = duplicate_line_coverage_threshold
......@@ -111,6 +112,7 @@ class DuplicateChecker:
self.metadata_duplicate_line_coverage_threshold = metadata_duplicate_line_coverage_threshold
self.metadata_duplicate_min_primary_lines = metadata_duplicate_min_primary_lines
self.auto_duplicate_min_primary_lines = auto_duplicate_min_primary_lines
self.exact_duplicate_min_primary_chars = exact_duplicate_min_primary_chars
def check_record_against_candidates(
self,
......@@ -209,6 +211,14 @@ class DuplicateChecker:
decision = DuplicateDecision.REVIEW
confidence = 0.95
reason = "原文哈希一致,但疑似整段翻译结构拆分置信度较低,需要人工复核"
elif not _has_enough_primary_text_chars(
query.normalized,
candidate.normalized,
min_chars=self.exact_duplicate_min_primary_chars,
):
decision = DuplicateDecision.REVIEW
confidence = 0.95
reason = "规范化后的原文哈希一致,但有效歌词过短,需要人工复核"
elif _is_generic_primary_lyrics(query.normalized) or _is_generic_primary_lyrics(candidate.normalized):
decision = DuplicateDecision.REVIEW
confidence = 0.95
......@@ -510,6 +520,21 @@ def _has_enough_primary_lyrics(
return len(meaningful) >= min_lines
def _has_enough_primary_text_chars(
left: NormalizedLyrics,
right: NormalizedLyrics,
*,
min_chars: int,
) -> bool:
left_chars = _normalized_primary_text_length(left)
right_chars = _normalized_primary_text_length(right)
return min(left_chars, right_chars) >= min_chars
def _normalized_primary_text_length(normalized: NormalizedLyrics) -> int:
return len(_normalize_meta("".join(normalized.primary_lines)))
def _is_metadata_like_line(line: str) -> bool:
normalized = _normalize_meta(line)
metadata_prefixes = (
......