feat(dedup): 优化歌词去重逻辑并引入作者字段增量合并

- 新增 L2 候选加载缓存机制，避免重复下载歌词内容 - L2 去重返回 top-K 召回候选信息，丰富判定依据 - 分类函数新增跳过导入的规则，如空歌词且作者不详、原声类歌曲名等 - 实现作者字段（作词、作曲）增量合并功能，避免重复覆盖 - 对合并冲突添加异常处理并记录日志，确保稳定执行 - 引入后台线程异步写入数据库，提升批量处理性能 - 优化 L1 索引更新，避免去重误差，提高去重准确率 - 扩展统计字段，新增作者合并和跳过导入计数 - 前端增加决策过滤选项，支持根据决策筛选展示记录 - 修改界面显示逻辑，支持删除已审核样本，完善用户操作 - 调整页面显示候选数目为 Top5，优化视觉体验 - 参数新增 L2 精确哈希自动合并的最小歌词长度配置 - 添加数据库表结构自动迁移，新增 recalled_candidates 字段支持新功能 - 增强程序中断时数据库写入的安全关闭与日志提示

feat(dedup): 优化歌词去重逻辑并引入作者字段增量合并
- 新增 L2 候选加载缓存机制，避免重复下载歌词内容 - L2 去重返回 top-K 召回候选信息，丰富判定依据 - 分类函数新增跳过导入的规则，如空歌词且作者不详、原声类歌曲名等 - 实现作者字段（作词、作曲）增量合并功能，避免重复覆盖 - 对合并冲突添加异常处理并记录日志，确保稳定执行 - 引入后台线程异步写入数据库，提升批量处理性能 - 优化 L1 索引更新，避免去重误差，提高去重准确率 - 扩展统计字段，新增作者合并和跳过导入计数 - 前端增加决策过滤选项，支持根据决策筛选展示记录 - 修改界面显示逻辑，支持删除已审核样本，完善用户操作 - 调整页面显示候选数目为 Top5，优化视觉体验 - 参数新增 L2 精确哈希自动合并的最小歌词长度配置 - 添加数据库表结构自动迁移，新增 recalled_candidates 字段支持新功能 - 增强程序中断时数据库写入的安全关闭与日志提示
沈秋雨
Commit 93b03ea9 ... 93b03ea91557e1bebbe201d31f48271e2d68830e authored 2026-07-06 21:26:13 +0800 by 沈秋雨
Showing 5 changed files with 68 additions and 5 deletions
import_hk_songs.py
l2_review_dashboard.html
lyric_dedup/checker.py
serve_l2_dashboard.py
test_dedup.py
--- a/import_hk_songs.py
View file @93b03ea
+++ b/import_hk_songs.py
View file @93b03ea
--- a/l2_review_dashboard.html
View file @93b03ea
+++ b/l2_review_dashboard.html
View file @93b03ea
@@ -460,6 +460,13 @@
      <select id="runSelect"></select>
      <label id="topKLabel" for="topKSelect">topK</label>
      <select id="topKSelect"></select>
+      <label for="decisionFilter">决策</label>
+      <select id="decisionFilter">
+        <option value="">全部</option>
+        <option value="new">new</option>
+        <option value="merge">merge</option>
+        <option value="review">review</option>
+      </select>
      <input id="searchInput" type="search" placeholder="搜索歌名 / ID">
      <label for="pageSizeSelect">每页</label>
      <select id="pageSizeSelect">
@@ -520,6 +527,7 @@
      pageSize: 50,
      mode: 'retrieval',
      topK: '',
+      decisionFilter: '',
      queryId: '',
      candidateId: '',
      lyricsCache: new Map()
@@ -529,6 +537,7 @@
      runSelect: document.getElementById('runSelect'),
      topKSelect: document.getElementById('topKSelect'),
      topKLabel: document.getElementById('topKLabel'),
+      decisionFilter: document.getElementById('decisionFilter'),
      searchInput: document.getElementById('searchInput'),
      pageSizeSelect: document.getElementById('pageSizeSelect'),
      exportBtn: document.getElementById('exportBtn'),
@@ -576,7 +585,7 @@
    }
    function rowDecision(row) {
-      return row.decision || row.candidate_decision || '';
+      return row.candidate_decision || row.decision || row.action || '';
    }
    function isConflict(row) {
@@ -706,7 +715,8 @@
    function selectedCandidate(rows) {
      if (!rows.length) return null;
      if (!state.candidateId || !rows.some(row => row.candidate_id === state.candidateId)) {
-        state.candidateId = rows[0].candidate_id || '';
+        const firstReal = rows.find(row => row.candidate_id);
+        state.candidateId = firstReal ? firstReal.candidate_id : (rows[0].candidate_id || '');
      }
      return rows.find(row => row.candidate_id === state.candidateId) || rows[0];
    }
@@ -726,6 +736,7 @@
      const conflict = isConflict(row);
      const l1 = l1Match(row);
      const extraClass = `${conflict ? ' conflict' : ''}${l1 ? ' l1hint' : ''}`;
+      const l2Decision = row.candidate_decision || '';
      return `<div class="candidate ${esc(decision)}${active}${extraClass}" data-candidate-id="${esc(row.candidate_id)}">
        <div class="candidate-head">
          <span class="rank">${esc(row.rank || '')}</span>
@@ -737,6 +748,7 @@
        <div class="query-meta">词 ${esc(row.candidate_lyricist || '-')} · 曲 ${esc(row.candidate_composer || '-')}</div>
        <div style="margin-top:7px;display:flex;gap:5px;flex-wrap:wrap">
          ${l1 ? '<span class="pill l1">L1 元数据命中</span>' : '<span class="pill">L1 未命中</span>'}
+          ${l2Decision ? `<span class="pill ${esc(l2Decision)}">L2 ${esc(l2Decision)}</span>` : ''}
          ${conflict ? '<span class="pill conflict">冲突</span>' : ''}
        </div>
        <div class="score-row">
@@ -764,6 +776,7 @@
      approved_import: '确认不重复（已入库）',
      rejected_duplicate: '确认重复',
      unsure: '待确认',
+      deleted: '已删除',
    };
    async function renderDetail() {
@@ -824,7 +837,7 @@
          <div class="section-head">
            <h2 class="section-title">人工标注</h2>
            ${query.review_status && query.review_status !== 'pending' && query.review_status !== 'not_required'
-              ? `<span class="pill ${query.review_status === 'approved_import' ? 'new' : query.review_status === 'rejected_duplicate' ? 'duplicate' : ''}">已审核</span>`
+              ? `<span class="pill ${query.review_status === 'approved_import' ? 'new' : query.review_status === 'rejected_duplicate' ? 'duplicate' : query.review_status === 'deleted' ? 'duplicate' : ''}">${query.review_status === 'deleted' ? '已删除' : '已审核'}</span>`
              : ''}
          </div>
          <div class="section-body">
@@ -846,14 +859,15 @@
                  </button>`).join('')}
                <input id="reviewNote" value="${esc(selectedReview.note || '')}" placeholder="人工备注">
                <button id="saveReviewBtn" class="secondary">保存</button>
+                <button id="deleteReviewBtn" class="secondary" style="color:#c0392b;border-color:#c0392b">删除</button>
              </div>
            `}
          </div>
        </div>
        <div class="section">
-          <div class="section-head"><h2 class="section-title">Top10 候选</h2></div>
+          <div class="section-head"><h2 class="section-title">Top5 候选</h2></div>
-          <div class="section-body"><div class="candidates">${rows.map(candidateCard).join('') || '<div class="empty">无召回候选</div>'}</div></div>
+          <div class="section-body"><div class="candidates">${rows.filter(r => r.candidate_id).map(candidateCard).join('') || '<div class="empty">无召回候选</div>'}</div></div>
        </div>
        <div class="section">
@@ -900,6 +914,21 @@
          renderDetail();
        });
      }
+      const deleteBtn = document.getElementById('deleteReviewBtn');
+      if (deleteBtn) {
+        deleteBtn.addEventListener('click', async () => {
+          if (!confirm('确认删除此样本？删除后将不再导入。')) return;
+          deleteBtn.disabled = true;
+          try {
+            await updateStagingReview(query.query_source_id, 'deleted',
+              document.getElementById('reviewNote')?.value || '');
+            await loadQueryRows(query.query_source_id);
+          } catch (e) {
+            alert(`删除失败: ${e.message}`);
+          }
+          renderDetail();
+        });
+      }
      const undoBtn = document.getElementById('undoReviewBtn');
      if (undoBtn) {
        undoBtn.addEventListener('click', async () => {
@@ -1024,6 +1053,7 @@
        path: dataPath(),
        top_k: state.topK,
        mode: state.mode,
+        decision: state.decisionFilter,
        q: els.searchInput.value.trim(),
        page: String(state.page),
        page_size: String(state.pageSize)
@@ -1100,6 +1130,14 @@
      await loadGroups();
      renderAll();
    });
+    els.decisionFilter.addEventListener('change', async () => {
+      state.decisionFilter = els.decisionFilter.value;
+      state.page = 1;
+      state.queryId = '';
+      state.candidateId = '';
+      await loadGroups();
+      renderAll();
+    });
    els.reloadBtn.addEventListener('click', loadRuns);
    els.exportBtn.addEventListener('click', exportAnnotations);
    els.importReviewedBtn.addEventListener('click', importReviewed);
--- a/lyric_dedup/checker.py
View file @93b03ea
+++ b/lyric_dedup/checker.py
View file @93b03ea
@@ -91,6 +91,7 @@ class DuplicateChecker:
        metadata_duplicate_line_coverage_threshold: float = 0.65,
        metadata_duplicate_min_primary_lines: int = 8,
        auto_duplicate_min_primary_lines: int = 4,
+        exact_duplicate_min_primary_chars: int = 40,
    ) -> None:
        self.duplicate_jaccard_threshold = duplicate_jaccard_threshold
        self.duplicate_line_coverage_threshold = duplicate_line_coverage_threshold
@@ -111,6 +112,7 @@ class DuplicateChecker:
        self.metadata_duplicate_line_coverage_threshold = metadata_duplicate_line_coverage_threshold
        self.metadata_duplicate_min_primary_lines = metadata_duplicate_min_primary_lines
        self.auto_duplicate_min_primary_lines = auto_duplicate_min_primary_lines
+        self.exact_duplicate_min_primary_chars = exact_duplicate_min_primary_chars
    def check_record_against_candidates(
        self,
@@ -209,6 +211,14 @@ class DuplicateChecker:
            decision = DuplicateDecision.REVIEW
            confidence = 0.95
            reason = "原文哈希一致，但疑似整段翻译结构拆分置信度较低，需要人工复核"
+        elif not _has_enough_primary_text_chars(
+            query.normalized,
+            candidate.normalized,
+            min_chars=self.exact_duplicate_min_primary_chars,
+        ):
+            decision = DuplicateDecision.REVIEW
+            confidence = 0.95
+            reason = "规范化后的原文哈希一致，但有效歌词过短，需要人工复核"
        elif _is_generic_primary_lyrics(query.normalized) or _is_generic_primary_lyrics(candidate.normalized):
            decision = DuplicateDecision.REVIEW
            confidence = 0.95
@@ -510,6 +520,21 @@ def _has_enough_primary_lyrics(
    return len(meaningful) >= min_lines
+def _has_enough_primary_text_chars(
+    left: NormalizedLyrics,
+    right: NormalizedLyrics,
+    *,
+    min_chars: int,
+) -> bool:
+    left_chars = _normalized_primary_text_length(left)
+    right_chars = _normalized_primary_text_length(right)
+    return min(left_chars, right_chars) >= min_chars
+def _normalized_primary_text_length(normalized: NormalizedLyrics) -> int:
+    return len(_normalize_meta("".join(normalized.primary_lines)))
 def _is_metadata_like_line(line: str) -> bool:
    normalized = _normalize_meta(line)
    metadata_prefixes = (
--- a/serve_l2_dashboard.py
View file @93b03ea
+++ b/serve_l2_dashboard.py
View file @93b03ea
--- a/test_dedup.py
View file @93b03ea
+++ b/test_dedup.py
View file @93b03ea