feat(audit): 添加hk_songs重复数据审计及处理脚本

- 新增 audit_hk_songs_duplicates.py，支持基于name、lyricist和composer字段检测重复数据
- 支持生成重复组和重复详情的SQL查询语句
- 实现重复数据的合并计划生成及软删除重复记录功能
- 支持修复暂存表中merge跳过的记录，实现existing_into_new和new_into_existing的合并处理
- 增加命令行参数，支持输出CSV、应用合并和修复合并方向处理

feat(backfill): 增量补全bpm_class字段脚本

- 新增 backfill_bpm_class.py，从源库获取录音BPM并映射为bpm_class分类
- 支持指定目标表和每批处理条数，支持dry-run模式仅打印计划
- 采用主版本优先策略查询录音BPM数据，跳过无效或异常BPM

fix(import): 优化歌词处理和缓存逻辑

- 判断歌词文本是否实质为空，避免无效歌词上传OSS
- 修正合并逻辑中merge_direction对staging状态和导入ID的设置
- 扩展原声正则匹配规则，增加用户创作原声匹配
- 新增构建歌词缓存项和追加写入缓存函数，减少重复下载
- 调整load_existing_l2_candidates增加seen_keys参数避免重复缓存加载

+#!/usr/bin/env python3
+"""增量补全 hk_songs_test（或任意目标表）中缺失的 bpm_class 字段。
+
+从源库 hk_music_record_state.bpm 取主版本录音的 BPM 值，按以下规则映射：
+    bpm = 0 或 NULL  →  NULL（跳过）
+    bpm < 80         →  1（慢歌）
+    80 ≤ bpm ≤ 120   →  2（中速）
+    bpm > 120        →  3（快歌）
+
+用法:
+    python backfill_bpm_class.py                     # 默认更新 hk_songs_test
+    python backfill_bpm_class.py --table hk_songs    # 更新其他表
+    python backfill_bpm_class.py --dry-run           # 只打印，不写库
+    python backfill_bpm_class.py --batch-size 500    # 每批处理条数（默认200）
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+
+import pymysql
+from dotenv import load_dotenv
+from tqdm import tqdm
+
+load_dotenv(os.path.join(os.path.dirname(__file__), ".env"))
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)s %(message)s",
+    datefmt="%H:%M:%S",
+)
+logger = logging.getLogger(__name__)
+
+TARGET_DB_CONFIG = {
+    "host": os.getenv("TARGET_DB_HOST"),
+    "port": int(os.getenv("TARGET_DB_PORT", 3306)),
+    "user": os.getenv("TARGET_DB_USER"),
+    "password": os.getenv("TARGET_DB_PASSWORD"),
+    "database": os.getenv("TARGET_DB_NAME"),
+    "charset": "utf8mb4",
+    "cursorclass": pymysql.cursors.DictCursor,
+}
+
+SOURCE_DB_CONFIG = {
+    "host": os.getenv("SOURCE_DB_HOST"),
+    "port": int(os.getenv("SOURCE_DB_PORT", 3306)),
+    "user": os.getenv("SOURCE_DB_USER"),
+    "password": os.getenv("SOURCE_DB_PASSWORD"),
+    "database": os.getenv("SOURCE_DB_NAME"),
+    "charset": "utf8mb4",
+    "cursorclass": pymysql.cursors.DictCursor,
+}
+
+# 主版本优先：同一首歌取 is_main_version=1 的录音，没有则取 min(record_id)
+BPM_QUERY = """
+SELECT sp.id AS song_id, rs.bpm
+FROM hk_song_platform sp
+INNER JOIN (
+    SELECT song_id,
+           COALESCE(
+               MIN(CASE WHEN is_main_version = 1 THEN record_id END),
+               MIN(record_id)
+           ) AS record_id
+    FROM hk_song_and_record
+    GROUP BY song_id
+) sr ON sr.song_id = sp.id
+LEFT JOIN hk_music_record r
+    ON r.id = sr.record_id AND r.deleted = 0x00
+LEFT JOIN hk_music_record_state rs
+    ON rs.record_id = r.id AND rs.deleted = 0x00
+WHERE sp.id IN ({placeholders})
+"""
+
+
+def bpm_to_class(bpm_str: str | None) -> int | None:
+    if not bpm_str:
+        return None
+    try:
+        bpm = int(float(bpm_str))
+    except (ValueError, TypeError):
+        return None
+    if bpm <= 0:
+        return None
+    if bpm < 80:
+        return 1
+    if bpm <= 120:
+        return 2
+    return 3
+
+
+def fetch_bpm_map(src_conn, song_ids: list[str]) -> dict[str, int | None]:
+    """从源库批量查 bpm，返回 {source_song_id: bpm_class}。"""
+    if not song_ids:
+        return {}
+    placeholders = ",".join(["%s"] * len(song_ids))
+    sql = BPM_QUERY.format(placeholders=placeholders)
+    with src_conn.cursor() as cur:
+        cur.execute(sql, song_ids)
+        rows = cur.fetchall()
+    return {str(r["song_id"]): bpm_to_class(r["bpm"]) for r in rows}
+
+
+def backfill(table: str, batch_size: int, dry_run: bool) -> None:
+    tgt = pymysql.connect(**TARGET_DB_CONFIG)
+    src = pymysql.connect(**SOURCE_DB_CONFIG)
+
+    try:
+        # 查需要补全的记录总数
+        with tgt.cursor() as cur:
+            cur.execute(
+                f"SELECT COUNT(*) AS cnt FROM `{table}` "
+                f"WHERE bpm_class IS NULL AND source_song_id IS NOT NULL AND deleted = '0'"
+            )
+            total = cur.fetchone()["cnt"]
+
+        logger.info(f"待补全 bpm_class 的记录数: {total}，表: {table}")
+        if total == 0:
+            logger.info("无需更新，退出。")
+            return
+
+        updated = no_bpm = 0
+        last_id = 0  # 用 id > last_id 游标代替 OFFSET，避免跳过无bpm记录
+
+        with tqdm(total=total, unit="条") as bar:
+            while True:
+                with tgt.cursor() as cur:
+                    cur.execute(
+                        f"SELECT id, source_song_id FROM `{table}` "
+                        f"WHERE bpm_class IS NULL AND source_song_id IS NOT NULL AND deleted = '0' "
+                        f"AND id > %s ORDER BY id LIMIT %s",
+                        (last_id, batch_size),
+                    )
+                    rows = cur.fetchall()
+
+                if not rows:
+                    break
+
+                last_id = rows[-1]["id"]
+                song_ids = [r["source_song_id"] for r in rows]
+                bpm_map = fetch_bpm_map(src, song_ids)
+
+                for row in rows:
+                    sid = row["source_song_id"]
+                    bpm_class = bpm_map.get(str(sid))
+
+                    if bpm_class is None:
+                        no_bpm += 1
+                        bar.update(1)
+                        continue
+
+                    if dry_run:
+                        logger.debug(f"[DRY-RUN] id={row['id']} source_song_id={sid} -> bpm_class={bpm_class}")
+                        updated += 1
+                        bar.update(1)
+                        continue
+
+                    with tgt.cursor() as cur:
+                        cur.execute(
+                            f"UPDATE `{table}` SET bpm_class = %s WHERE id = %s",
+                            (bpm_class, row["id"]),
+                        )
+                    tgt.commit()
+                    updated += 1
+                    bar.update(1)
+
+        prefix = "[DRY-RUN] " if dry_run else ""
+        print(f"\n{'=' * 50}")
+        print(f"{prefix}bpm_class 补全完成")
+        print(f"  成功更新: {updated} 条")
+        print(f"  源库无bpm: {no_bpm} 条（保持 NULL）")
+        print(f"  合计处理: {updated + no_bpm} 条（共 {total} 条待补全）")
+        print(f"{'=' * 50}")
+
+    finally:
+        tgt.close()
+        src.close()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="增量补全 bpm_class 字段")
+    parser.add_argument("--table", default="hk_songs_test", help="目标表名（默认 hk_songs_test）")
+    parser.add_argument("--batch-size", type=int, default=200, help="每批处理条数（默认 200）")
+    parser.add_argument("--dry-run", action="store_true", help="只打印计划，不写库")
+    args = parser.parse_args()
+    backfill(args.table, args.batch_size, args.dry_run)
+
+
+if __name__ == "__main__":
+    main()

@@ -667,7 +667,8 @@
         metric('平均召回', row.avg_recalled_candidates || '-'),
         metric('平均召回', row.avg_recalled_candidates || '-'),
         metric('命中数', row.hit_count || '0'),
         metric('命中数', row.hit_count || '0'),
         metric('merge', row.duplicate_count || '0'),
         metric('merge', row.duplicate_count || '0'),
-        metric('review', row.review_count || '0')
+        metric('review', row.review_count || '0'),
+        metric('skip', row.skip_count || '0')
       ].join('');
       ].join('');
     }
     }
 
 

@@ -211,10 +211,17 @@ class DuplicateChecker:
             decision = DuplicateDecision.REVIEW
             decision = DuplicateDecision.REVIEW
             confidence = 0.95
             confidence = 0.95
             reason = "原文哈希一致，但疑似整段翻译结构拆分置信度较低，需要人工复核"
             reason = "原文哈希一致，但疑似整段翻译结构拆分置信度较低，需要人工复核"
-        elif not _has_enough_primary_text_chars(
+        elif not (
+            _has_enough_primary_text_chars(
                 query.normalized,
                 query.normalized,
                 candidate.normalized,
                 candidate.normalized,
                 min_chars=self.exact_duplicate_min_primary_chars,
                 min_chars=self.exact_duplicate_min_primary_chars,
+            )
+            or _has_enough_primary_lyrics(
+                query.normalized,
+                candidate.normalized,
+                min_lines=self.auto_duplicate_min_primary_lines,
+            )
         ):
         ):
             decision = DuplicateDecision.REVIEW
             decision = DuplicateDecision.REVIEW
             confidence = 0.95
             confidence = 0.95

@@ -103,7 +103,8 @@ def _staging_summary() -> list[dict[str, str]]:
       COUNT(DISTINCT source_song_id) AS total_count,
       COUNT(DISTINCT source_song_id) AS total_count,
       SUM(CASE WHEN dedup_action = 'new' THEN 1 ELSE 0 END) AS new_count,
       SUM(CASE WHEN dedup_action = 'new' THEN 1 ELSE 0 END) AS new_count,
       SUM(CASE WHEN dedup_action = 'merge' THEN 1 ELSE 0 END) AS merge_count,
       SUM(CASE WHEN dedup_action = 'merge' THEN 1 ELSE 0 END) AS merge_count,
-      SUM(CASE WHEN dedup_action = 'review' THEN 1 ELSE 0 END) AS review_count
+      SUM(CASE WHEN dedup_action = 'review' THEN 1 ELSE 0 END) AS review_count,
+      SUM(CASE WHEN dedup_action = 'skip' THEN 1 ELSE 0 END) AS skip_count
     FROM (
     FROM (
       SELECT source_song_id, dedup_action
       SELECT source_song_id, dedup_action
       FROM {TARGET_TABLE_NAME_TMP}
       FROM {TARGET_TABLE_NAME_TMP}
@@ -125,6 +126,7 @@ def _staging_summary() -> list[dict[str, str]]:
         "hit_count": str((row.get("merge_count") or 0) + (row.get("review_count") or 0)),
         "hit_count": str((row.get("merge_count") or 0) + (row.get("review_count") or 0)),
         "duplicate_count": str(row.get("merge_count") or 0),
         "duplicate_count": str(row.get("merge_count") or 0),
         "review_count": str(row.get("review_count") or 0),
         "review_count": str(row.get("review_count") or 0),
+        "skip_count": str(row.get("skip_count") or 0),
         "new_count": str(row.get("new_count") or 0),
         "new_count": str(row.get("new_count") or 0),
         "total_count": str(row.get("total_count") or 0),
         "total_count": str(row.get("total_count") or 0),
     }]
     }]

+#!/usr/bin/env python3
+"""Tests for hk_songs duplicate audit SQL/report helpers."""
+
+from audit_hk_songs_duplicates import (
+    build_duplicate_detail_sql,
+    build_duplicate_group_sql,
+    build_merge_plan,
+    build_soft_delete_sql,
+    summarize_duplicate_groups,
+)
+
+
+def test_group_sql_uses_name_lyricist_composer_and_deleted_filter():
+    sql = build_duplicate_group_sql("hk_songs_test", include_deleted=False, limit=20)
+
+    assert "`hk_songs_test`" in sql
+    assert "COALESCE(NULLIF(TRIM(name), ''), '') COLLATE utf8mb4_bin" in sql
+    assert "COALESCE(NULLIF(TRIM(lyricist), ''), '') COLLATE utf8mb4_bin" in sql
+    assert "COALESCE(NULLIF(TRIM(composer), ''), '') COLLATE utf8mb4_bin" in sql
+    assert "WHERE deleted = '0'" in sql
+    assert "HAVING COUNT(*) > 1" in sql
+    assert "LIMIT 20" in sql
+
+
+def test_group_sql_can_use_database_default_case_insensitive_collation():
+    sql = build_duplicate_group_sql("hk_songs_test", case_sensitive=False)
+
+    assert "COLLATE utf8mb4_bin" not in sql
+
+
+def test_detail_sql_joins_duplicate_groups_back_to_rows():
+    sql = build_duplicate_detail_sql("hk_songs_test", include_deleted=True, limit=None)
+
+    assert "JOIN (" in sql
+    assert "dup.name_key" in sql
+    assert "s.name_key" in sql
+    assert "WHERE deleted = '0'" not in sql
+    assert "LIMIT" not in sql
+
+
+def test_summary_counts_groups_rows_and_extra_duplicates():
+    groups = [
+        {"duplicate_count": 2},
+        {"duplicate_count": 5},
+    ]
+
+    summary = summarize_duplicate_groups(groups)
+
+    assert summary == {
+        "duplicate_groups": 2,
+        "duplicate_rows": 7,
+        "extra_duplicate_rows": 5,
+    }
+
+
+def test_merge_plan_keeps_row_with_largest_record_count():
+    rows = [
+        {"id": 10, "name": "A", "lyricist": "L", "composer": "C", "source_song_id": "s10", "record_count": 1},
+        {"id": 11, "name": "A", "lyricist": "L", "composer": "C", "source_song_id": "s11", "record_count": 3},
+        {"id": 12, "name": "B", "lyricist": "", "composer": "", "source_song_id": "s12", "record_count": None},
+        {"id": 13, "name": "B", "lyricist": None, "composer": None, "source_song_id": "s13", "record_count": 2},
+    ]
+
+    plan = build_merge_plan(rows)
+
+    assert [item["survivor_id"] for item in plan] == [11, 13]
+    assert [item["loser_id"] for item in plan] == [10, 12]
+    assert plan[0]["survivor_record_count"] == 3
+    assert plan[0]["loser_record_count"] == 1
+
+
+def test_merge_plan_uses_lowest_id_as_tie_breaker():
+    rows = [
+        {"id": 22, "name": "A", "lyricist": "L", "composer": "C", "source_song_id": "s22", "record_count": 2},
+        {"id": 21, "name": "A", "lyricist": "L", "composer": "C", "source_song_id": "s21", "record_count": 2},
+    ]
+
+    plan = build_merge_plan(rows)
+
+    assert plan[0]["survivor_id"] == 21
+    assert plan[0]["loser_id"] == 22
+
+
+def test_soft_delete_sql_updates_only_planned_losers():
+    sql = build_soft_delete_sql("hk_songs_test", 3)
+
+    assert "UPDATE `hk_songs_test`" in sql
+    assert "SET deleted = '1'" in sql
+    assert "WHERE deleted = '0'" in sql
+    assert "id IN (%s,%s,%s)" in sql