feat(audit): 新增 L2 审计脚本与作者补丁脚本
Showing
2 changed files
with
149 additions
and
0 deletions
audit_hk_songs_l2.py
0 → 100644
This diff is collapsed.
Click to expand it.
patch_merge_authors.py
0 → 100644
| 1 | #!/usr/bin/env python3 | ||
| 2 | """补处理 merge+skipped 中 merge_authors=1 的 60 条记录。 | ||
| 3 | |||
| 4 | 暂存表 matched_song_id 存的是 source_song_id,通过 source_song_id 找到 | ||
| 5 | 目标库实际记录,再做作者字段增量合并。 | ||
| 6 | """ | ||
| 7 | |||
| 8 | from __future__ import annotations | ||
| 9 | |||
| 10 | import os | ||
| 11 | import sys | ||
| 12 | from pathlib import Path | ||
| 13 | |||
| 14 | import pymysql | ||
| 15 | from dotenv import load_dotenv | ||
| 16 | |||
| 17 | sys.path.insert(0, str(Path(__file__).resolve().parent)) | ||
| 18 | from import_hk_songs import _merge_author_field | ||
| 19 | |||
| 20 | load_dotenv() | ||
| 21 | |||
| 22 | TARGET_DB_CONFIG = { | ||
| 23 | "host": os.getenv("TARGET_DB_HOST"), | ||
| 24 | "port": int(os.getenv("TARGET_DB_PORT", 3306)), | ||
| 25 | "user": os.getenv("TARGET_DB_USER"), | ||
| 26 | "password": os.getenv("TARGET_DB_PASSWORD"), | ||
| 27 | "database": os.getenv("TARGET_DB_NAME"), | ||
| 28 | "charset": "utf8mb4", | ||
| 29 | "cursorclass": pymysql.cursors.DictCursor, | ||
| 30 | } | ||
| 31 | TABLE = os.getenv("TARGET_TABLE_NAME", "hk_songs") | ||
| 32 | STAGING = "hk_songs_import_staging" | ||
| 33 | |||
| 34 | |||
| 35 | def main(dry_run: bool = False) -> None: | ||
| 36 | conn = pymysql.connect(**TARGET_DB_CONFIG) | ||
| 37 | try: | ||
| 38 | # 1. 拉取所有需要补作者合并的暂存记录 | ||
| 39 | with conn.cursor() as cur: | ||
| 40 | cur.execute(f""" | ||
| 41 | SELECT staging_id, matched_song_id, lyricist, composer | ||
| 42 | FROM `{STAGING}` | ||
| 43 | WHERE dedup_action = 'merge' | ||
| 44 | AND staging_status = 'skipped' | ||
| 45 | AND merge_authors = 1 | ||
| 46 | ORDER BY staging_id | ||
| 47 | """) | ||
| 48 | staging_rows = cur.fetchall() | ||
| 49 | |||
| 50 | print(f"待处理记录数: {len(staging_rows)}") | ||
| 51 | |||
| 52 | # 2. 批量查目标库(按 source_song_id) | ||
| 53 | source_ids = [r["matched_song_id"] for r in staging_rows if r["matched_song_id"]] | ||
| 54 | ph = ",".join(["%s"] * len(source_ids)) | ||
| 55 | with conn.cursor() as cur: | ||
| 56 | cur.execute( | ||
| 57 | f"SELECT id, name, lyricist, composer, source_song_id FROM `{TABLE}` WHERE source_song_id IN ({ph})", | ||
| 58 | source_ids, | ||
| 59 | ) | ||
| 60 | target_rows = {str(r["source_song_id"]): r for r in cur.fetchall()} | ||
| 61 | |||
| 62 | print(f"目标库命中: {len(target_rows)} / {len(source_ids)}") | ||
| 63 | |||
| 64 | # 3. 计算合并结果 | ||
| 65 | updates: list[dict] = [] | ||
| 66 | skipped: list[dict] = [] | ||
| 67 | |||
| 68 | for s in staging_rows: | ||
| 69 | sid = s["matched_song_id"] | ||
| 70 | target = target_rows.get(str(sid)) | ||
| 71 | if not target: | ||
| 72 | skipped.append({"staging_id": s["staging_id"], "source_song_id": sid, "reason": "目标库未找到"}) | ||
| 73 | continue | ||
| 74 | |||
| 75 | merged_lyricist = _merge_author_field(target["lyricist"], s["lyricist"]) | ||
| 76 | merged_composer = _merge_author_field(target["composer"], s["composer"]) | ||
| 77 | |||
| 78 | lyricist_changed = bool(merged_lyricist) and merged_lyricist != (target["lyricist"] or "") | ||
| 79 | composer_changed = bool(merged_composer) and merged_composer != (target["composer"] or "") | ||
| 80 | |||
| 81 | if not lyricist_changed and not composer_changed: | ||
| 82 | skipped.append({"staging_id": s["staging_id"], "source_song_id": sid, "reason": "合并后无变化"}) | ||
| 83 | continue | ||
| 84 | |||
| 85 | updates.append({ | ||
| 86 | "target_id": target["id"], | ||
| 87 | "name": target["name"], | ||
| 88 | "source_song_id": sid, | ||
| 89 | "staging_id": s["staging_id"], | ||
| 90 | "orig_lyricist": target["lyricist"], | ||
| 91 | "orig_composer": target["composer"], | ||
| 92 | "new_lyricist": merged_lyricist if lyricist_changed else None, | ||
| 93 | "new_composer": merged_composer if composer_changed else None, | ||
| 94 | "staging_lyricist": s["lyricist"], | ||
| 95 | "staging_composer": s["composer"], | ||
| 96 | }) | ||
| 97 | |||
| 98 | print(f"\n需要 UPDATE: {len(updates)} 条") | ||
| 99 | print(f"无需操作: {len(skipped)} 条") | ||
| 100 | |||
| 101 | if dry_run: | ||
| 102 | print("\n[dry-run] 预览变更(前 20 条):") | ||
| 103 | for u in updates[:20]: | ||
| 104 | print(f" id={u['target_id']} name={u['name']}") | ||
| 105 | if u["new_lyricist"] is not None: | ||
| 106 | print(f" lyricist: {u['orig_lyricist']!r} -> {u['new_lyricist']!r}") | ||
| 107 | if u["new_composer"] is not None: | ||
| 108 | print(f" composer: {u['orig_composer']!r} -> {u['new_composer']!r}") | ||
| 109 | return | ||
| 110 | |||
| 111 | # 4. 执行 UPDATE | ||
| 112 | updated = 0 | ||
| 113 | errors = 0 | ||
| 114 | with conn.cursor() as cur: | ||
| 115 | for u in updates: | ||
| 116 | try: | ||
| 117 | if u["new_lyricist"] is not None: | ||
| 118 | cur.execute( | ||
| 119 | f"UPDATE `{TABLE}` SET lyricist = %s WHERE id = %s", | ||
| 120 | (u["new_lyricist"], u["target_id"]), | ||
| 121 | ) | ||
| 122 | if u["new_composer"] is not None: | ||
| 123 | cur.execute( | ||
| 124 | f"UPDATE `{TABLE}` SET composer = %s WHERE id = %s", | ||
| 125 | (u["new_composer"], u["target_id"]), | ||
| 126 | ) | ||
| 127 | updated += 1 | ||
| 128 | except Exception as e: | ||
| 129 | errors += 1 | ||
| 130 | print(f" [错误] target_id={u['target_id']}: {e}") | ||
| 131 | |||
| 132 | conn.commit() | ||
| 133 | print(f"\n完成: 已更新 {updated} 条,错误 {errors} 条") | ||
| 134 | |||
| 135 | if skipped: | ||
| 136 | print(f"\n无需操作的 {len(skipped)} 条:") | ||
| 137 | for s in skipped: | ||
| 138 | print(f" staging_id={s['staging_id']} source_song_id={s['source_song_id']} 原因={s['reason']}") | ||
| 139 | |||
| 140 | finally: | ||
| 141 | conn.close() | ||
| 142 | |||
| 143 | |||
| 144 | if __name__ == "__main__": | ||
| 145 | import argparse | ||
| 146 | parser = argparse.ArgumentParser(description="补处理 merge+skipped 作者合并") | ||
| 147 | parser.add_argument("--dry-run", action="store_true", help="只预览,不写库") | ||
| 148 | args = parser.parse_args() | ||
| 149 | main(dry_run=args.dry_run) |
-
Please register or sign in to post a comment