Commit 0f853eb5 0f853eb5f4f47c9139d79e866410a37927653c46 by 沈秋雨

feat(check): 新增录音名与平台歌曲标题核对工具

1 parent 59e59a59
1 #!/usr/bin/env python3
2 """核对音眼录音名与 crawler 平台歌曲标题,并输出 Excel。
3
4 数据链路:
5 CRAWLER_DB.yinyan_song_records.record_id
6 -> SOURCE_DB.hk_music_record.id -> record_name
7 CRAWLER_DB.yinyan_song_records.(platform, platform_song_id)
8 -> 对应 crawler_*_songs.platform_song_id -> title
9
10 用法:
11 .venv/bin/python check_record_titles.py
12 .venv/bin/python check_record_titles.py --output output/record_title_check.xlsx
13 """
14
15 import argparse
16 import logging
17 from pathlib import Path
18 from typing import Iterable
19
20 from openpyxl import Workbook
21 from openpyxl.styles import Alignment, Font, PatternFill
22 from openpyxl.utils import get_column_letter
23
24 from etl_to_crawler.connections import close_all_pools, get_pg_conn, get_source_conn
25
26
27 log = logging.getLogger(__name__)
28
29 DEFAULT_OUTPUT = "record_title_check.xlsx"
30 DEFAULT_BATCH_SIZE = 2000
31 ALLOWED_IMPORT_TABLES = {"yinyan_song_records", "yinyan_song_records2"}
32
33 HEADER_FILL = PatternFill(fill_type="solid", fgColor="4472C4")
34 MATCH_FILL = PatternFill(fill_type="solid", fgColor="C6EFCE")
35 HEADER_FONT = Font(color="FFFFFF", bold=True)
36
37
38 def chunked(values: list[int], size: int) -> Iterable[list[int]]:
39 """将列表切成固定大小的批次。"""
40 for start in range(0, len(values), size):
41 yield values[start:start + size]
42
43
44 def fetch_crawler_rows(pg_conn, import_table: str) -> list[dict]:
45 """读取关联记录,并按 platform 从对应平台歌曲表取得 title。"""
46 if import_table not in ALLOWED_IMPORT_TABLES:
47 raise ValueError(f"不支持的导入表: {import_table}")
48
49 sql = f"""
50 SELECT
51 ysr.record_id,
52 ysr.platform,
53 ysr.platform_song_id,
54 ysr.recording_id,
55 CASE ysr.platform
56 WHEN '1' THEN qq.title
57 WHEN '2' THEN kg.title
58 WHEN '4' THEN ne.title
59 ELSE NULL
60 END AS title
61 FROM {import_table} AS ysr
62 LEFT JOIN crawler_qqmusic_songs AS qq
63 ON ysr.platform = '1'
64 AND ysr.platform_song_id = qq.platform_song_id
65 LEFT JOIN crawler_kugou_songs AS kg
66 ON ysr.platform = '2'
67 AND ysr.platform_song_id = kg.platform_song_id
68 LEFT JOIN crawler_netease_songs AS ne
69 ON ysr.platform = '4'
70 AND ysr.platform_song_id = ne.platform_song_id
71 WHERE ysr.is_archive_push = TRUE
72 ORDER BY ysr.record_id, ysr.platform, ysr.platform_song_id
73 """
74 with pg_conn.cursor() as cursor:
75 cursor.execute(sql)
76 rows = cursor.fetchall()
77
78 return [
79 {
80 "record_id": row[0],
81 "platform": row[1],
82 "platform_song_id": row[2],
83 "recording_id": row[3],
84 "title": row[4],
85 }
86 for row in rows
87 ]
88
89
90 def fetch_record_names(mysql_conn, record_ids: Iterable[int], batch_size: int) -> dict[int, str | None]:
91 """按 record_id 分批读取 hk_music_record.record_name。"""
92 unique_ids = sorted({int(record_id) for record_id in record_ids if record_id is not None})
93 names: dict[int, str | None] = {}
94
95 for batch in chunked(unique_ids, batch_size):
96 placeholders = ", ".join(["%s"] * len(batch))
97 sql = f"""
98 SELECT id, record_name
99 FROM hk_music_record
100 WHERE id IN ({placeholders})
101 """
102 with mysql_conn.cursor() as cursor:
103 cursor.execute(sql, tuple(batch))
104 for row in cursor.fetchall():
105 names[int(row["id"])] = row["record_name"]
106
107 return names
108
109
110 def combine_rows(crawler_rows: list[dict], record_names: dict[int, str | None]) -> list[dict]:
111 """合并两个数据库的查询结果,并计算是否完全相同。"""
112 result = []
113 for row in crawler_rows:
114 record_id = row["record_id"]
115 record_name = record_names.get(int(record_id)) if record_id is not None else None
116 title = row["title"]
117 result.append(
118 {
119 "record_id": record_id,
120 "record_name": record_name,
121 "platform": row["platform"],
122 "platform_song_id": row["platform_song_id"],
123 "recording_id": row["recording_id"],
124 "title": title,
125 # 两边都必须有值;不做 trim、大小写或繁简转换,按库中原值精确比较。
126 "matched": record_name is not None and title is not None and record_name == title,
127 }
128 )
129 return result
130
131
132 def write_excel(rows: list[dict], output_path: Path) -> None:
133 """写出三列表格;record_name 与 title 相同时整行标绿。"""
134 output_path.parent.mkdir(parents=True, exist_ok=True)
135
136 workbook = Workbook()
137 worksheet = workbook.active
138 worksheet.title = "录音名与标题核对"
139 worksheet.freeze_panes = "A2"
140
141 headers = (
142 "record_id",
143 "record_name",
144 "title",
145 "platform",
146 "platform_song_id",
147 "recording_id",
148 )
149 for column, header in enumerate(headers, start=1):
150 cell = worksheet.cell(row=1, column=column, value=header)
151 cell.fill = HEADER_FILL
152 cell.font = HEADER_FONT
153 cell.alignment = Alignment(horizontal="center", vertical="center")
154
155 for row_number, row in enumerate(rows, start=2):
156 values = (
157 row["record_id"],
158 row["record_name"],
159 row["title"],
160 row["platform"],
161 row["platform_song_id"],
162 row["recording_id"],
163 )
164 for column, value in enumerate(values, start=1):
165 cell = worksheet.cell(row=row_number, column=column, value=value)
166 cell.alignment = Alignment(vertical="top", wrap_text=True)
167 if row["matched"]:
168 cell.fill = MATCH_FILL
169
170 for column, width in enumerate((16, 48, 48, 12, 20, 40), start=1):
171 worksheet.column_dimensions[get_column_letter(column)].width = width
172 worksheet.auto_filter.ref = f"A1:F{max(1, worksheet.max_row)}"
173
174 workbook.save(output_path)
175
176
177 def parse_args() -> argparse.Namespace:
178 parser = argparse.ArgumentParser(description="核对 hk_music_record.record_name 与 crawler title")
179 parser.add_argument("--output", default=DEFAULT_OUTPUT, help=f"Excel 输出路径(默认: {DEFAULT_OUTPUT})")
180 parser.add_argument(
181 "--table",
182 choices=sorted(ALLOWED_IMPORT_TABLES),
183 default="yinyan_song_records",
184 help="待核对的 CRAWLER_DB 关联表",
185 )
186 parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE, help="MySQL 单批查询数量")
187 args = parser.parse_args()
188 if args.batch_size <= 0:
189 parser.error("--batch-size 必须大于 0")
190 return args
191
192
193 def main() -> None:
194 args = parse_args()
195 output_path = Path(args.output).expanduser().resolve()
196
197 try:
198 pg_conn = get_pg_conn()
199 mysql_conn = get_source_conn()
200
201 crawler_rows = fetch_crawler_rows(pg_conn, args.table)
202 log.info("从 %s 读取 %d 条关联记录", args.table, len(crawler_rows))
203
204 record_names = fetch_record_names(
205 mysql_conn,
206 (row["record_id"] for row in crawler_rows),
207 args.batch_size,
208 )
209 log.info("从 hk_music_record 读取 %d 个录音名", len(record_names))
210
211 rows = combine_rows(crawler_rows, record_names)
212 write_excel(rows, output_path)
213
214 matched = sum(row["matched"] for row in rows)
215 missing_name = sum(row["record_name"] is None for row in rows)
216 missing_title = sum(row["title"] is None for row in rows)
217 log.info(
218 "核对完成:总数=%d,相同=%d,不同=%d,缺少 record_name=%d,缺少 title=%d",
219 len(rows), matched, len(rows) - matched, missing_name, missing_title,
220 )
221 log.info("Excel 已输出至 %s", output_path)
222 finally:
223 close_all_pools()
224
225
226 if __name__ == "__main__":
227 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
228 main()