feat(check): 新增录音名与平台歌曲标题核对工具
Showing
1 changed file
with
228 additions
and
0 deletions
check_record_titles.py
0 → 100644
| 1 | #!/usr/bin/env python3 | ||
| 2 | """核对音眼录音名与 crawler 平台歌曲标题,并输出 Excel。 | ||
| 3 | |||
| 4 | 数据链路: | ||
| 5 | CRAWLER_DB.yinyan_song_records.record_id | ||
| 6 | -> SOURCE_DB.hk_music_record.id -> record_name | ||
| 7 | CRAWLER_DB.yinyan_song_records.(platform, platform_song_id) | ||
| 8 | -> 对应 crawler_*_songs.platform_song_id -> title | ||
| 9 | |||
| 10 | 用法: | ||
| 11 | .venv/bin/python check_record_titles.py | ||
| 12 | .venv/bin/python check_record_titles.py --output output/record_title_check.xlsx | ||
| 13 | """ | ||
| 14 | |||
| 15 | import argparse | ||
| 16 | import logging | ||
| 17 | from pathlib import Path | ||
| 18 | from typing import Iterable | ||
| 19 | |||
| 20 | from openpyxl import Workbook | ||
| 21 | from openpyxl.styles import Alignment, Font, PatternFill | ||
| 22 | from openpyxl.utils import get_column_letter | ||
| 23 | |||
| 24 | from etl_to_crawler.connections import close_all_pools, get_pg_conn, get_source_conn | ||
| 25 | |||
| 26 | |||
| 27 | log = logging.getLogger(__name__) | ||
| 28 | |||
| 29 | DEFAULT_OUTPUT = "record_title_check.xlsx" | ||
| 30 | DEFAULT_BATCH_SIZE = 2000 | ||
| 31 | ALLOWED_IMPORT_TABLES = {"yinyan_song_records", "yinyan_song_records2"} | ||
| 32 | |||
| 33 | HEADER_FILL = PatternFill(fill_type="solid", fgColor="4472C4") | ||
| 34 | MATCH_FILL = PatternFill(fill_type="solid", fgColor="C6EFCE") | ||
| 35 | HEADER_FONT = Font(color="FFFFFF", bold=True) | ||
| 36 | |||
| 37 | |||
| 38 | def chunked(values: list[int], size: int) -> Iterable[list[int]]: | ||
| 39 | """将列表切成固定大小的批次。""" | ||
| 40 | for start in range(0, len(values), size): | ||
| 41 | yield values[start:start + size] | ||
| 42 | |||
| 43 | |||
| 44 | def fetch_crawler_rows(pg_conn, import_table: str) -> list[dict]: | ||
| 45 | """读取关联记录,并按 platform 从对应平台歌曲表取得 title。""" | ||
| 46 | if import_table not in ALLOWED_IMPORT_TABLES: | ||
| 47 | raise ValueError(f"不支持的导入表: {import_table}") | ||
| 48 | |||
| 49 | sql = f""" | ||
| 50 | SELECT | ||
| 51 | ysr.record_id, | ||
| 52 | ysr.platform, | ||
| 53 | ysr.platform_song_id, | ||
| 54 | ysr.recording_id, | ||
| 55 | CASE ysr.platform | ||
| 56 | WHEN '1' THEN qq.title | ||
| 57 | WHEN '2' THEN kg.title | ||
| 58 | WHEN '4' THEN ne.title | ||
| 59 | ELSE NULL | ||
| 60 | END AS title | ||
| 61 | FROM {import_table} AS ysr | ||
| 62 | LEFT JOIN crawler_qqmusic_songs AS qq | ||
| 63 | ON ysr.platform = '1' | ||
| 64 | AND ysr.platform_song_id = qq.platform_song_id | ||
| 65 | LEFT JOIN crawler_kugou_songs AS kg | ||
| 66 | ON ysr.platform = '2' | ||
| 67 | AND ysr.platform_song_id = kg.platform_song_id | ||
| 68 | LEFT JOIN crawler_netease_songs AS ne | ||
| 69 | ON ysr.platform = '4' | ||
| 70 | AND ysr.platform_song_id = ne.platform_song_id | ||
| 71 | WHERE ysr.is_archive_push = TRUE | ||
| 72 | ORDER BY ysr.record_id, ysr.platform, ysr.platform_song_id | ||
| 73 | """ | ||
| 74 | with pg_conn.cursor() as cursor: | ||
| 75 | cursor.execute(sql) | ||
| 76 | rows = cursor.fetchall() | ||
| 77 | |||
| 78 | return [ | ||
| 79 | { | ||
| 80 | "record_id": row[0], | ||
| 81 | "platform": row[1], | ||
| 82 | "platform_song_id": row[2], | ||
| 83 | "recording_id": row[3], | ||
| 84 | "title": row[4], | ||
| 85 | } | ||
| 86 | for row in rows | ||
| 87 | ] | ||
| 88 | |||
| 89 | |||
| 90 | def fetch_record_names(mysql_conn, record_ids: Iterable[int], batch_size: int) -> dict[int, str | None]: | ||
| 91 | """按 record_id 分批读取 hk_music_record.record_name。""" | ||
| 92 | unique_ids = sorted({int(record_id) for record_id in record_ids if record_id is not None}) | ||
| 93 | names: dict[int, str | None] = {} | ||
| 94 | |||
| 95 | for batch in chunked(unique_ids, batch_size): | ||
| 96 | placeholders = ", ".join(["%s"] * len(batch)) | ||
| 97 | sql = f""" | ||
| 98 | SELECT id, record_name | ||
| 99 | FROM hk_music_record | ||
| 100 | WHERE id IN ({placeholders}) | ||
| 101 | """ | ||
| 102 | with mysql_conn.cursor() as cursor: | ||
| 103 | cursor.execute(sql, tuple(batch)) | ||
| 104 | for row in cursor.fetchall(): | ||
| 105 | names[int(row["id"])] = row["record_name"] | ||
| 106 | |||
| 107 | return names | ||
| 108 | |||
| 109 | |||
| 110 | def combine_rows(crawler_rows: list[dict], record_names: dict[int, str | None]) -> list[dict]: | ||
| 111 | """合并两个数据库的查询结果,并计算是否完全相同。""" | ||
| 112 | result = [] | ||
| 113 | for row in crawler_rows: | ||
| 114 | record_id = row["record_id"] | ||
| 115 | record_name = record_names.get(int(record_id)) if record_id is not None else None | ||
| 116 | title = row["title"] | ||
| 117 | result.append( | ||
| 118 | { | ||
| 119 | "record_id": record_id, | ||
| 120 | "record_name": record_name, | ||
| 121 | "platform": row["platform"], | ||
| 122 | "platform_song_id": row["platform_song_id"], | ||
| 123 | "recording_id": row["recording_id"], | ||
| 124 | "title": title, | ||
| 125 | # 两边都必须有值;不做 trim、大小写或繁简转换,按库中原值精确比较。 | ||
| 126 | "matched": record_name is not None and title is not None and record_name == title, | ||
| 127 | } | ||
| 128 | ) | ||
| 129 | return result | ||
| 130 | |||
| 131 | |||
| 132 | def write_excel(rows: list[dict], output_path: Path) -> None: | ||
| 133 | """写出三列表格;record_name 与 title 相同时整行标绿。""" | ||
| 134 | output_path.parent.mkdir(parents=True, exist_ok=True) | ||
| 135 | |||
| 136 | workbook = Workbook() | ||
| 137 | worksheet = workbook.active | ||
| 138 | worksheet.title = "录音名与标题核对" | ||
| 139 | worksheet.freeze_panes = "A2" | ||
| 140 | |||
| 141 | headers = ( | ||
| 142 | "record_id", | ||
| 143 | "record_name", | ||
| 144 | "title", | ||
| 145 | "platform", | ||
| 146 | "platform_song_id", | ||
| 147 | "recording_id", | ||
| 148 | ) | ||
| 149 | for column, header in enumerate(headers, start=1): | ||
| 150 | cell = worksheet.cell(row=1, column=column, value=header) | ||
| 151 | cell.fill = HEADER_FILL | ||
| 152 | cell.font = HEADER_FONT | ||
| 153 | cell.alignment = Alignment(horizontal="center", vertical="center") | ||
| 154 | |||
| 155 | for row_number, row in enumerate(rows, start=2): | ||
| 156 | values = ( | ||
| 157 | row["record_id"], | ||
| 158 | row["record_name"], | ||
| 159 | row["title"], | ||
| 160 | row["platform"], | ||
| 161 | row["platform_song_id"], | ||
| 162 | row["recording_id"], | ||
| 163 | ) | ||
| 164 | for column, value in enumerate(values, start=1): | ||
| 165 | cell = worksheet.cell(row=row_number, column=column, value=value) | ||
| 166 | cell.alignment = Alignment(vertical="top", wrap_text=True) | ||
| 167 | if row["matched"]: | ||
| 168 | cell.fill = MATCH_FILL | ||
| 169 | |||
| 170 | for column, width in enumerate((16, 48, 48, 12, 20, 40), start=1): | ||
| 171 | worksheet.column_dimensions[get_column_letter(column)].width = width | ||
| 172 | worksheet.auto_filter.ref = f"A1:F{max(1, worksheet.max_row)}" | ||
| 173 | |||
| 174 | workbook.save(output_path) | ||
| 175 | |||
| 176 | |||
| 177 | def parse_args() -> argparse.Namespace: | ||
| 178 | parser = argparse.ArgumentParser(description="核对 hk_music_record.record_name 与 crawler title") | ||
| 179 | parser.add_argument("--output", default=DEFAULT_OUTPUT, help=f"Excel 输出路径(默认: {DEFAULT_OUTPUT})") | ||
| 180 | parser.add_argument( | ||
| 181 | "--table", | ||
| 182 | choices=sorted(ALLOWED_IMPORT_TABLES), | ||
| 183 | default="yinyan_song_records", | ||
| 184 | help="待核对的 CRAWLER_DB 关联表", | ||
| 185 | ) | ||
| 186 | parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE, help="MySQL 单批查询数量") | ||
| 187 | args = parser.parse_args() | ||
| 188 | if args.batch_size <= 0: | ||
| 189 | parser.error("--batch-size 必须大于 0") | ||
| 190 | return args | ||
| 191 | |||
| 192 | |||
| 193 | def main() -> None: | ||
| 194 | args = parse_args() | ||
| 195 | output_path = Path(args.output).expanduser().resolve() | ||
| 196 | |||
| 197 | try: | ||
| 198 | pg_conn = get_pg_conn() | ||
| 199 | mysql_conn = get_source_conn() | ||
| 200 | |||
| 201 | crawler_rows = fetch_crawler_rows(pg_conn, args.table) | ||
| 202 | log.info("从 %s 读取 %d 条关联记录", args.table, len(crawler_rows)) | ||
| 203 | |||
| 204 | record_names = fetch_record_names( | ||
| 205 | mysql_conn, | ||
| 206 | (row["record_id"] for row in crawler_rows), | ||
| 207 | args.batch_size, | ||
| 208 | ) | ||
| 209 | log.info("从 hk_music_record 读取 %d 个录音名", len(record_names)) | ||
| 210 | |||
| 211 | rows = combine_rows(crawler_rows, record_names) | ||
| 212 | write_excel(rows, output_path) | ||
| 213 | |||
| 214 | matched = sum(row["matched"] for row in rows) | ||
| 215 | missing_name = sum(row["record_name"] is None for row in rows) | ||
| 216 | missing_title = sum(row["title"] is None for row in rows) | ||
| 217 | log.info( | ||
| 218 | "核对完成:总数=%d,相同=%d,不同=%d,缺少 record_name=%d,缺少 title=%d", | ||
| 219 | len(rows), matched, len(rows) - matched, missing_name, missing_title, | ||
| 220 | ) | ||
| 221 | log.info("Excel 已输出至 %s", output_path) | ||
| 222 | finally: | ||
| 223 | close_all_pools() | ||
| 224 | |||
| 225 | |||
| 226 | if __name__ == "__main__": | ||
| 227 | logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") | ||
| 228 | main() |
-
Please register or sign in to post a comment