Commit ae633f56 ae633f56f1aa24cc8606645f550c60decbb50c25 by 沈秋雨

feat(env): 更新目标库配置及添加 PostgreSQL 环境变量

- 目标库名称由 hk_songs 修改为 hk_songs_test,新增中间表 hk_songs_import_staging
- 添加测试环境 crawler_dev PostgreSQL 连接配置变量
- 添加正式环境 archive_crawler PostgreSQL 连接配置变量

refactor(check_record_titles): 调整平台原始标题核对逻辑

- 变更核对数据库源为 archive_crawler 与 hikoon-data-spider
- 按平台分表批量查询 spider 原始标题替代原录音名查询
- 合并结果中使用 spider_title 与 crawler_title 精确比对
- 输出 Excel 标题及字段相应更新,匹配时整行高亮
- 修改脚本参数说明和默认批处理数量描述
- 日志及信息输出文本调整以反映新数据源

feat(fix_record_titles): 新增基于 spider 标题的归一化批量修复工具

- 支持从 archive_crawler 读取待修复记录的推送关系
- 结合 spider 标题,计算归一化标题和版本号
- 支持对不同平台数据批量生成更新 SQL 预览或应用
- 支持备份更新前数据,支持安全回退操作
- 导出更新报告 Excel,突出显示变化字段
- 支持长文本分批过滤,避免数据库字段长度异常
- 增加命令行参数控制应用范围、批量大小及排除标题列表
- 使用严格的 SQL 语句构造与变更检测优化更新效率
- 实现基于 run_id 的多表回滚机制,保证数据一致性
1 parent 179a08f8
......@@ -5,13 +5,14 @@ SOURCE_DB_USER=
SOURCE_DB_PASSWORD=
SOURCE_DB_NAME=
# 目标库 - 词曲
# 目标库 - 词曲测试
TARGET_DB_HOST=
TARGET_DB_PORT=3306
TARGET_DB_USER=
TARGET_DB_PASSWORD=
TARGET_DB_NAME=
TARGET_TABLE_NAME=hk_songs
TARGET_TABLE_NAME=hk_songs_test
TARGET_TABLE_NAME_TMP=hk_songs_import_staging
# run_etl.py 默认读取并回写的导入状态表:yinyan_song_records 或 yinyan_song_records2
YINYAN_IMPORT_TABLE=yinyan_song_records
......@@ -22,3 +23,21 @@ OSS_ACCESS_KEY_SECRET=
OSS_ENDPOINT=
OSS_BUCKET_NAME=
OSS_FILE_BASE_NAME=
# 测试环境 - crawler_dev PostgreSQL
TEST_CRAWLER_DB_HOST=
TEST_CRAWLER_DB_PORT=5432
TEST_CRAWLER_DB_USER=
TEST_CRAWLER_DB_PASSWORD=
TEST_CRAWLER_DB_NAME=crawler_dev
TEST_CRAWLER_DB_SSL=false
TEST_ARCHIVE_DATA_DB_NAME=data_dev
# 正式环境 - archive_crawler PostgreSQL
ARCHIVE_CRAWLER_DB_HOST=
ARCHIVE_CRAWLER_DB_PORT=5432
ARCHIVE_CRAWLER_DB_USER=
ARCHIVE_CRAWLER_DB_PASSWORD=
ARCHIVE_CRAWLER_DB_NAME=archive_crawler
ARCHIVE_CRAWLER_DB_SSL=false
ARCHIVE_DATA_DB_NAME=archive_data
......
#!/usr/bin/env python3
"""核对音眼录音名与 crawler 平台歌曲标题,并输出 Excel。
"""核对 spider 平台原始标题与正式 crawler 歌曲标题,并输出 Excel。
数据链路:
CRAWLER_DB.yinyan_song_records.record_id
-> SOURCE_DB.hk_music_record.id -> record_name
CRAWLER_DB.yinyan_song_records.(platform, platform_song_id)
-> 对应 crawler_*_songs.platform_song_id -> title
archive_crawler.yinyan_song_records.(platform, platform_song_id)
-> hikoon-data-spider.media_*_songs.id -> spider_title
-> archive_crawler.crawler_*_songs.platform_song_id -> crawler_title
用法:
.venv/bin/python check_record_titles.py
......@@ -21,7 +20,7 @@ from openpyxl import Workbook
from openpyxl.styles import Alignment, Font, PatternFill
from openpyxl.utils import get_column_letter
from etl_to_crawler.connections import close_all_pools, get_pg_conn, get_source_conn
from etl_to_crawler.connections import close_all_pools, get_archive_crawler_conn, get_spider_conn
log = logging.getLogger(__name__)
......@@ -29,6 +28,11 @@ log = logging.getLogger(__name__)
DEFAULT_OUTPUT = "record_title_check.xlsx"
DEFAULT_BATCH_SIZE = 2000
ALLOWED_IMPORT_TABLES = {"yinyan_song_records", "yinyan_song_records2"}
SPIDER_TABLES = {
"1": "media_tencent_songs",
"2": "media_ku_gou_songs",
"4": "media_netease_songs",
}
HEADER_FILL = PatternFill(fill_type="solid", fgColor="4472C4")
MATCH_FILL = PatternFill(fill_type="solid", fgColor="C6EFCE")
......@@ -81,67 +85,77 @@ def fetch_crawler_rows(pg_conn, import_table: str) -> list[dict]:
"platform": row[1],
"platform_song_id": row[2],
"recording_id": row[3],
"title": row[4],
"crawler_title": row[4],
}
for row in rows
]
def fetch_record_names(mysql_conn, record_ids: Iterable[int], batch_size: int) -> dict[int, str | None]:
"""按 record_id 分批读取 hk_music_record.record_name。"""
unique_ids = sorted({int(record_id) for record_id in record_ids if record_id is not None})
names: dict[int, str | None] = {}
for batch in chunked(unique_ids, batch_size):
placeholders = ", ".join(["%s"] * len(batch))
sql = f"""
SELECT id, record_name
FROM hk_music_record
WHERE id IN ({placeholders})
"""
with mysql_conn.cursor() as cursor:
cursor.execute(sql, tuple(batch))
for row in cursor.fetchall():
names[int(row["id"])] = row["record_name"]
return names
def combine_rows(crawler_rows: list[dict], record_names: dict[int, str | None]) -> list[dict]:
def fetch_spider_titles(mysql_conn, crawler_rows: list[dict], batch_size: int) -> dict[tuple[str, int], str]:
"""按 platform_song_id 分平台批量读取 spider 原始 title。"""
titles: dict[tuple[str, int], str] = {}
for platform, table in SPIDER_TABLES.items():
ids = sorted({
int(row["platform_song_id"])
for row in crawler_rows
if str(row["platform"]) == platform and row["platform_song_id"] is not None
})
for batch in chunked(ids, batch_size):
placeholders = ", ".join(["%s"] * len(batch))
with mysql_conn.cursor() as cursor:
cursor.execute(
f"SELECT id, title FROM {table} WHERE id IN ({placeholders})",
tuple(batch),
)
for row in cursor.fetchall():
if row["title"] is not None:
titles[(platform, int(row["id"]))] = str(row["title"])
return titles
def combine_rows(crawler_rows: list[dict], spider_titles: dict[tuple[str, int], str]) -> list[dict]:
"""合并两个数据库的查询结果,并计算是否完全相同。"""
result = []
for row in crawler_rows:
record_id = row["record_id"]
record_name = record_names.get(int(record_id)) if record_id is not None else None
title = row["title"]
platform = str(row["platform"])
platform_song_id = row["platform_song_id"]
spider_title = (
spider_titles.get((platform, int(platform_song_id)))
if platform_song_id is not None else None
)
crawler_title = row["crawler_title"]
result.append(
{
"record_id": record_id,
"record_name": record_name,
"platform": row["platform"],
"platform_song_id": row["platform_song_id"],
"record_id": row["record_id"],
"spider_title": spider_title,
"crawler_title": crawler_title,
"platform": platform,
"platform_song_id": platform_song_id,
"recording_id": row["recording_id"],
"title": title,
# 两边都必须有值;不做 trim、大小写或繁简转换,按库中原值精确比较。
"matched": record_name is not None and title is not None and record_name == title,
"matched": (
spider_title is not None
and crawler_title is not None
and spider_title == crawler_title
),
}
)
return result
def write_excel(rows: list[dict], output_path: Path) -> None:
"""写出三列表格;record_name 与 title 相同时整行标绿。"""
"""写出核对表;spider_title 与 crawler_title 相同时整行标绿。"""
output_path.parent.mkdir(parents=True, exist_ok=True)
workbook = Workbook()
worksheet = workbook.active
worksheet.title = "录音名与标题核对"
worksheet.title = "Spider与Crawler标题核对"
worksheet.freeze_panes = "A2"
headers = (
"record_id",
"record_name",
"title",
"spider_title",
"crawler_title",
"platform",
"platform_song_id",
"recording_id",
......@@ -155,8 +169,8 @@ def write_excel(rows: list[dict], output_path: Path) -> None:
for row_number, row in enumerate(rows, start=2):
values = (
row["record_id"],
row["record_name"],
row["title"],
row["spider_title"],
row["crawler_title"],
row["platform"],
row["platform_song_id"],
row["recording_id"],
......@@ -175,15 +189,15 @@ def write_excel(rows: list[dict], output_path: Path) -> None:
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="核对 hk_music_record.record_name 与 crawler title")
parser = argparse.ArgumentParser(description="核对 spider title 与正式 crawler title")
parser.add_argument("--output", default=DEFAULT_OUTPUT, help=f"Excel 输出路径(默认: {DEFAULT_OUTPUT})")
parser.add_argument(
"--table",
choices=sorted(ALLOWED_IMPORT_TABLES),
default="yinyan_song_records",
help="待核对的 CRAWLER_DB 关联表",
help="待核对的 ARCHIVE_CRAWLER_DB 关联表",
)
parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE, help="MySQL 单批查询数量")
parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE, help="spider MySQL 单批查询数量")
args = parser.parse_args()
if args.batch_size <= 0:
parser.error("--batch-size 必须大于 0")
......@@ -195,28 +209,24 @@ def main() -> None:
output_path = Path(args.output).expanduser().resolve()
try:
pg_conn = get_pg_conn()
mysql_conn = get_source_conn()
pg_conn = get_archive_crawler_conn()
mysql_conn = get_spider_conn()
crawler_rows = fetch_crawler_rows(pg_conn, args.table)
log.info("从 %s 读取 %d 条关联记录", args.table, len(crawler_rows))
record_names = fetch_record_names(
mysql_conn,
(row["record_id"] for row in crawler_rows),
args.batch_size,
)
log.info("从 hk_music_record 读取 %d 个录音名", len(record_names))
spider_titles = fetch_spider_titles(mysql_conn, crawler_rows, args.batch_size)
log.info("从 hikoon-data-spider 读取 %d 个平台标题", len(spider_titles))
rows = combine_rows(crawler_rows, record_names)
rows = combine_rows(crawler_rows, spider_titles)
write_excel(rows, output_path)
matched = sum(row["matched"] for row in rows)
missing_name = sum(row["record_name"] is None for row in rows)
missing_title = sum(row["title"] is None for row in rows)
missing_spider = sum(row["spider_title"] is None for row in rows)
missing_crawler = sum(row["crawler_title"] is None for row in rows)
log.info(
"核对完成:总数=%d,相同=%d,不同=%d,缺少 record_name=%d,缺少 title=%d",
len(rows), matched, len(rows) - matched, missing_name, missing_title,
"核对完成:总数=%d,相同=%d,不同=%d,缺少 spider_title=%d,缺少 crawler_title=%d",
len(rows), matched, len(rows) - matched, missing_spider, missing_crawler,
)
log.info("Excel 已输出至 %s", output_path)
finally:
......
......@@ -21,13 +21,32 @@ HK_SONGS_DB = {
'charset': 'utf8mb4',
}
CRAWLER_DB = {
'host': os.environ['CRAWLER_DB_HOST'],
'port': int(os.environ.get('CRAWLER_DB_PORT', '5432')),
'user': os.environ['CRAWLER_DB_USER'],
'password': os.environ['CRAWLER_DB_PASSWORD'],
'database': os.environ['CRAWLER_DB_NAME'],
'ssl_context': None if os.environ.get('CRAWLER_DB_SSL', 'false').lower() != 'true' else True,
TEST_CRAWLER_DB = {
'host': os.environ['TEST_CRAWLER_DB_HOST'],
'port': int(os.environ.get('TEST_CRAWLER_DB_PORT', '5432')),
'user': os.environ['TEST_CRAWLER_DB_USER'],
'password': os.environ['TEST_CRAWLER_DB_PASSWORD'],
'database': os.environ['TEST_CRAWLER_DB_NAME'],
'ssl_context': None if os.environ.get('TEST_CRAWLER_DB_SSL', 'false').lower() != 'true' else True,
}
TEST_ARCHIVE_DATA_DB = {
**TEST_CRAWLER_DB,
'database': os.environ.get('TEST_ARCHIVE_DATA_DB_NAME', 'data_dev'),
}
ARCHIVE_CRAWLER_DB = {
'host': os.environ['ARCHIVE_CRAWLER_DB_HOST'],
'port': int(os.environ.get('ARCHIVE_CRAWLER_DB_PORT', '5432')),
'user': os.environ['ARCHIVE_CRAWLER_DB_USER'],
'password': os.environ['ARCHIVE_CRAWLER_DB_PASSWORD'],
'database': os.environ.get('ARCHIVE_CRAWLER_DB_NAME', 'archive_crawler'),
'ssl_context': None if os.environ.get('ARCHIVE_CRAWLER_DB_SSL', 'false').lower() != 'true' else True,
}
ARCHIVE_DATA_DB = {
**ARCHIVE_CRAWLER_DB,
'database': os.environ.get('ARCHIVE_DATA_DB_NAME', 'archive_data'),
}
OSS_CONFIG = {
......
......@@ -3,7 +3,16 @@ import pymysql
import pymysql.cursors
import pg8000
import oss2
from .config import SOURCE_DB, HK_SONGS_DB, CRAWLER_DB, OSS_CONFIG, OSS_CONNECTION_POOL_SIZE
from .config import (
ARCHIVE_CRAWLER_DB,
ARCHIVE_DATA_DB,
SOURCE_DB,
HK_SONGS_DB,
TEST_CRAWLER_DB,
TEST_ARCHIVE_DATA_DB,
OSS_CONFIG,
OSS_CONNECTION_POOL_SIZE,
)
CONN_POOL_SIZE = 4
......@@ -238,7 +247,10 @@ class _PgConnPool:
_source_pool = None
_hk_songs_pool = None
_spider_pool = None
_pg_pool = None
_test_crawler_pool = None
_test_archive_pool = None
_archive_pool = None
_archive_crawler_pool = None
def _get_source_pool() -> _MySqlConnPool:
......@@ -264,11 +276,32 @@ def _get_spider_pool() -> _MySqlConnPool:
return _spider_pool
def _get_pg_pool() -> _PgConnPool:
global _pg_pool
if _pg_pool is None:
_pg_pool = _PgConnPool(CRAWLER_DB)
return _pg_pool
def _get_test_crawler_pool() -> _PgConnPool:
global _test_crawler_pool
if _test_crawler_pool is None:
_test_crawler_pool = _PgConnPool(TEST_CRAWLER_DB)
return _test_crawler_pool
def _get_test_archive_pool() -> _PgConnPool:
global _test_archive_pool
if _test_archive_pool is None:
_test_archive_pool = _PgConnPool(TEST_ARCHIVE_DATA_DB)
return _test_archive_pool
def _get_archive_pool() -> _PgConnPool:
global _archive_pool
if _archive_pool is None:
_archive_pool = _PgConnPool(ARCHIVE_DATA_DB)
return _archive_pool
def _get_archive_crawler_pool() -> _PgConnPool:
global _archive_crawler_pool
if _archive_crawler_pool is None:
_archive_crawler_pool = _PgConnPool(ARCHIVE_CRAWLER_DB)
return _archive_crawler_pool
def get_source_conn() -> pymysql.Connection:
......@@ -283,8 +316,20 @@ def get_hk_songs_conn() -> pymysql.Connection:
return _get_hk_songs_pool().get()
def get_pg_conn() -> _NulSafePgConnection:
return _get_pg_pool().get()
def get_test_crawler_conn() -> _NulSafePgConnection:
return _get_test_crawler_pool().get()
def get_test_archive_conn() -> _NulSafePgConnection:
return _get_test_archive_pool().get()
def get_archive_conn() -> _NulSafePgConnection:
return _get_archive_pool().get()
def get_archive_crawler_conn() -> _NulSafePgConnection:
return _get_archive_crawler_pool().get()
def get_oss_bucket() -> oss2.Bucket:
......@@ -298,18 +343,24 @@ def close_all_pools():
for pool in (_source_pool, _hk_songs_pool, _spider_pool):
if pool is not None:
pool.close_all()
if _pg_pool is not None:
_pg_pool.close_all()
if _test_crawler_pool is not None:
_test_crawler_pool.close_all()
if _test_archive_pool is not None:
_test_archive_pool.close_all()
if _archive_pool is not None:
_archive_pool.close_all()
if _archive_crawler_pool is not None:
_archive_crawler_pool.close_all()
def refresh_conn(conn, pool_type: str):
"""验证连接健康度,断开时返回新连接,正常时原样返回。
用法:在批次循环开头调用,如 ``conn = refresh_conn(conn, 'pg')``。
pool_type 取 'source' | 'hk_songs' | 'spider' | 'pg'。
用法:在批次循环开头调用,如 ``conn = refresh_conn(conn, 'test_crawler')``。
pool_type 取 'source' | 'hk_songs' | 'spider' | 'test_crawler'。
"""
try:
if pool_type == 'pg':
if pool_type == 'test_crawler':
conn.run("SELECT 1")
return conn
# MySQL
......@@ -322,7 +373,7 @@ def refresh_conn(conn, pool_type: str):
'source': _get_source_pool,
'hk_songs': _get_hk_songs_pool,
'spider': _get_spider_pool,
'pg': _get_pg_pool,
'test_crawler': _get_test_crawler_pool,
}
pool = pools.get(pool_type)
if pool is None:
......
......@@ -8,7 +8,7 @@ from .config import (
PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE, BATCH_SIZE, BACKFILL_BATCH_SIZE,
OSS_CONFIG, YINYAN_IMPORT_TABLE,
)
from .connections import get_hk_songs_conn, get_source_conn, get_spider_conn, get_pg_conn, get_oss_bucket, refresh_conn, close_all_pools
from .connections import get_hk_songs_conn, get_source_conn, get_spider_conn, get_test_crawler_conn, get_oss_bucket, refresh_conn, close_all_pools
from .reader import (
iter_hk_songs_batches,
fetch_hk_songs_by_source_ids,
......@@ -1337,7 +1337,7 @@ def _pick_record_with_singer(records: list[dict], spider_conn) -> dict | None:
def initialize_yinyan_song_records(platforms: list[str], max_batches: int | None = None) -> None:
hk_conn = get_hk_songs_conn()
src_conn = get_source_conn()
pg_conn = get_pg_conn()
pg_conn = get_test_crawler_conn()
# 查询已存在的 song_id,用于跳过已处理的记录
with pg_conn.cursor() as pg_cur:
......@@ -1351,7 +1351,7 @@ def initialize_yinyan_song_records(platforms: list[str], max_batches: int | None
# 长任务连接可能断开,每批次开始前刷新
hk_conn = refresh_conn(hk_conn, 'hk_songs')
src_conn = refresh_conn(src_conn, 'source')
pg_conn = refresh_conn(pg_conn, 'pg')
pg_conn = refresh_conn(pg_conn, 'test_crawler')
if max_batches is not None and i >= max_batches:
break
......@@ -1416,7 +1416,7 @@ def initialize_yinyan_song_records2(platforms: list[str], max_batches: int | Non
后续导入允许歌曲的 singers 为空。
"""
src_conn = get_source_conn()
pg_conn = get_pg_conn()
pg_conn = get_test_crawler_conn()
total_inserted = 0
try:
......@@ -1427,7 +1427,7 @@ def initialize_yinyan_song_records2(platforms: list[str], max_batches: int | Non
for index, start in enumerate(tqdm(range(0, len(song_ids), BATCH_SIZE), desc='init-yinyan-records2')):
# 长任务连接可能断开,每批次开始前刷新
src_conn = refresh_conn(src_conn, 'source')
pg_conn = refresh_conn(pg_conn, 'pg')
pg_conn = refresh_conn(pg_conn, 'test_crawler')
if max_batches is not None and index >= max_batches:
break
......@@ -1467,7 +1467,7 @@ def initialize_yinyan_song_records2(platforms: list[str], max_batches: int | Non
def backfill_yinyan_record_platforms(max_batches: int | None = None) -> None:
src_conn = get_source_conn()
pg_conn = get_pg_conn()
pg_conn = get_test_crawler_conn()
total = 0
skipped = 0
......@@ -1477,7 +1477,7 @@ def backfill_yinyan_record_platforms(max_batches: int | None = None) -> None:
while max_batches is None or batch_index < max_batches:
# 长任务连接可能断开,每批次开始前刷新
src_conn = refresh_conn(src_conn, 'source')
pg_conn = refresh_conn(pg_conn, 'pg')
pg_conn = refresh_conn(pg_conn, 'test_crawler')
with pg_conn.cursor() as pg_cur:
rows = fetch_yinyan_records_missing_platform(pg_cur, BACKFILL_BATCH_SIZE)
......@@ -1523,7 +1523,7 @@ def _backfill_qq_singers(spider_conn, pg_conn, bucket, base_url, max_batches):
while max_batches is None or batch_index < max_batches:
# 长任务连接可能断开,每批次开始前刷新
spider_conn = refresh_conn(spider_conn, 'spider')
pg_conn = refresh_conn(pg_conn, 'pg')
pg_conn = refresh_conn(pg_conn, 'test_crawler')
with pg_conn.cursor() as pg_cur:
rows = fetch_qq_songs_missing_singers(pg_cur, BATCH_SIZE)
......@@ -1603,7 +1603,7 @@ def _backfill_kugou_singers(spider_conn, pg_conn, bucket, base_url, max_batches)
while max_batches is None or batch_index < max_batches:
# 长任务连接可能断开,每批次开始前刷新
spider_conn = refresh_conn(spider_conn, 'spider')
pg_conn = refresh_conn(pg_conn, 'pg')
pg_conn = refresh_conn(pg_conn, 'test_crawler')
with pg_conn.cursor() as pg_cur:
rows = fetch_kugou_songs_missing_singers(pg_cur, BATCH_SIZE)
......@@ -1677,7 +1677,7 @@ def _backfill_netease_singers(spider_conn, pg_conn, bucket, base_url, max_batche
while max_batches is None or batch_index < max_batches:
# 长任务连接可能断开,每批次开始前刷新
spider_conn = refresh_conn(spider_conn, 'spider')
pg_conn = refresh_conn(pg_conn, 'pg')
pg_conn = refresh_conn(pg_conn, 'test_crawler')
with pg_conn.cursor() as pg_cur:
rows = fetch_netease_songs_missing_singers(pg_cur, BATCH_SIZE)
......@@ -1746,7 +1746,7 @@ def _backfill_netease_singers(spider_conn, pg_conn, bucket, base_url, max_batche
def backfill_empty_singers(platforms: list[str], max_batches: int | None = None) -> None:
spider_conn = get_spider_conn()
pg_conn = get_pg_conn()
pg_conn = get_test_crawler_conn()
bucket = get_oss_bucket()
base_url = OSS_CONFIG['base_url']
try:
......@@ -1769,7 +1769,7 @@ def backfill_qq_invalid_covers(max_batches: int | None = None) -> None:
hk_conn = get_hk_songs_conn()
src_conn = get_source_conn()
spider_conn = get_spider_conn()
pg_conn = get_pg_conn()
pg_conn = get_test_crawler_conn()
bucket = get_oss_bucket()
base_url = OSS_CONFIG['base_url']
batch_index = replaced = filtered = retry_later = 0
......@@ -1780,7 +1780,7 @@ def backfill_qq_invalid_covers(max_batches: int | None = None) -> None:
hk_conn = refresh_conn(hk_conn, 'hk_songs')
src_conn = refresh_conn(src_conn, 'source')
spider_conn = refresh_conn(spider_conn, 'spider')
pg_conn = refresh_conn(pg_conn, 'pg')
pg_conn = refresh_conn(pg_conn, 'test_crawler')
with pg_conn.cursor() as pg_cur:
invalid_rows = fetch_qq_songs_with_invalid_covers(
......@@ -1879,7 +1879,7 @@ def run(
hk_conn = get_hk_songs_conn()
src_conn = get_source_conn()
spider_conn = get_spider_conn()
pg_conn = get_pg_conn()
pg_conn = get_test_crawler_conn()
bucket = get_oss_bucket()
base_url = OSS_CONFIG['base_url']
total_ok = total_err = 0
......@@ -1905,7 +1905,7 @@ def run(
hk_conn = refresh_conn(hk_conn, 'hk_songs')
src_conn = refresh_conn(src_conn, 'source')
spider_conn = refresh_conn(spider_conn, 'spider')
pg_conn = refresh_conn(pg_conn, 'pg')
pg_conn = refresh_conn(pg_conn, 'test_crawler')
with pg_conn.cursor() as pg_cur:
pending_records = fetch_pending_yinyan_song_records(pg_cur, BATCH_SIZE, platforms)
......
......@@ -15,7 +15,7 @@ import argparse
import logging
from etl_to_crawler.config import PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE
from etl_to_crawler.connections import get_source_conn, get_pg_conn, close_all_pools
from etl_to_crawler.connections import get_source_conn, get_test_crawler_conn, close_all_pools
from etl_to_crawler.utils import split_title_version
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
......@@ -134,7 +134,7 @@ def main():
args = parser.parse_args()
mysql_conn = get_source_conn()
pg_conn = get_pg_conn()
pg_conn = get_test_crawler_conn()
total_updated = 0
total_diff = 0
......
import title_norm as tn
def test_normalize_name():
cases = [
("周杰伦", "周杰伦"),
(" 周杰伦 ", "周杰伦"),
("Glass Animals", "glass animals"),
("Jay Chou", "jay chou"),
("", ""),
]
for raw, want in cases:
assert tn.normalize_name(raw) == want, raw
def test_singer_identity_key():
assert tn.singer_identity_key("crawler_qqmusic_singers", 162918) == "qqmusic:162918"
def test_author_identity_key():
assert tn.author_identity_key(" 周杰伦 ") == "name:周杰伦"
def test_platform_from_singer_table():
cases = [
("crawler_qqmusic_singers", "qqmusic"),
("crawler_netease_singers", "netease"),
("crawler_kugou_songs", "kugou"),
]
for raw, want in cases:
assert tn.platform_from_singer_table(raw) == want
def test_split_names():
got = tn.split_names("周杰伦/方文山")
assert got == ["周杰伦", "方文山"]
assert tn.split_names("") is None
def test_normalize_title():
cases = [
("七里香", "七里香", ""),
(" 七里香 ", "七里香", ""),
("七里香(Live)", "七里香", "live"),
("七里香(Live版)", "七里香", "live"),
("七里香 [Remix]", "七里香", "remix"),
("七里香(混音版)", "七里香", "remix"),
("七里香(现场版)", "七里香", "live"),
("七里香(DJ版)", "七里香", "dj版"),
("七里香(ass version)", "七里香", "ass version"),
("ANGEL (天使)", "angel (天使)", ""),
("Radio(Dum-Dum)", "radio(dum-dum)", ""),
("七里香(完整版)", "七里香", ""),
("七里香(原版)", "七里香", ""),
("《七里香》", "七里香", ""),
("《七裡香》(Live)", "七里香", "live"),
("七裡香", "七里香", ""),
("七里香 - DJ版", "七里香", "dj版"),
("七里香 - 周杰伦", "七里香 - 周杰伦", ""),
("七里香—Live", "七里香", "live"),
("七里香(天使)(Live)", "七里香(天使)", "live"),
("七里香 现场版", "七里香", "live"),
]
for raw, want_title, want_ver in cases:
got_title, got_ver = tn.normalize_title(raw)
assert (got_title, got_ver) == (want_title, want_ver), raw
def test_normalize_name_set():
a = tn.normalize_name_set("周杰伦,方文山")
b = tn.normalize_name_set("方文山、周杰伦")
assert a == b
assert tn.normalize_name_set("") is None
def test_strip_clip_markers():
cases = [
("父亲写的散文诗-剪辑版", "父亲写的散文诗"),
("人生路漫漫主歌", "人生路漫漫"),
("Stay with me(氛围beat)-全曲", "Stay with me(氛围beat)"),
("咏春(副歌)", "咏春(副歌)"),
("小气鬼(加速版)", "小气鬼"),
("枪火(剪辑一)", "枪火"),
("恋人", "恋人"),
]
for raw, want in cases:
assert tn.strip_clip_markers(raw) == want, raw
def test_is_version_marker():
cases = [
("DJ版", True),
("钢琴版", True),
("女生版", True),
("完整版", False),
("原版", False),
("ass version", True),
("TV Version", True),
("Extended Version", True),
("live", True),
("Live", True),
("伴奏", True),
("对唱", True),
("dj", True),
("remix", True),
("0.8x", True),
("1.1X", True),
("×0.93", True),
("0.8倍", True),
("1.2倍速", True),
("天使", False),
("Dum-Dum", False),
("周杰伦", False),
("", False),
]
for raw, want in cases:
assert tn.is_version_marker(raw) == want, raw
def test_split_names_clip_authors():
cases = [
("唐伯虎Annie、伯爵Johnny", 2),
("梨香JZH & 口古口古", 2),
("派偉俊/mac ova sea", 2),
("李荣浩", 1),
]
for raw, want in cases:
got = tn.split_names(raw)
assert len(got or []) == want, raw
def test_normalize_set():
a = tn.normalize_set(["方文山", "周杰伦"])
b = tn.normalize_set(["周杰伦", "方文山"])
assert a == b
assert tn.normalize_set(None) is None
......@@ -2,7 +2,7 @@ from pathlib import Path
from openpyxl import load_workbook
from check_record_titles import combine_rows, fetch_crawler_rows, fetch_record_names, write_excel
from check_record_titles import combine_rows, fetch_crawler_rows, fetch_spider_titles, write_excel
class Cursor:
......@@ -42,7 +42,7 @@ def test_fetch_crawler_rows_joins_all_platform_tables():
"platform": "1",
"platform_song_id": 101,
"recording_id": "recording-1",
"title": "同名",
"crawler_title": "同名",
}]
assert "WHERE ysr.is_archive_push = TRUE" in cursor.calls[0][0]
......@@ -58,31 +58,35 @@ def test_fetch_crawler_rows_rejects_unknown_table():
raise AssertionError("应拒绝非白名单表")
def test_fetch_record_names_batches_queries():
def test_fetch_spider_titles_batches_queries_by_platform():
connection = Connection([
[{"id": 1, "record_name": "甲"}, {"id": 2, "record_name": "乙"}],
[{"id": 3, "record_name": "丙"}],
[{"id": 101, "title": "甲"}],
[{"id": 102, "title": "乙"}],
])
crawler_rows = [
{"platform": "1", "platform_song_id": 101},
{"platform": "2", "platform_song_id": 102},
]
names = fetch_record_names(connection, [3, 1, 2, 2], batch_size=2)
titles = fetch_spider_titles(connection, crawler_rows, batch_size=2)
assert names == {1: "甲", 2: "乙", 3: "丙"}
assert titles == {("1", 101): "甲", ("2", 102): "乙"}
def test_excel_marks_only_exact_non_null_matches_green(tmp_path: Path):
crawler_rows = [
{"record_id": 1, "platform": "1", "platform_song_id": 101, "recording_id": "r1", "title": "相同"},
{"record_id": 2, "platform": "2", "platform_song_id": 102, "recording_id": "r2", "title": " 名称 "},
{"record_id": 3, "platform": "4", "platform_song_id": 103, "recording_id": None, "title": None},
{"record_id": 1, "platform": "1", "platform_song_id": 101, "recording_id": "r1", "crawler_title": "相同"},
{"record_id": 2, "platform": "2", "platform_song_id": 102, "recording_id": "r2", "crawler_title": " 名称 "},
{"record_id": 3, "platform": "4", "platform_song_id": 103, "recording_id": None, "crawler_title": None},
]
rows = combine_rows(crawler_rows, {1: "相同", 2: "名称", 3: None})
rows = combine_rows(crawler_rows, {("1", 101): "相同", ("2", 102): "名称"})
output = tmp_path / "check.xlsx"
write_excel(rows, output)
worksheet = load_workbook(output).active
assert [worksheet.cell(1, column).value for column in range(1, 7)] == [
"record_id", "record_name", "title", "platform", "platform_song_id", "recording_id",
"record_id", "spider_title", "crawler_title", "platform", "platform_song_id", "recording_id",
]
assert worksheet["A2"].fill.fgColor.rgb == "00C6EFCE"
assert worksheet["A3"].fill.fill_type is None
......
from openpyxl import load_workbook
from datetime import datetime, timezone
from fix_record_titles import (
InputRow,
_archive_recording_sql,
_archive_source_sql,
_crawler_sql,
_rollback_sql,
_row_value,
build_backup_records,
build_report_rows,
build_fix_rows,
fetch_input_rows,
load_backup,
write_backup_atomic,
write_report,
)
class Cursor:
def __init__(self, rows):
self.rows = rows
self.calls = []
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
return False
def execute(self, sql, params=None):
self.calls.append((sql, params))
def fetchall(self):
return self.rows
class Connection:
def __init__(self, rows):
self.raw_cursor = Cursor(rows)
def cursor(self):
return self.raw_cursor
def test_fetch_input_rows_reads_archive_push_relations_directly():
connection = Connection([
(1, "1", 101, "REC-1"),
(2, "2", 102, "REC-2"),
])
rows, stats = fetch_input_rows(connection)
assert rows == [
InputRow(1, "1", 101, "REC-1"),
InputRow(2, "2", 102, "REC-2"),
]
assert stats == {"total": 2, "missing_key": 0, "unsupported_platform": 0}
sql = connection.raw_cursor.calls[0][0]
assert "FROM yinyan_song_records" in sql
assert "WHERE is_archive_push = TRUE" in sql
def test_fetch_input_rows_applies_limit():
connection = Connection([(1, "1", 101, "REC-1")])
fetch_input_rows(connection, limit=10)
sql, params = connection.raw_cursor.calls[0]
assert "LIMIT %s" in sql
assert params == (10,)
def test_build_fix_rows_uses_spider_title_for_normalization():
input_row = InputRow(1, "1", 101, "REC-1")
rows, fix_stats = build_fix_rows(
[input_row],
{("1", 101): "七里香(Live版)"},
)
assert fix_stats["missing_spider"] == 0
assert rows[0].spider_title == "七里香(Live版)"
assert rows[0].title_norm == "七里香"
assert rows[0].version == "live"
assert _row_value(rows[0], "archive_platform") == "qqmusic"
assert _row_value(rows[0], "archive_version") == "live"
def test_build_fix_rows_skips_missing_spider_title():
input_row = InputRow(1, "4", 101, "REC-1")
rows, fix_stats = build_fix_rows([input_row], {})
assert rows == []
assert fix_stats["missing_spider"] == 1
def test_build_fix_rows_temporarily_excludes_exact_spider_title():
excluded = 'Waiting On A Wish (From "Disney\'s Snow White"/Soundtrack Version|Reprise)'
input_rows = [
InputRow(1, "1", 101, "REC-1"),
InputRow(2, "1", 102, "REC-2"),
]
spider_titles = {
("1", 101): excluded,
("1", 102): "Waiting On A Wish",
}
rows, fix_stats = build_fix_rows(
input_rows,
spider_titles,
exclude_titles={excluded},
)
assert [row.platform_song_id for row in rows] == [102]
assert fix_stats["excluded_title"] == 1
def test_update_sql_uses_spider_values_and_only_updates_differences():
recording_sql, recording_params = _archive_recording_sql(1, True)
source_sql, source_params = _archive_source_sql(1, True)
crawler_sql, crawler_params = _crawler_sql("crawler_qqmusic_songs", 1, True)
assert recording_params == ["recording_id", "spider_title", "title_norm", "archive_version"]
assert "title_norm = source.new_title_norm" in recording_sql
assert "IS DISTINCT FROM source.new_title" in recording_sql
assert "target.platform_song_id = source.platform_song_id" in source_sql
assert "spider_title" in source_params
assert crawler_params == ["platform_song_id", "spider_title", "version"]
assert "source.platform_song_id::bigint" in crawler_sql
assert "version = source.new_version" in crawler_sql
assert "RETURNING target.recording_id" in recording_sql
assert "RETURNING target.platform_song_id" in crawler_sql
locked_sql, _ = _archive_recording_sql(1, False, lock=True)
assert "FOR UPDATE OF target" in locked_sql
def test_report_lists_only_returned_target_keys(tmp_path):
fix = build_fix_rows(
[InputRow(1, "1", 101, "REC-1")],
{("1", 101): "七里香(Live版)"},
)[0][0]
matched = {
"archive_recording": [
("REC-1", "旧标题", "七里香(Live版)", "旧标题", "七里香", None, "live"),
],
"archive_recording_platform_source": [
("REC-1", "qqmusic", "101", "旧标题", "七里香(Live版)", None, "七里香", None, "live"),
],
"crawler_qqmusic_songs": [
(101, "旧标题", "七里香(Live版)", "", "live"),
],
"crawler_kugou_songs": [],
"crawler_netease_songs": [],
}
rows = build_report_rows([fix], matched, apply=False)
output = tmp_path / "report.xlsx"
write_report(output, rows)
worksheet = load_workbook(output).active
assert len(rows) == 3
assert worksheet.max_row == 4
assert worksheet["A1"].value == "database"
assert worksheet["H1"].value == "old_title"
assert worksheet["I1"].value == "new_title"
assert worksheet["H2"].value == "旧标题"
assert worksheet["I2"].value == "七里香(Live版)"
assert worksheet["J3"].value == "<NULL>"
assert worksheet["J3"].fill.fgColor.rgb == "00FFC7CE"
assert worksheet["K3"].fill.fgColor.rgb == "00C6EFCE"
assert worksheet["N2"].value == "would_update"
def test_backup_round_trip_preserves_null_empty_and_updated_at(tmp_path):
fix = build_fix_rows(
[InputRow(1, "1", 101, "REC-1")],
{("1", 101): "七里香(Live版)"},
)[0][0]
updated_at = datetime(2026, 7, 15, 10, 0, tzinfo=timezone.utc)
preview = {
"archive_recording": [
("REC-1", "旧标题", "七里香(Live版)", None, "七里香", None, "live", updated_at),
],
"archive_recording_platform_source": [],
"crawler_qqmusic_songs": [
(101, "旧标题", "七里香(Live版)", "", "live", updated_at.replace(tzinfo=None)),
],
"crawler_kugou_songs": [],
"crawler_netease_songs": [],
}
records = build_backup_records([fix], preview, "run-1")
path = tmp_path / "backup.jsonl"
write_backup_atomic(path, records, "run-1")
metadata, loaded = load_backup(path)
assert metadata["change_count"] == 2
assert loaded[0]["old"]["title_norm"] is None
assert loaded[1]["old"]["version"] == ""
assert loaded[0]["old"]["updated_at"] == updated_at.isoformat()
try:
write_backup_atomic(path, records, "run-2")
except FileExistsError:
pass
else:
raise AssertionError("已有备份不应被覆盖")
def test_rollback_sql_restores_old_values_with_new_value_guard():
archive_sql, _ = _rollback_sql("archive_recording", 1)
crawler_sql, _ = _rollback_sql("crawler_qqmusic_songs", 1)
assert "SET title = source.old_title" in archive_sql
assert "updated_at = source.old_updated_at::timestamptz" in archive_sql
assert "target.title IS NOT DISTINCT FROM source.new_title" in archive_sql
assert "target.title_norm IS NOT DISTINCT FROM source.new_title_norm" in archive_sql
assert "updated_at = source.old_updated_at::timestamp" in crawler_sql
assert "target.version IS NOT DISTINCT FROM source.new_version" in crawler_sql
......@@ -375,7 +375,7 @@ def test_run_imports_only_pending_yinyan_platform_record(monkeypatch):
monkeypatch.setattr(runner, 'get_hk_songs_conn', lambda: _Connection())
monkeypatch.setattr(runner, 'get_source_conn', lambda: _Connection())
monkeypatch.setattr(runner, 'get_spider_conn', lambda: _Connection())
monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'get_test_crawler_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'get_oss_bucket', lambda: object())
monkeypatch.setattr(runner, 'refresh_conn', lambda conn, name: conn)
monkeypatch.setattr(runner, 'fetch_pending_yinyan_song_records', lambda cur, batch_size, platforms: [
......@@ -436,7 +436,7 @@ def test_run_retry_failed_resets_failures_and_restores_hk_sources(monkeypatch):
monkeypatch.setattr(runner, 'get_hk_songs_conn', lambda: hk_conn)
monkeypatch.setattr(runner, 'get_source_conn', lambda: _Connection())
monkeypatch.setattr(runner, 'get_spider_conn', lambda: _Connection())
monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'get_test_crawler_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'get_oss_bucket', lambda: object())
monkeypatch.setattr(runner, 'refresh_conn', lambda conn, name: conn)
monkeypatch.setattr(runner, 'reset_failed_yinyan_song_records', reset)
......@@ -456,7 +456,7 @@ def test_initialize_yinyan_song_records_inserts_primary_records(monkeypatch):
monkeypatch.setattr(runner, 'get_hk_songs_conn', lambda: _Connection())
monkeypatch.setattr(runner, 'get_source_conn', lambda: _Connection())
monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'get_test_crawler_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'refresh_conn', lambda conn, name: conn)
monkeypatch.setattr(runner, 'fetch_existing_yinyan_song_ids', lambda cur: set())
......@@ -501,7 +501,7 @@ def test_initialize_yinyan_song_records2_writes_all_relations_then_deduplicates(
dedupe_calls = []
monkeypatch.setattr(runner, 'get_source_conn', lambda: _Connection())
monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'get_test_crawler_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'refresh_conn', lambda conn, name: conn)
monkeypatch.setattr(runner, 'fetch_yinyan_song_ids', lambda cur: [10])
monkeypatch.setattr(runner, 'fetch_all_song_record_relations', lambda conn, song_ids: [
......@@ -532,7 +532,7 @@ def test_initialize_yinyan_song_records2_keeps_rows_without_valid_singers(monkey
monkeypatch.setattr(runner, 'get_source_conn', lambda: _Connection())
get_spider_conn = MagicMock(side_effect=AssertionError('records2 init must not query spider'))
monkeypatch.setattr(runner, 'get_spider_conn', get_spider_conn)
monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'get_test_crawler_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'refresh_conn', lambda conn, name: conn)
monkeypatch.setattr(runner, 'fetch_yinyan_song_ids', lambda cur: [10])
monkeypatch.setattr(runner, 'fetch_all_song_record_relations', lambda conn, song_ids: [
......@@ -561,7 +561,7 @@ def test_backfill_yinyan_record_platforms_updates_missing_platform_rows(monkeypa
updated = []
monkeypatch.setattr(runner, 'get_source_conn', lambda: _Connection())
monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'get_test_crawler_conn', lambda: pg_conn)
monkeypatch.setattr(runner, 'refresh_conn', lambda conn, name: conn)
monkeypatch.setattr(runner, 'fetch_yinyan_records_missing_platform', lambda cur, batch_size: [
{'song_id': 10, 'record_id': 100},
......