Commit bc4f610a bc4f610a0326dd65c093f4e0b858cec9a3fbd744 by 沈秋雨

feat(etl): runner 运行后打印导入记录列表

1 parent 62fc7695
......@@ -125,6 +125,7 @@ def _process_qq(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url):
upsert_qq_singer_songs(pg_cur, [(sg['singer_id'], song_uuid) for sg in singer_list])
if album_id:
upsert_qq_singer_albums(pg_cur, [(sg['singer_id'], album_id) for sg in singer_list])
return {'platform': 'qq', 'platform_song_id': platform_song_id, 'mid': mid, 'title': hk_row['name']}
def _process_kugou(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url):
......@@ -211,6 +212,7 @@ def _process_kugou(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url
upsert_kugou_singer_songs(pg_cur, [(sg['singer_id'], song_uuid) for sg in singer_list])
if album_id:
upsert_kugou_singer_albums(pg_cur, [(sg['singer_id'], album_id) for sg in singer_list])
return {'platform': 'kugou', 'platform_song_id': song_id, 'hash': sp.get('hid', ''), 'title': hk_row['name']}
def _process_netease(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url):
......@@ -295,6 +297,7 @@ def _process_netease(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_u
upsert_netease_singer_songs(pg_cur, [(sg['singer_id'], song_uuid) for sg in singer_list])
if album_id:
upsert_netease_singer_albums(pg_cur, [(sg['singer_id'], album_id) for sg in singer_list])
return {'platform': 'netease', 'platform_song_id': song_id, 'title': hk_row['name']}
_PROCESSORS = {
......@@ -313,6 +316,7 @@ def run(platforms: list[str], max_batches: int | None = None) -> None:
base_url = OSS_CONFIG['base_url']
total_ok = total_err = 0
imported: list[dict] = []
try:
for i, batch in enumerate(tqdm(iter_hk_songs_batches(hk_conn, BATCH_SIZE), desc='batches')):
......@@ -338,9 +342,11 @@ def run(platforms: list[str], max_batches: int | None = None) -> None:
continue
pg_cur.execute('SAVEPOINT sp_song')
try:
processor(hk_row, pr, spider_conn, pg_cur, bucket, base_url)
result = processor(hk_row, pr, spider_conn, pg_cur, bucket, base_url)
pg_cur.execute('RELEASE SAVEPOINT sp_song')
total_ok += 1
if result:
imported.append(result)
except Exception as e:
pg_cur.execute('ROLLBACK TO SAVEPOINT sp_song')
pg_cur.execute('RELEASE SAVEPOINT sp_song')
......@@ -356,3 +362,13 @@ def run(platforms: list[str], max_batches: int | None = None) -> None:
pg_conn.close()
log.info("Done. OK=%d ERR=%d", total_ok, total_err)
if imported:
print(f"\n导入记录(共 {len(imported)} 条):")
for r in imported:
p = r['platform']
if p == 'qq':
print(f" [QQ] platform_song_id={r['platform_song_id']} mid={r['mid']} title={r['title']}")
elif p == 'kugou':
print(f" [酷狗] platform_song_id={r['platform_song_id']} hash={r['hash']} title={r['title']}")
elif p == 'netease':
print(f" [网易] platform_song_id={r['platform_song_id']} title={r['title']}")
......