run_etl.py
2.62 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
#!/usr/bin/env python3
import argparse
from etl_to_crawler.config import PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE, PLATFORMS, YINYAN_IMPORT_TABLE
from etl_to_crawler.runner import backfill_yinyan_record_platforms, initialize_yinyan_song_records, initialize_yinyan_song_records2, run, backfill_empty_singers, backfill_qq_invalid_covers
PLATFORM_MAP = {
'qq': PLATFORM_QQ,
'kugou': PLATFORM_KUGOU,
'netease': PLATFORM_NETEASE,
}
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='ETL: hk_songs_test → crawler_dev')
parser.add_argument('--platform', default='all',
choices=['qq', 'kugou', 'netease', 'all'],
help='要导入的平台（默认 all）')
parser.add_argument('--max-batches', type=int, default=None,
help='最多处理多少批次（冒烟测试用）')
parser.add_argument('--init-yinyan-records', action='store_true',
help='只初始化 yinyan_song_records 待导入状态表，不执行 crawler 导入')
parser.add_argument('--init-yinyan-records2', action='store_true',
help='写入全部合格 song-record 关联至 yinyan_song_records2，再排除 yinyan_song_records 已有关系')
parser.add_argument('--backfill-yinyan-platforms', action='store_true',
help='只回填 yinyan_song_records 中为空的 platform 平台代码')
parser.add_argument('--backfill-empty-singers', action='store_true',
help='回填三平台 singers 为空的歌曲记录')
parser.add_argument('--backfill-qq-invalid-covers', action='store_true',
help='替换 QQ 无专辑占位封面的录音；无候选时删除 crawler 记录并软删 hk_songs_test')
args = parser.parse_args()
if args.platform == 'all':
platforms = PLATFORMS
else:
platforms = [PLATFORM_MAP[args.platform]]
print(f"Starting ETL for platforms: {platforms}, import table: {YINYAN_IMPORT_TABLE}")
if args.backfill_yinyan_platforms:
backfill_yinyan_record_platforms(max_batches=args.max_batches)
elif args.init_yinyan_records:
initialize_yinyan_song_records(platforms, max_batches=args.max_batches)
elif args.init_yinyan_records2:
initialize_yinyan_song_records2(platforms, max_batches=args.max_batches)
elif args.backfill_empty_singers:
backfill_empty_singers(platforms, max_batches=args.max_batches)
elif args.backfill_qq_invalid_covers:
backfill_qq_invalid_covers(max_batches=args.max_batches)
else:
run(platforms, max_batches=args.max_batches)