run_etl.py
1.6 KB
#!/usr/bin/env python3
import argparse
from etl_to_crawler.config import PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE, PLATFORMS
from etl_to_crawler.runner import backfill_yinyan_record_platforms, initialize_yinyan_song_records, run
PLATFORM_MAP = {
'qq': PLATFORM_QQ,
'kugou': PLATFORM_KUGOU,
'netease': PLATFORM_NETEASE,
}
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='ETL: hk_songs_test → crawler_dev')
parser.add_argument('--platform', default='all',
choices=['qq', 'kugou', 'netease', 'all'],
help='要导入的平台(默认 all)')
parser.add_argument('--max-batches', type=int, default=None,
help='最多处理多少批次(冒烟测试用)')
parser.add_argument('--init-yinyan-records', action='store_true',
help='只初始化 yinyan_song_records 待导入状态表,不执行 crawler 导入')
parser.add_argument('--backfill-yinyan-platforms', action='store_true',
help='只回填 yinyan_song_records 中为空的 platform 平台代码')
args = parser.parse_args()
if args.platform == 'all':
platforms = PLATFORMS
else:
platforms = [PLATFORM_MAP[args.platform]]
print(f"Starting ETL for platforms: {platforms}")
if args.backfill_yinyan_platforms:
backfill_yinyan_record_platforms(max_batches=args.max_batches)
elif args.init_yinyan_records:
initialize_yinyan_song_records(platforms, max_batches=args.max_batches)
else:
run(platforms, max_batches=args.max_batches)