Commit 62fc7695 62fc7695c58a9ab67ba41c37e2e9b01617e853bd by 沈秋雨

feat(etl): 实现批次编排与 CLI 入口,支持 --max-batches 冒烟测试参数

1 parent 0a14389c
1 #!/usr/bin/env python3
2 import argparse
3 from etl_to_crawler.config import PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE, PLATFORMS
4 from etl_to_crawler.runner import run
5
6 PLATFORM_MAP = {
7 'qq': PLATFORM_QQ,
8 'kugou': PLATFORM_KUGOU,
9 'netease': PLATFORM_NETEASE,
10 }
11
12 if __name__ == '__main__':
13 parser = argparse.ArgumentParser(description='ETL: hk_songs_test → crawler_dev')
14 parser.add_argument('--platform', default='all',
15 choices=['qq', 'kugou', 'netease', 'all'],
16 help='要导入的平台(默认 all)')
17 parser.add_argument('--max-batches', type=int, default=None,
18 help='最多处理多少批次(冒烟测试用)')
19 args = parser.parse_args()
20
21 if args.platform == 'all':
22 platforms = PLATFORMS
23 else:
24 platforms = [PLATFORM_MAP[args.platform]]
25
26 print(f"Starting ETL for platforms: {platforms}")
27 run(platforms, max_batches=args.max_batches)