feat(etl): 实现批次编排与 CLI 入口,支持 --max-batches 冒烟测试参数
Showing
2 changed files
with
27 additions
and
0 deletions
etl_to_crawler/runner.py
0 → 100644
This diff is collapsed.
Click to expand it.
run_etl.py
0 → 100644
| 1 | #!/usr/bin/env python3 | ||
| 2 | import argparse | ||
| 3 | from etl_to_crawler.config import PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE, PLATFORMS | ||
| 4 | from etl_to_crawler.runner import run | ||
| 5 | |||
| 6 | PLATFORM_MAP = { | ||
| 7 | 'qq': PLATFORM_QQ, | ||
| 8 | 'kugou': PLATFORM_KUGOU, | ||
| 9 | 'netease': PLATFORM_NETEASE, | ||
| 10 | } | ||
| 11 | |||
| 12 | if __name__ == '__main__': | ||
| 13 | parser = argparse.ArgumentParser(description='ETL: hk_songs_test → crawler_dev') | ||
| 14 | parser.add_argument('--platform', default='all', | ||
| 15 | choices=['qq', 'kugou', 'netease', 'all'], | ||
| 16 | help='要导入的平台(默认 all)') | ||
| 17 | parser.add_argument('--max-batches', type=int, default=None, | ||
| 18 | help='最多处理多少批次(冒烟测试用)') | ||
| 19 | args = parser.parse_args() | ||
| 20 | |||
| 21 | if args.platform == 'all': | ||
| 22 | platforms = PLATFORMS | ||
| 23 | else: | ||
| 24 | platforms = [PLATFORM_MAP[args.platform]] | ||
| 25 | |||
| 26 | print(f"Starting ETL for platforms: {platforms}") | ||
| 27 | run(platforms, max_batches=args.max_batches) |
-
Please register or sign in to post a comment