Commit 62fc7695 62fc7695c58a9ab67ba41c37e2e9b01617e853bd by 沈秋雨

feat(etl): 实现批次编排与 CLI 入口,支持 --max-batches 冒烟测试参数

1 parent 0a14389c
#!/usr/bin/env python3
import argparse
from etl_to_crawler.config import PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE, PLATFORMS
from etl_to_crawler.runner import run
PLATFORM_MAP = {
'qq': PLATFORM_QQ,
'kugou': PLATFORM_KUGOU,
'netease': PLATFORM_NETEASE,
}
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='ETL: hk_songs_test → crawler_dev')
parser.add_argument('--platform', default='all',
choices=['qq', 'kugou', 'netease', 'all'],
help='要导入的平台(默认 all)')
parser.add_argument('--max-batches', type=int, default=None,
help='最多处理多少批次(冒烟测试用)')
args = parser.parse_args()
if args.platform == 'all':
platforms = PLATFORMS
else:
platforms = [PLATFORM_MAP[args.platform]]
print(f"Starting ETL for platforms: {platforms}")
run(platforms, max_batches=args.max_batches)