Commit d4e1bfc7 d4e1bfc75f6a5ae9a7d21bb628ccb16aeb9f470b by 沈秋雨

feat(etl): 初始化 etl_to_crawler 模块,验证三端连接

- 创建 etl_to_crawler/__init__.py, config.py, connections.py
- 使用 pg8000 替代 psycopg2-binary(macOS 二进制兼容问题)
- 验证 hikoon-data / hikoon-data-spider / new_music_library / crawler_dev / OSS 全部 OK
- 更新 requirements.txt

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 666fd63f
1 import os
2 from dotenv import load_dotenv
3
4 load_dotenv()
5
6 SOURCE_DB = {
7 'host': os.environ['SOURCE_DB_HOST'],
8 'port': int(os.environ['SOURCE_DB_PORT']),
9 'user': os.environ['SOURCE_DB_USER'],
10 'password': os.environ['SOURCE_DB_PASSWORD'],
11 'database': os.environ['SOURCE_DB_NAME'],
12 'charset': 'utf8mb4',
13 }
14
15 HK_SONGS_DB = {
16 'host': os.environ['TARGET_DB_HOST'],
17 'port': int(os.environ['TARGET_DB_PORT']),
18 'user': os.environ['TARGET_DB_USER'],
19 'password': os.environ['TARGET_DB_PASSWORD'],
20 'database': os.environ['TARGET_DB_NAME'],
21 'charset': 'utf8mb4',
22 }
23
24 CRAWLER_DB = {
25 'host': os.environ['CRAWLER_DB_HOST'],
26 'port': int(os.environ.get('CRAWLER_DB_PORT', '5432')),
27 'user': os.environ['CRAWLER_DB_USER'],
28 'password': os.environ['CRAWLER_DB_PASSWORD'],
29 'database': os.environ['CRAWLER_DB_NAME'],
30 'ssl_context': None,
31 }
32
33 OSS_CONFIG = {
34 'access_key_id': os.environ['OSS_ACCESS_KEY_ID'],
35 'access_key_secret': os.environ['OSS_ACCESS_KEY_SECRET'],
36 'endpoint': os.environ['OSS_ENDPOINT'],
37 'bucket_name': os.environ['OSS_BUCKET_NAME'],
38 'base_url': f"https://{os.environ['OSS_BUCKET_NAME']}.{os.environ['OSS_ENDPOINT']}",
39 }
40
41 PLATFORM_QQ = '1'
42 PLATFORM_KUGOU = '2'
43 PLATFORM_NETEASE = '4'
44 PLATFORMS = [PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE]
45
46 BATCH_SIZE = 500
1 import pymysql
2 import pymysql.cursors
3 import pg8000
4 import oss2
5 from .config import SOURCE_DB, HK_SONGS_DB, CRAWLER_DB, OSS_CONFIG
6
7
8 def get_source_conn() -> pymysql.Connection:
9 return pymysql.connect(**SOURCE_DB, cursorclass=pymysql.cursors.DictCursor)
10
11
12 def get_spider_conn() -> pymysql.Connection:
13 cfg = SOURCE_DB.copy()
14 cfg['database'] = 'hikoon-data-spider'
15 return pymysql.connect(**cfg, cursorclass=pymysql.cursors.DictCursor)
16
17
18 def get_hk_songs_conn() -> pymysql.Connection:
19 return pymysql.connect(**HK_SONGS_DB, cursorclass=pymysql.cursors.DictCursor)
20
21
22 def get_pg_conn() -> pg8000.Connection:
23 return pg8000.connect(**CRAWLER_DB)
24
25
26 def get_oss_bucket() -> oss2.Bucket:
27 auth = oss2.Auth(OSS_CONFIG['access_key_id'], OSS_CONFIG['access_key_secret'])
28 return oss2.Bucket(auth, OSS_CONFIG['endpoint'], OSS_CONFIG['bucket_name'])
1 pymysql 1 aliyun-python-sdk-core==2.16.0
2 python-dotenv 2 aliyun-python-sdk-kms==2.16.5
3 oss2 3 asn1crypto==1.5.1
4 requests 4 certifi==2026.6.17
5 tqdm 5 cffi==2.0.0
6 opencc-python-reimplemented 6 charset-normalizer==3.4.7
7 pytest 7 cos_python_sdk_v5==1.9.44
8 crcmod==1.7
9 cryptography==49.0.0
10 et_xmlfile==2.0.0
11 exceptiongroup==1.3.1
12 idna==3.18
13 iniconfig==2.3.0
14 jmespath==0.10.0
15 OpenCC==1.4.0
16 opencc-python-reimplemented==0.1.7
17 openpyxl==3.1.5
18 oss2==2.19.1
19 packaging==26.2
20 pg8000==1.31.5
21 pluggy==1.6.0
22 psycopg2==2.9.12
23 pycparser==3.0
24 pycryptodome==3.23.0
25 Pygments==2.20.0
26 PyMySQL==1.2.0
27 pytest==9.1.1
28 python-dateutil==2.9.0.post0
29 python-dotenv==1.2.2
30 requests==2.34.2
31 scramp==1.4.12
32 six==1.17.0
33 tomli==2.4.1
34 tqdm==4.68.3
35 typing_extensions==4.15.0
36 urllib3==2.7.0
37 xmltodict==1.0.4
......