Commit d4e1bfc7 d4e1bfc75f6a5ae9a7d21bb628ccb16aeb9f470b by 沈秋雨

feat(etl): 初始化 etl_to_crawler 模块,验证三端连接

- 创建 etl_to_crawler/__init__.py, config.py, connections.py
- 使用 pg8000 替代 psycopg2-binary(macOS 二进制兼容问题)
- 验证 hikoon-data / hikoon-data-spider / new_music_library / crawler_dev / OSS 全部 OK
- 更新 requirements.txt

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 666fd63f
import os
from dotenv import load_dotenv
load_dotenv()
SOURCE_DB = {
'host': os.environ['SOURCE_DB_HOST'],
'port': int(os.environ['SOURCE_DB_PORT']),
'user': os.environ['SOURCE_DB_USER'],
'password': os.environ['SOURCE_DB_PASSWORD'],
'database': os.environ['SOURCE_DB_NAME'],
'charset': 'utf8mb4',
}
HK_SONGS_DB = {
'host': os.environ['TARGET_DB_HOST'],
'port': int(os.environ['TARGET_DB_PORT']),
'user': os.environ['TARGET_DB_USER'],
'password': os.environ['TARGET_DB_PASSWORD'],
'database': os.environ['TARGET_DB_NAME'],
'charset': 'utf8mb4',
}
CRAWLER_DB = {
'host': os.environ['CRAWLER_DB_HOST'],
'port': int(os.environ.get('CRAWLER_DB_PORT', '5432')),
'user': os.environ['CRAWLER_DB_USER'],
'password': os.environ['CRAWLER_DB_PASSWORD'],
'database': os.environ['CRAWLER_DB_NAME'],
'ssl_context': None,
}
OSS_CONFIG = {
'access_key_id': os.environ['OSS_ACCESS_KEY_ID'],
'access_key_secret': os.environ['OSS_ACCESS_KEY_SECRET'],
'endpoint': os.environ['OSS_ENDPOINT'],
'bucket_name': os.environ['OSS_BUCKET_NAME'],
'base_url': f"https://{os.environ['OSS_BUCKET_NAME']}.{os.environ['OSS_ENDPOINT']}",
}
PLATFORM_QQ = '1'
PLATFORM_KUGOU = '2'
PLATFORM_NETEASE = '4'
PLATFORMS = [PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE]
BATCH_SIZE = 500
import pymysql
import pymysql.cursors
import pg8000
import oss2
from .config import SOURCE_DB, HK_SONGS_DB, CRAWLER_DB, OSS_CONFIG
def get_source_conn() -> pymysql.Connection:
return pymysql.connect(**SOURCE_DB, cursorclass=pymysql.cursors.DictCursor)
def get_spider_conn() -> pymysql.Connection:
cfg = SOURCE_DB.copy()
cfg['database'] = 'hikoon-data-spider'
return pymysql.connect(**cfg, cursorclass=pymysql.cursors.DictCursor)
def get_hk_songs_conn() -> pymysql.Connection:
return pymysql.connect(**HK_SONGS_DB, cursorclass=pymysql.cursors.DictCursor)
def get_pg_conn() -> pg8000.Connection:
return pg8000.connect(**CRAWLER_DB)
def get_oss_bucket() -> oss2.Bucket:
auth = oss2.Auth(OSS_CONFIG['access_key_id'], OSS_CONFIG['access_key_secret'])
return oss2.Bucket(auth, OSS_CONFIG['endpoint'], OSS_CONFIG['bucket_name'])
pymysql
python-dotenv
oss2
requests
tqdm
opencc-python-reimplemented
pytest
aliyun-python-sdk-core==2.16.0
aliyun-python-sdk-kms==2.16.5
asn1crypto==1.5.1
certifi==2026.6.17
cffi==2.0.0
charset-normalizer==3.4.7
cos_python_sdk_v5==1.9.44
crcmod==1.7
cryptography==49.0.0
et_xmlfile==2.0.0
exceptiongroup==1.3.1
idna==3.18
iniconfig==2.3.0
jmespath==0.10.0
OpenCC==1.4.0
opencc-python-reimplemented==0.1.7
openpyxl==3.1.5
oss2==2.19.1
packaging==26.2
pg8000==1.31.5
pluggy==1.6.0
psycopg2==2.9.12
pycparser==3.0
pycryptodome==3.23.0
Pygments==2.20.0
PyMySQL==1.2.0
pytest==9.1.1
python-dateutil==2.9.0.post0
python-dotenv==1.2.2
requests==2.34.2
scramp==1.4.12
six==1.17.0
tomli==2.4.1
tqdm==4.68.3
typing_extensions==4.15.0
urllib3==2.7.0
xmltodict==1.0.4
......