feat(etl): 初始化 etl_to_crawler 模块,验证三端连接
- 创建 etl_to_crawler/__init__.py, config.py, connections.py - 使用 pg8000 替代 psycopg2-binary(macOS 二进制兼容问题) - 验证 hikoon-data / hikoon-data-spider / new_music_library / crawler_dev / OSS 全部 OK - 更新 requirements.txt Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Showing
4 changed files
with
111 additions
and
7 deletions
etl_to_crawler/__init__.py
0 → 100644
File mode changed
etl_to_crawler/config.py
0 → 100644
| 1 | import os | ||
| 2 | from dotenv import load_dotenv | ||
| 3 | |||
| 4 | load_dotenv() | ||
| 5 | |||
| 6 | SOURCE_DB = { | ||
| 7 | 'host': os.environ['SOURCE_DB_HOST'], | ||
| 8 | 'port': int(os.environ['SOURCE_DB_PORT']), | ||
| 9 | 'user': os.environ['SOURCE_DB_USER'], | ||
| 10 | 'password': os.environ['SOURCE_DB_PASSWORD'], | ||
| 11 | 'database': os.environ['SOURCE_DB_NAME'], | ||
| 12 | 'charset': 'utf8mb4', | ||
| 13 | } | ||
| 14 | |||
| 15 | HK_SONGS_DB = { | ||
| 16 | 'host': os.environ['TARGET_DB_HOST'], | ||
| 17 | 'port': int(os.environ['TARGET_DB_PORT']), | ||
| 18 | 'user': os.environ['TARGET_DB_USER'], | ||
| 19 | 'password': os.environ['TARGET_DB_PASSWORD'], | ||
| 20 | 'database': os.environ['TARGET_DB_NAME'], | ||
| 21 | 'charset': 'utf8mb4', | ||
| 22 | } | ||
| 23 | |||
| 24 | CRAWLER_DB = { | ||
| 25 | 'host': os.environ['CRAWLER_DB_HOST'], | ||
| 26 | 'port': int(os.environ.get('CRAWLER_DB_PORT', '5432')), | ||
| 27 | 'user': os.environ['CRAWLER_DB_USER'], | ||
| 28 | 'password': os.environ['CRAWLER_DB_PASSWORD'], | ||
| 29 | 'database': os.environ['CRAWLER_DB_NAME'], | ||
| 30 | 'ssl_context': None, | ||
| 31 | } | ||
| 32 | |||
| 33 | OSS_CONFIG = { | ||
| 34 | 'access_key_id': os.environ['OSS_ACCESS_KEY_ID'], | ||
| 35 | 'access_key_secret': os.environ['OSS_ACCESS_KEY_SECRET'], | ||
| 36 | 'endpoint': os.environ['OSS_ENDPOINT'], | ||
| 37 | 'bucket_name': os.environ['OSS_BUCKET_NAME'], | ||
| 38 | 'base_url': f"https://{os.environ['OSS_BUCKET_NAME']}.{os.environ['OSS_ENDPOINT']}", | ||
| 39 | } | ||
| 40 | |||
| 41 | PLATFORM_QQ = '1' | ||
| 42 | PLATFORM_KUGOU = '2' | ||
| 43 | PLATFORM_NETEASE = '4' | ||
| 44 | PLATFORMS = [PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE] | ||
| 45 | |||
| 46 | BATCH_SIZE = 500 |
etl_to_crawler/connections.py
0 → 100644
| 1 | import pymysql | ||
| 2 | import pymysql.cursors | ||
| 3 | import pg8000 | ||
| 4 | import oss2 | ||
| 5 | from .config import SOURCE_DB, HK_SONGS_DB, CRAWLER_DB, OSS_CONFIG | ||
| 6 | |||
| 7 | |||
| 8 | def get_source_conn() -> pymysql.Connection: | ||
| 9 | return pymysql.connect(**SOURCE_DB, cursorclass=pymysql.cursors.DictCursor) | ||
| 10 | |||
| 11 | |||
| 12 | def get_spider_conn() -> pymysql.Connection: | ||
| 13 | cfg = SOURCE_DB.copy() | ||
| 14 | cfg['database'] = 'hikoon-data-spider' | ||
| 15 | return pymysql.connect(**cfg, cursorclass=pymysql.cursors.DictCursor) | ||
| 16 | |||
| 17 | |||
| 18 | def get_hk_songs_conn() -> pymysql.Connection: | ||
| 19 | return pymysql.connect(**HK_SONGS_DB, cursorclass=pymysql.cursors.DictCursor) | ||
| 20 | |||
| 21 | |||
| 22 | def get_pg_conn() -> pg8000.Connection: | ||
| 23 | return pg8000.connect(**CRAWLER_DB) | ||
| 24 | |||
| 25 | |||
| 26 | def get_oss_bucket() -> oss2.Bucket: | ||
| 27 | auth = oss2.Auth(OSS_CONFIG['access_key_id'], OSS_CONFIG['access_key_secret']) | ||
| 28 | return oss2.Bucket(auth, OSS_CONFIG['endpoint'], OSS_CONFIG['bucket_name']) |
| 1 | pymysql | 1 | aliyun-python-sdk-core==2.16.0 |
| 2 | python-dotenv | 2 | aliyun-python-sdk-kms==2.16.5 |
| 3 | oss2 | 3 | asn1crypto==1.5.1 |
| 4 | requests | 4 | certifi==2026.6.17 |
| 5 | tqdm | 5 | cffi==2.0.0 |
| 6 | opencc-python-reimplemented | 6 | charset-normalizer==3.4.7 |
| 7 | pytest | 7 | cos_python_sdk_v5==1.9.44 |
| 8 | crcmod==1.7 | ||
| 9 | cryptography==49.0.0 | ||
| 10 | et_xmlfile==2.0.0 | ||
| 11 | exceptiongroup==1.3.1 | ||
| 12 | idna==3.18 | ||
| 13 | iniconfig==2.3.0 | ||
| 14 | jmespath==0.10.0 | ||
| 15 | OpenCC==1.4.0 | ||
| 16 | opencc-python-reimplemented==0.1.7 | ||
| 17 | openpyxl==3.1.5 | ||
| 18 | oss2==2.19.1 | ||
| 19 | packaging==26.2 | ||
| 20 | pg8000==1.31.5 | ||
| 21 | pluggy==1.6.0 | ||
| 22 | psycopg2==2.9.12 | ||
| 23 | pycparser==3.0 | ||
| 24 | pycryptodome==3.23.0 | ||
| 25 | Pygments==2.20.0 | ||
| 26 | PyMySQL==1.2.0 | ||
| 27 | pytest==9.1.1 | ||
| 28 | python-dateutil==2.9.0.post0 | ||
| 29 | python-dotenv==1.2.2 | ||
| 30 | requests==2.34.2 | ||
| 31 | scramp==1.4.12 | ||
| 32 | six==1.17.0 | ||
| 33 | tomli==2.4.1 | ||
| 34 | tqdm==4.68.3 | ||
| 35 | typing_extensions==4.15.0 | ||
| 36 | urllib3==2.7.0 | ||
| 37 | xmltodict==1.0.4 | ... | ... |
-
Please register or sign in to post a comment