config.py
3.21 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
import os
from dotenv import load_dotenv
load_dotenv()
SOURCE_DB = {
'host': os.environ['SOURCE_DB_HOST'],
'port': int(os.environ['SOURCE_DB_PORT']),
'user': os.environ['SOURCE_DB_USER'],
'password': os.environ['SOURCE_DB_PASSWORD'],
'database': os.environ['SOURCE_DB_NAME'],
'charset': 'utf8mb4',
}
HK_SONGS_DB = {
'host': os.environ['TARGET_DB_HOST'],
'port': int(os.environ['TARGET_DB_PORT']),
'user': os.environ['TARGET_DB_USER'],
'password': os.environ['TARGET_DB_PASSWORD'],
'database': os.environ['TARGET_DB_NAME'],
'charset': 'utf8mb4',
}
TEST_CRAWLER_DB = {
'host': os.environ['TEST_CRAWLER_DB_HOST'],
'port': int(os.environ.get('TEST_CRAWLER_DB_PORT', '5432')),
'user': os.environ['TEST_CRAWLER_DB_USER'],
'password': os.environ['TEST_CRAWLER_DB_PASSWORD'],
'database': os.environ['TEST_CRAWLER_DB_NAME'],
'ssl_context': None if os.environ.get('TEST_CRAWLER_DB_SSL', 'false').lower() != 'true' else True,
}
TEST_ARCHIVE_DATA_DB = {
**TEST_CRAWLER_DB,
'database': os.environ.get('TEST_ARCHIVE_DATA_DB_NAME', 'data_dev'),
}
ARCHIVE_CRAWLER_DB = {
'host': os.environ['ARCHIVE_CRAWLER_DB_HOST'],
'port': int(os.environ.get('ARCHIVE_CRAWLER_DB_PORT', '5432')),
'user': os.environ['ARCHIVE_CRAWLER_DB_USER'],
'password': os.environ['ARCHIVE_CRAWLER_DB_PASSWORD'],
'database': os.environ.get('ARCHIVE_CRAWLER_DB_NAME', 'archive_crawler'),
'ssl_context': None if os.environ.get('ARCHIVE_CRAWLER_DB_SSL', 'false').lower() != 'true' else True,
}
ARCHIVE_DATA_DB = {
**ARCHIVE_CRAWLER_DB,
'database': os.environ.get('ARCHIVE_DATA_DB_NAME', 'archive_data'),
}
OSS_CONFIG = {
'access_key_id': os.environ['OSS_ACCESS_KEY_ID'],
'access_key_secret': os.environ['OSS_ACCESS_KEY_SECRET'],
'endpoint': os.environ['OSS_ENDPOINT'],
'bucket_name': os.environ['OSS_BUCKET_NAME'],
'base_url': os.environ.get(
'OSS_FILE_BASE_NAME',
f"https://{os.environ['OSS_BUCKET_NAME']}.{os.environ['OSS_ENDPOINT']}",
).rstrip('/'),
'download_base_url': os.environ.get('OSS_DOWNLOAD_BASE_URL', '').rstrip('/'),
'download_rewrite_from_base_url': os.environ.get('OSS_DOWNLOAD_REWRITE_FROM_BASE_URL', '').rstrip('/'),
}
PLATFORM_QQ = '1'
PLATFORM_KUGOU = '2'
PLATFORM_NETEASE = '4'
PLATFORMS = [PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE]
YINYAN_IMPORT_TABLE = os.environ.get('YINYAN_IMPORT_TABLE', 'yinyan_song_records').strip()
_ALLOWED_YINYAN_IMPORT_TABLES = {'yinyan_song_records', 'yinyan_song_records2'}
if YINYAN_IMPORT_TABLE not in _ALLOWED_YINYAN_IMPORT_TABLES:
raise ValueError(
f"YINYAN_IMPORT_TABLE must be one of {sorted(_ALLOWED_YINYAN_IMPORT_TABLES)}, "
f"got {YINYAN_IMPORT_TABLE!r}"
)
TARGET_TABLE_NAME = os.environ.get('TARGET_TABLE_NAME', 'hk_songs').strip()
BATCH_SIZE = 1000
BACKFILL_BATCH_SIZE = int(os.environ.get('BACKFILL_BATCH_SIZE', '5000'))
HTTP_POOL_MAXSIZE = int(os.environ.get('HTTP_POOL_MAXSIZE', '128'))
HTTP_TRANSFER_RETRIES = int(os.environ.get('HTTP_TRANSFER_RETRIES', '3'))
HTTP_TRANSFER_RETRY_BACKOFF_SECONDS = float(os.environ.get('HTTP_TRANSFER_RETRY_BACKOFF_SECONDS', '1'))
OSS_CONNECTION_POOL_SIZE = int(os.environ.get('OSS_CONNECTION_POOL_SIZE', str(HTTP_POOL_MAXSIZE)))