Commit a87080e9 a87080e935811317f309dd9d0217a14fea42668b by 沈秋雨

fix(config): 调整批处理大小及关联关系查询条件

- 修改关联关系查询,允许 pub_time 为空,放宽录音元数据要求
1 parent ef648808
......@@ -58,7 +58,7 @@ if YINYAN_IMPORT_TABLE not in _ALLOWED_YINYAN_IMPORT_TABLES:
TARGET_TABLE_NAME = os.environ.get('TARGET_TABLE_NAME', 'hk_songs').strip()
BATCH_SIZE = 10
BATCH_SIZE = 1000
BACKFILL_BATCH_SIZE = int(os.environ.get('BACKFILL_BATCH_SIZE', '5000'))
HTTP_POOL_MAXSIZE = int(os.environ.get('HTTP_POOL_MAXSIZE', '128'))
HTTP_TRANSFER_RETRIES = int(os.environ.get('HTTP_TRANSFER_RETRIES', '3'))
......
......@@ -77,7 +77,7 @@ WHERE mr.id IN ({placeholders})
# 同 _PLATFORM_QUERY,但不做 per-platform 去重,保留同平台全部录音供 singer fallback 遍历
_ALL_PLATFORM_RECORDS_QUERY = _PLATFORM_QUERY
# records2 只接收具备完整录音元数据的关联关系
# records2 只接收具备必要录音元数据的关联关系,pub_time 允许为空
_RECORDS2_RELATIONS_QUERY = """
SELECT
sar.song_id AS source_song_id,
......@@ -104,7 +104,6 @@ WHERE sar.song_id IN ({placeholders})
(mr.storage_url IS NOT NULL AND TRIM(mr.storage_url) != '')
OR (mr.platform_play_url IS NOT NULL AND TRIM(mr.platform_play_url) != '')
)
AND mr.pub_time IS NOT NULL
ORDER BY sar.song_id, mr.id
"""
......
......@@ -132,7 +132,7 @@ def test_fetch_all_song_record_relations_keeps_all_records_for_a_song():
assert 'mr.platform_index_url IS NOT NULL' in sql
assert 'mr.storage_url IS NOT NULL' in sql
assert 'mr.platform_play_url IS NOT NULL' in sql
assert 'mr.pub_time IS NOT NULL' in sql
assert 'mr.pub_time IS NOT NULL' not in sql
assert params == [10]
assert [(row['record_id'], row['platform']) for row in result] == [
(100, '1'), (101, '1'), (200, '2'),
......