Skip to content

沈秋雨 / import_hk_songs

Go to a project
Toggle navigation
Toggle navigation pinning
Commit a87080e9 a87080e935811317f309dd9d0217a14fea42668b by 沈秋雨
Options

fix(config): 调整批处理大小及关联关系查询条件 

- 修改关联关系查询，允许 pub_time 为空，放宽录音元数据要求
1 parent ef648808
Inline Side-by-side
Showing 3 changed files with 3 additions and 4 deletions
...@@ -58,7 +58,7 @@ if YINYAN_IMPORT_TABLE not in _ALLOWED_YINYAN_IMPORT_TABLES: ...@@ -58,7 +58,7 @@ if YINYAN_IMPORT_TABLE not in _ALLOWED_YINYAN_IMPORT_TABLES:
58 58
59 TARGET_TABLE_NAME = os.environ.get('TARGET_TABLE_NAME', 'hk_songs').strip() 59 TARGET_TABLE_NAME = os.environ.get('TARGET_TABLE_NAME', 'hk_songs').strip()
60 60
61 BATCH_SIZE = 10 61 BATCH_SIZE = 1000
62 BACKFILL_BATCH_SIZE = int(os.environ.get('BACKFILL_BATCH_SIZE', '5000')) 62 BACKFILL_BATCH_SIZE = int(os.environ.get('BACKFILL_BATCH_SIZE', '5000'))
63 HTTP_POOL_MAXSIZE = int(os.environ.get('HTTP_POOL_MAXSIZE', '128')) 63 HTTP_POOL_MAXSIZE = int(os.environ.get('HTTP_POOL_MAXSIZE', '128'))
64 HTTP_TRANSFER_RETRIES = int(os.environ.get('HTTP_TRANSFER_RETRIES', '3')) 64 HTTP_TRANSFER_RETRIES = int(os.environ.get('HTTP_TRANSFER_RETRIES', '3'))
......
...@@ -77,7 +77,7 @@ WHERE mr.id IN ({placeholders}) ...@@ -77,7 +77,7 @@ WHERE mr.id IN ({placeholders})
77 # 同 _PLATFORM_QUERY，但不做 per-platform 去重，保留同平台全部录音供 singer fallback 遍历 77 # 同 _PLATFORM_QUERY，但不做 per-platform 去重，保留同平台全部录音供 singer fallback 遍历
78 _ALL_PLATFORM_RECORDS_QUERY = _PLATFORM_QUERY 78 _ALL_PLATFORM_RECORDS_QUERY = _PLATFORM_QUERY
79 79
80 # records2 只接收具备完整录音元数据的关联关系。 80 # records2 只接收具备必要录音元数据的关联关系，pub_time 允许为空。
81 _RECORDS2_RELATIONS_QUERY = """ 81 _RECORDS2_RELATIONS_QUERY = """
82 SELECT 82 SELECT
83 sar.song_id AS source_song_id, 83 sar.song_id AS source_song_id,
...@@ -104,7 +104,6 @@ WHERE sar.song_id IN ({placeholders}) ...@@ -104,7 +104,6 @@ WHERE sar.song_id IN ({placeholders})
104 (mr.storage_url IS NOT NULL AND TRIM(mr.storage_url) != '') 104 (mr.storage_url IS NOT NULL AND TRIM(mr.storage_url) != '')
105 OR (mr.platform_play_url IS NOT NULL AND TRIM(mr.platform_play_url) != '') 105 OR (mr.platform_play_url IS NOT NULL AND TRIM(mr.platform_play_url) != '')
106 ) 106 )
107 AND mr.pub_time IS NOT NULL
108 ORDER BY sar.song_id, mr.id 107 ORDER BY sar.song_id, mr.id
109 """ 108 """
110 109
......
...@@ -132,7 +132,7 @@ def test_fetch_all_song_record_relations_keeps_all_records_for_a_song(): ...@@ -132,7 +132,7 @@ def test_fetch_all_song_record_relations_keeps_all_records_for_a_song():
132 assert 'mr.platform_index_url IS NOT NULL' in sql 132 assert 'mr.platform_index_url IS NOT NULL' in sql
133 assert 'mr.storage_url IS NOT NULL' in sql 133 assert 'mr.storage_url IS NOT NULL' in sql
134 assert 'mr.platform_play_url IS NOT NULL' in sql 134 assert 'mr.platform_play_url IS NOT NULL' in sql
135 assert 'mr.pub_time IS NOT NULL' in sql 135 assert 'mr.pub_time IS NOT NULL' not in sql
136 assert params == [10] 136 assert params == [10]
137 assert [(row['record_id'], row['platform']) for row in result] == [ 137 assert [(row['record_id'], row['platform']) for row in result] == [
138 (100, '1'), (101, '1'), (200, '2'), 138 (100, '1'), (101, '1'), (200, '2'),
......