Skip to content
Toggle navigation
Toggle navigation
This project
Loading...
Sign in
沈秋雨
/
import_hk_songs
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Network
Create a new issue
Builds
Commits
Issue Boards
Files
Commits
Network
Compare
Branches
Tags
Commit
a87080e9
...
a87080e935811317f309dd9d0217a14fea42668b
authored
2026-07-13 16:22:15 +0800
by
沈秋雨
Browse Files
Options
Browse Files
Tag
Download
Email Patches
Plain Diff
fix(config): 调整批处理大小及关联关系查询条件
- 修改关联关系查询,允许 pub_time 为空,放宽录音元数据要求
1 parent
ef648808
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
3 additions
and
4 deletions
etl_to_crawler/config.py
etl_to_crawler/reader.py
tests/test_reader.py
etl_to_crawler/config.py
View file @
a87080e
...
...
@@ -58,7 +58,7 @@ if YINYAN_IMPORT_TABLE not in _ALLOWED_YINYAN_IMPORT_TABLES:
TARGET_TABLE_NAME
=
os
.
environ
.
get
(
'TARGET_TABLE_NAME'
,
'hk_songs'
)
.
strip
()
BATCH_SIZE
=
10
BATCH_SIZE
=
10
00
BACKFILL_BATCH_SIZE
=
int
(
os
.
environ
.
get
(
'BACKFILL_BATCH_SIZE'
,
'5000'
))
HTTP_POOL_MAXSIZE
=
int
(
os
.
environ
.
get
(
'HTTP_POOL_MAXSIZE'
,
'128'
))
HTTP_TRANSFER_RETRIES
=
int
(
os
.
environ
.
get
(
'HTTP_TRANSFER_RETRIES'
,
'3'
))
...
...
etl_to_crawler/reader.py
View file @
a87080e
...
...
@@ -77,7 +77,7 @@ WHERE mr.id IN ({placeholders})
# 同 _PLATFORM_QUERY,但不做 per-platform 去重,保留同平台全部录音供 singer fallback 遍历
_ALL_PLATFORM_RECORDS_QUERY
=
_PLATFORM_QUERY
# records2 只接收具备
完整录音元数据的关联关系
。
# records2 只接收具备
必要录音元数据的关联关系,pub_time 允许为空
。
_RECORDS2_RELATIONS_QUERY
=
"""
SELECT
sar.song_id AS source_song_id,
...
...
@@ -104,7 +104,6 @@ WHERE sar.song_id IN ({placeholders})
(mr.storage_url IS NOT NULL AND TRIM(mr.storage_url) != '')
OR (mr.platform_play_url IS NOT NULL AND TRIM(mr.platform_play_url) != '')
)
AND mr.pub_time IS NOT NULL
ORDER BY sar.song_id, mr.id
"""
...
...
tests/test_reader.py
View file @
a87080e
...
...
@@ -132,7 +132,7 @@ def test_fetch_all_song_record_relations_keeps_all_records_for_a_song():
assert
'mr.platform_index_url IS NOT NULL'
in
sql
assert
'mr.storage_url IS NOT NULL'
in
sql
assert
'mr.platform_play_url IS NOT NULL'
in
sql
assert
'mr.pub_time IS NOT NULL'
in
sql
assert
'mr.pub_time IS NOT NULL'
not
in
sql
assert
params
==
[
10
]
assert
[(
row
[
'record_id'
],
row
[
'platform'
])
for
row
in
result
]
==
[
(
100
,
'1'
),
(
101
,
'1'
),
(
200
,
'2'
),
...
...
Please
register
or
sign in
to post a comment