Skip to content
Toggle navigation
Toggle navigation
This project
Loading...
Sign in
沈秋雨
/
import_hk_songs
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Network
Create a new issue
Builds
Commits
Issue Boards
Files
Commits
Network
Compare
Branches
Tags
Commit
7fdc8f0d
...
7fdc8f0dc9797b188983f533a05dd14477b1f573
authored
2026-07-10 14:06:27 +0800
by
沈秋雨
Browse Files
Options
Browse Files
Tag
Download
Email Patches
Plain Diff
修复歌手为[]时误导入情况
1 parent
1bc953d1
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
52 additions
and
28 deletions
etl_to_crawler/runner.py
tests/test_runner.py
etl_to_crawler/runner.py
View file @
7fdc8f0
...
...
@@ -153,15 +153,16 @@ def _select_qq_import_record(
current_record
:
dict
,
candidate_records
:
list
[
dict
],
songs_by_mid
:
dict
[
str
,
dict
],
)
->
dict
|
None
:
"""为导入选择 QQ 录音;当前录音的歌词或封面不可用时,改选同源的合格候选。"""
)
->
tuple
[
dict
|
None
,
str
]:
"""为导入选择 QQ 录音;当前录音的歌词或封面不可用时,改选同源的合格候选。
返回 (record, reason),record 为 None 时 reason 说明失败原因。"""
current_song
=
songs_by_mid
.
get
(
current_record
[
'platform_unique_key'
])
if
not
current_song
:
return
None
return
None
,
'spider data missing'
needs_lyric
=
not
_has_usable_qq_lyric
(
current_song
)
needs_cover
=
not
_is_usable_qq_cover
(
_qq_cover_source
(
hk_row
,
current_song
))
or
not
current_song
.
get
(
'album_id'
)
if
not
needs_lyric
and
not
needs_cover
:
return
current_record
return
current_record
,
'ok'
for
candidate
in
candidate_records
:
song
=
songs_by_mid
.
get
(
candidate
[
'platform_unique_key'
])
...
...
@@ -171,8 +172,13 @@ def _select_qq_import_record(
continue
if
needs_cover
and
(
not
song
.
get
(
'album_id'
)
or
not
_is_usable_qq_cover
(
song
.
get
(
'cover'
))):
continue
return
candidate
return
None
return
candidate
,
'ok'
missing
=
[]
if
needs_lyric
:
missing
.
append
(
'lyric'
)
if
needs_cover
:
missing
.
append
(
'cover'
)
return
None
,
f
"no candidate with usable {' and '.join(missing)}"
def
_select_kugou_import_record
(
...
...
@@ -180,16 +186,17 @@ def _select_kugou_import_record(
current_record
:
dict
,
candidate_records
:
list
[
dict
],
songs_by_id
:
dict
[
int
,
dict
],
)
->
dict
|
None
:
"""为导入选择酷狗录音;当前录音的歌词或封面不可用时,改选同源的合格候选。"""
)
->
tuple
[
dict
|
None
,
str
]:
"""为导入选择酷狗录音;当前录音的歌词或封面不可用时,改选同源的合格候选。
返回 (record, reason),record 为 None 时 reason 说明失败原因。"""
song_id
=
int
(
current_record
[
'platform_unique_key'
])
current_song
=
songs_by_id
.
get
(
song_id
)
if
not
current_song
:
return
None
return
None
,
'spider data missing'
needs_lyric
=
not
_has_kugou_lyric
(
current_song
)
needs_cover
=
not
_has_kugou_cover
(
hk_row
,
current_song
)
if
not
needs_lyric
and
not
needs_cover
:
return
current_record
return
current_record
,
'ok'
for
candidate
in
candidate_records
:
cid
=
int
(
candidate
[
'platform_unique_key'
])
...
...
@@ -200,8 +207,13 @@ def _select_kugou_import_record(
continue
if
needs_cover
and
not
_has_kugou_cover
(
hk_row
,
song
):
continue
return
candidate
return
None
return
candidate
,
'ok'
missing
=
[]
if
needs_lyric
:
missing
.
append
(
'lyric'
)
if
needs_cover
:
missing
.
append
(
'cover'
)
return
None
,
f
"no candidate with usable {' and '.join(missing)}"
def
_select_netease_import_record
(
...
...
@@ -209,16 +221,17 @@ def _select_netease_import_record(
current_record
:
dict
,
candidate_records
:
list
[
dict
],
songs_by_id
:
dict
[
int
,
dict
],
)
->
dict
|
None
:
"""为导入选择网易录音;当前录音的歌词或封面不可用时,改选同源的合格候选。"""
)
->
tuple
[
dict
|
None
,
str
]:
"""为导入选择网易录音;当前录音的歌词或封面不可用时,改选同源的合格候选。
返回 (record, reason),record 为 None 时 reason 说明失败原因。"""
song_id
=
int
(
current_record
[
'platform_unique_key'
])
current_song
=
songs_by_id
.
get
(
song_id
)
if
not
current_song
:
return
None
return
None
,
'spider data missing'
needs_lyric
=
not
_has_netease_lyric
(
current_song
)
needs_cover
=
not
_has_netease_cover
(
hk_row
,
current_song
)
if
not
needs_lyric
and
not
needs_cover
:
return
current_record
return
current_record
,
'ok'
for
candidate
in
candidate_records
:
cid
=
int
(
candidate
[
'platform_unique_key'
])
...
...
@@ -229,8 +242,13 @@ def _select_netease_import_record(
continue
if
needs_cover
and
not
_has_netease_cover
(
hk_row
,
song
):
continue
return
candidate
return
None
return
candidate
,
'ok'
missing
=
[]
if
needs_lyric
:
missing
.
append
(
'lyric'
)
if
needs_cover
:
missing
.
append
(
'cover'
)
return
None
,
f
"no candidate with usable {' and '.join(missing)}"
def
_prepare_qq_payload
(
hk_row
:
dict
,
pr
:
dict
,
bucket
,
base_url
,
sp
:
dict
,
singer_list
:
list
[
dict
])
->
dict
:
...
...
@@ -626,6 +644,8 @@ def _prepare_import_payload(
return
None
singer_key
=
int
(
song_data
[
'id'
])
if
platform
==
PLATFORM_QQ
else
int
(
platform_unique_id
)
singer_list
=
singers_maps
.
get
(
platform
,
{})
.
get
(
singer_key
,
[])
if
not
singer_list
:
return
None
payload
=
preparer
(
hk_row
,
pr
,
bucket
,
base_url
,
song_data
,
singer_list
)
payload
[
'yinyan_record'
]
=
{
'song_id'
:
int
(
pending
[
'song_id'
]),
...
...
@@ -1708,23 +1728,23 @@ def run(
)
continue
if
platform
==
PLATFORM_QQ
:
selected
=
_select_qq_import_record
(
selected
,
reason
=
_select_qq_import_record
(
hk_row
,
pr
,
qq_records_by_song
.
get
(
src_id
,
[]),
qq_songs_map
,
)
elif
platform
==
PLATFORM_KUGOU
:
selected
=
_select_kugou_import_record
(
selected
,
reason
=
_select_kugou_import_record
(
hk_row
,
pr
,
kugou_records_by_song
.
get
(
src_id
,
[]),
kugou_songs_map
,
)
elif
platform
==
PLATFORM_NETEASE
:
selected
=
_select_netease_import_record
(
selected
,
reason
=
_select_netease_import_record
(
hk_row
,
pr
,
netease_records_by_song
.
get
(
src_id
,
[]),
netease_songs_map
,
)
else
:
selected
=
pr
selected
,
reason
=
pr
,
'ok'
if
selected
is
None
:
log
.
warning
(
'Skip import
without usable lyric/cover candidate
: platform=
%
s source_song_id=
%
s record_id=
%
s'
,
platform
,
src_id
,
pending
[
'record_id'
],
'Skip import
(
%
s)
: platform=
%
s source_song_id=
%
s record_id=
%
s'
,
reason
,
platform
,
src_id
,
pending
[
'record_id'
],
)
rejected_pending
.
append
(
pending
)
continue
...
...
tests/test_runner.py
View file @
7fdc8f0
...
...
@@ -102,7 +102,7 @@ def test_qq_cover_source_keeps_valid_hk_cover():
def
test_select_qq_import_record_replaces_missing_album_cover_with_same_source_candidate
():
current
=
{
'record_id'
:
10
,
'platform_unique_key'
:
'bad-mid'
}
candidate
=
{
'record_id'
:
20
,
'platform_unique_key'
:
'good-mid'
}
selected
=
runner
.
_select_qq_import_record
(
selected
,
reason
=
runner
.
_select_qq_import_record
(
{
'cover_url'
:
runner
.
QQ_MISSING_ALBUM_COVER
},
current
,
[
current
,
candidate
],
...
...
@@ -112,12 +112,13 @@ def test_select_qq_import_record_replaces_missing_album_cover_with_same_source_c
},
)
assert
selected
==
candidate
assert
reason
==
'ok'
def
test_select_qq_import_record_replaces_missing_lyric_with_same_source_candidate
():
current
=
{
'record_id'
:
10
,
'platform_unique_key'
:
'without-lyric'
}
candidate
=
{
'record_id'
:
20
,
'platform_unique_key'
:
'with-lyric'
}
selected
=
runner
.
_select_qq_import_record
(
selected
,
reason
=
runner
.
_select_qq_import_record
(
{
'cover_url'
:
'https://example.com/hk-cover.jpg'
},
current
,
[
current
,
candidate
],
...
...
@@ -127,16 +128,19 @@ def test_select_qq_import_record_replaces_missing_lyric_with_same_source_candida
},
)
assert
selected
==
candidate
assert
reason
==
'ok'
def
test_select_qq_import_record_returns_none_when_no_candidate_satisfies_missing_fields
():
current
=
{
'record_id'
:
10
,
'platform_unique_key'
:
'bad-mid'
}
assert
runner
.
_select_qq_import_record
(
result
,
reason
=
runner
.
_select_qq_import_record
(
{
'cover_url'
:
runner
.
QQ_MISSING_ALBUM_COVER
},
current
,
[
current
],
{
'bad-mid'
:
{
'album_id'
:
0
,
'cover'
:
runner
.
QQ_MISSING_ALBUM_COVER
,
'lyric'
:
''
}},
)
is
None
)
assert
result
is
None
assert
'lyric'
in
reason
and
'cover'
in
reason
def
test_run_imports_only_pending_yinyan_platform_record
(
monkeypatch
):
...
...
Please
register
or
sign in
to post a comment