修复歌手为[]时误导入情况

沈秋雨
Commit 7fdc8f0d ... 7fdc8f0dc9797b188983f533a05dd14477b1f573 authored 2026-07-10 14:06:27 +0800 by 沈秋雨
Showing 2 changed files with 52 additions and 28 deletions
etl_to_crawler/runner.py
tests/test_runner.py
--- a/etl_to_crawler/runner.py
View file @7fdc8f0
+++ b/etl_to_crawler/runner.py
View file @7fdc8f0
@@ -153,15 +153,16 @@ def _select_qq_import_record(
    current_record: dict,
    candidate_records: list[dict],
    songs_by_mid: dict[str, dict],
-) -> dict | None:
+) -> tuple[dict | None, str]:
-    """为导入选择 QQ 录音；当前录音的歌词或封面不可用时，改选同源的合格候选。"""
+    """为导入选择 QQ 录音；当前录音的歌词或封面不可用时，改选同源的合格候选。
+    返回 (record, reason)，record 为 None 时 reason 说明失败原因。"""
    current_song = songs_by_mid.get(current_record['platform_unique_key'])
    if not current_song:
-        return None
+        return None, 'spider data missing'
    needs_lyric = not _has_usable_qq_lyric(current_song)
    needs_cover = not _is_usable_qq_cover(_qq_cover_source(hk_row, current_song)) or not current_song.get('album_id')
    if not needs_lyric and not needs_cover:
-        return current_record
+        return current_record, 'ok'
    for candidate in candidate_records:
        song = songs_by_mid.get(candidate['platform_unique_key'])
@@ -171,8 +172,13 @@ def _select_qq_import_record(
            continue
        if needs_cover and (not song.get('album_id') or not _is_usable_qq_cover(song.get('cover'))):
            continue
-        return candidate
+        return candidate, 'ok'
-    return None
+    missing = []
+    if needs_lyric:
+        missing.append('lyric')
+    if needs_cover:
+        missing.append('cover')
+    return None, f"no candidate with usable {' and '.join(missing)}"
 def _select_kugou_import_record(
@@ -180,16 +186,17 @@ def _select_kugou_import_record(
    current_record: dict,
    candidate_records: list[dict],
    songs_by_id: dict[int, dict],
-) -> dict | None:
+) -> tuple[dict | None, str]:
-    """为导入选择酷狗录音；当前录音的歌词或封面不可用时，改选同源的合格候选。"""
+    """为导入选择酷狗录音；当前录音的歌词或封面不可用时，改选同源的合格候选。
+    返回 (record, reason)，record 为 None 时 reason 说明失败原因。"""
    song_id = int(current_record['platform_unique_key'])
    current_song = songs_by_id.get(song_id)
    if not current_song:
-        return None
+        return None, 'spider data missing'
    needs_lyric = not _has_kugou_lyric(current_song)
    needs_cover = not _has_kugou_cover(hk_row, current_song)
    if not needs_lyric and not needs_cover:
-        return current_record
+        return current_record, 'ok'
    for candidate in candidate_records:
        cid = int(candidate['platform_unique_key'])
@@ -200,8 +207,13 @@ def _select_kugou_import_record(
            continue
        if needs_cover and not _has_kugou_cover(hk_row, song):
            continue
-        return candidate
+        return candidate, 'ok'
-    return None
+    missing = []
+    if needs_lyric:
+        missing.append('lyric')
+    if needs_cover:
+        missing.append('cover')
+    return None, f"no candidate with usable {' and '.join(missing)}"
 def _select_netease_import_record(
@@ -209,16 +221,17 @@ def _select_netease_import_record(
    current_record: dict,
    candidate_records: list[dict],
    songs_by_id: dict[int, dict],
-) -> dict | None:
+) -> tuple[dict | None, str]:
-    """为导入选择网易录音；当前录音的歌词或封面不可用时，改选同源的合格候选。"""
+    """为导入选择网易录音；当前录音的歌词或封面不可用时，改选同源的合格候选。
+    返回 (record, reason)，record 为 None 时 reason 说明失败原因。"""
    song_id = int(current_record['platform_unique_key'])
    current_song = songs_by_id.get(song_id)
    if not current_song:
-        return None
+        return None, 'spider data missing'
    needs_lyric = not _has_netease_lyric(current_song)
    needs_cover = not _has_netease_cover(hk_row, current_song)
    if not needs_lyric and not needs_cover:
-        return current_record
+        return current_record, 'ok'
    for candidate in candidate_records:
        cid = int(candidate['platform_unique_key'])
@@ -229,8 +242,13 @@ def _select_netease_import_record(
            continue
        if needs_cover and not _has_netease_cover(hk_row, song):
            continue
-        return candidate
+        return candidate, 'ok'
-    return None
+    missing = []
+    if needs_lyric:
+        missing.append('lyric')
+    if needs_cover:
+        missing.append('cover')
+    return None, f"no candidate with usable {' and '.join(missing)}"
 def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, singer_list: list[dict]) -> dict:
@@ -626,6 +644,8 @@ def _prepare_import_payload(
        return None
    singer_key = int(song_data['id']) if platform == PLATFORM_QQ else int(platform_unique_id)
    singer_list = singers_maps.get(platform, {}).get(singer_key, [])
+    if not singer_list:
+        return None
    payload = preparer(hk_row, pr, bucket, base_url, song_data, singer_list)
    payload['yinyan_record'] = {
        'song_id': int(pending['song_id']),
@@ -1708,23 +1728,23 @@ def run(
                    )
                    continue
                if platform == PLATFORM_QQ:
-                    selected = _select_qq_import_record(
+                    selected, reason = _select_qq_import_record(
                        hk_row, pr, qq_records_by_song.get(src_id, []), qq_songs_map,
                    )
                elif platform == PLATFORM_KUGOU:
-                    selected = _select_kugou_import_record(
+                    selected, reason = _select_kugou_import_record(
                        hk_row, pr, kugou_records_by_song.get(src_id, []), kugou_songs_map,
                    )
                elif platform == PLATFORM_NETEASE:
-                    selected = _select_netease_import_record(
+                    selected, reason = _select_netease_import_record(
                        hk_row, pr, netease_records_by_song.get(src_id, []), netease_songs_map,
                    )
                else:
-                    selected = pr
+                    selected, reason = pr, 'ok'
                if selected is None:
                    log.warning(
-                        'Skip import without usable lyric/cover candidate: platform=%s source_song_id=%s record_id=%s',
+                        'Skip import (%s): platform=%s source_song_id=%s record_id=%s',
-                        platform, src_id, pending['record_id'],
+                        reason, platform, src_id, pending['record_id'],
                    )
                    rejected_pending.append(pending)
                    continue
--- a/tests/test_runner.py
View file @7fdc8f0
+++ b/tests/test_runner.py
View file @7fdc8f0
@@ -102,7 +102,7 @@ def test_qq_cover_source_keeps_valid_hk_cover():
 def test_select_qq_import_record_replaces_missing_album_cover_with_same_source_candidate():
    current = {'record_id': 10, 'platform_unique_key': 'bad-mid'}
    candidate = {'record_id': 20, 'platform_unique_key': 'good-mid'}
-    selected = runner._select_qq_import_record(
+    selected, reason = runner._select_qq_import_record(
        {'cover_url': runner.QQ_MISSING_ALBUM_COVER},
        current,
        [current, candidate],
@@ -112,12 +112,13 @@ def test_select_qq_import_record_replaces_missing_album_cover_with_same_source_c
        },
    )
    assert selected == candidate
+    assert reason == 'ok'
 def test_select_qq_import_record_replaces_missing_lyric_with_same_source_candidate():
    current = {'record_id': 10, 'platform_unique_key': 'without-lyric'}
    candidate = {'record_id': 20, 'platform_unique_key': 'with-lyric'}
-    selected = runner._select_qq_import_record(
+    selected, reason = runner._select_qq_import_record(
        {'cover_url': 'https://example.com/hk-cover.jpg'},
        current,
        [current, candidate],
@@ -127,16 +128,19 @@ def test_select_qq_import_record_replaces_missing_lyric_with_same_source_candida
        },
    )
    assert selected == candidate
+    assert reason == 'ok'
 def test_select_qq_import_record_returns_none_when_no_candidate_satisfies_missing_fields():
    current = {'record_id': 10, 'platform_unique_key': 'bad-mid'}
-    assert runner._select_qq_import_record(
+    result, reason = runner._select_qq_import_record(
        {'cover_url': runner.QQ_MISSING_ALBUM_COVER},
        current,
        [current],
        {'bad-mid': {'album_id': 0, 'cover': runner.QQ_MISSING_ALBUM_COVER, 'lyric': ''}},
-    ) is None
+    )
+    assert result is None
+    assert 'lyric' in reason and 'cover' in reason
 def test_run_imports_only_pending_yinyan_platform_record(monkeypatch):