Commit 443980dc 443980dc1b929d6398b24b3e4159819a7cc502e2 by 沈秋雨

refactor(lyric): 优化歌词转存逻辑及更新冲突处理

1 parent 55c6e4fb
...@@ -100,28 +100,55 @@ def _safe_transfer_audio(url, oss_key, bucket, base_url) -> tuple[str, str]: ...@@ -100,28 +100,55 @@ def _safe_transfer_audio(url, oss_key, bucket, base_url) -> tuple[str, str]:
100 return '', '' # 不将未转存的音频外链写入 crawler 100 return '', '' # 不将未转存的音频外链写入 crawler
101 101
102 102
103 def _safe_upload_lyric( 103 def _safe_prepare_lyric(
104 platform: str, 104 platform: str,
105 unique_id: str, 105 unique_id: str,
106 spider_lyric: str | None, 106 spider_lyric: str | None,
107 hk_lyrics_url: str | None, 107 hk_lyrics_url: str | None,
108 bucket, 108 bucket,
109 base_url: str, 109 base_url: str,
110 ) -> str: 110 ) -> tuple[str, str]:
111 """优先将音眼歌词 URL 转存到 crawler OSS;spider 歌词文本仅作回退。""" 111 """返回实际写入的 (lyric_url, lyric_text),音眼歌词优先、spider 回退。"""
112 candidates: list[tuple[str, str]] = []
112 if str(hk_lyrics_url or '').strip(): 113 if str(hk_lyrics_url or '').strip():
113 try: 114 try:
114 hk_lyric = download_text_url(hk_lyrics_url, base_url) 115 hk_lyric = download_text_url(hk_lyrics_url, base_url)
115 uploaded_url = upload_plain_lyric_to_bucket(platform, unique_id, hk_lyric, bucket, base_url) 116 if str(hk_lyric or '').strip():
117 candidates.append(('HK', hk_lyric))
118 except Exception as e:
119 log.warning('HK lyric download failed for %s/%s: %s', platform, unique_id, e)
120
121 spider_text = spider_lyric or ''
122 if str(spider_text).strip() and all(text != spider_text for _, text in candidates):
123 candidates.append(('spider', spider_text))
124
125 for source, lyric_text in candidates:
126 try:
127 uploaded_url = upload_plain_lyric_to_bucket(
128 platform, unique_id, lyric_text, bucket, base_url,
129 )
116 if uploaded_url: 130 if uploaded_url:
117 return uploaded_url 131 return uploaded_url, lyric_text
118 except Exception as e: 132 except Exception as e:
119 log.warning('HK lyric transfer failed for %s/%s: %s', platform, unique_id, e) 133 log.warning('%s lyric upload failed for %s/%s: %s', source, platform, unique_id, e)
120 try: 134
121 return upload_plain_lyric_to_bucket(platform, unique_id, spider_lyric or '', bucket, base_url) 135 # records2 允许 URL 转存失败时入库;若文本已成功获取,仍尽量保留文本。
122 except Exception as e: 136 return '', candidates[0][1] if candidates else ''
123 log.warning("Lyric upload failed for %s/%s: %s", platform, unique_id, e) 137
124 return '' 138
139 def _safe_upload_lyric(
140 platform: str,
141 unique_id: str,
142 spider_lyric: str | None,
143 hk_lyrics_url: str | None,
144 bucket,
145 base_url: str,
146 ) -> str:
147 """兼容仅需要歌词 URL 的调用方。"""
148 lyric_url, _ = _safe_prepare_lyric(
149 platform, unique_id, spider_lyric, hk_lyrics_url, bucket, base_url,
150 )
151 return lyric_url
125 152
126 153
127 def _required_import_assets() -> tuple[str, ...]: 154 def _required_import_assets() -> tuple[str, ...]:
...@@ -134,7 +161,9 @@ def _required_import_assets() -> tuple[str, ...]: ...@@ -134,7 +161,9 @@ def _required_import_assets() -> tuple[str, ...]:
134 def _require_primary_assets(assets: dict, required: tuple[str, ...] | None = None) -> None: 161 def _require_primary_assets(assets: dict, required: tuple[str, ...] | None = None) -> None:
135 """校验当前导入支线要求的资源,允许非必需资源以空值入库。""" 162 """校验当前导入支线要求的资源,允许非必需资源以空值入库。"""
136 audio_url, _ = assets.get('audio', ('', '')) 163 audio_url, _ = assets.get('audio', ('', ''))
137 values = {'audio': audio_url, 'cover': assets.get('cover'), 'lyric': assets.get('lyric')} 164 lyric_asset = assets.get('lyric')
165 lyric_url = lyric_asset[0] if isinstance(lyric_asset, tuple) else lyric_asset
166 values = {'audio': audio_url, 'cover': assets.get('cover'), 'lyric': lyric_url}
138 required = required or ('audio', 'cover', 'lyric') 167 required = required or ('audio', 'cover', 'lyric')
139 missing = [name for name in required if not values.get(name)] 168 missing = [name for name in required if not values.get(name)]
140 if missing: 169 if missing:
...@@ -330,7 +359,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing ...@@ -330,7 +359,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing
330 build_oss_key('qq', 'cover', str(song_id_int) + '.jpg'), 359 build_oss_key('qq', 'cover', str(song_id_int) + '.jpg'),
331 bucket, base_url, 360 bucket, base_url,
332 ), 361 ),
333 'lyric': lambda: _safe_upload_lyric('qq', mid, raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), 362 'lyric': lambda: _safe_prepare_lyric('qq', mid, raw_lyric, hk_row.get('lyrics_url'), bucket, base_url),
334 } 363 }
335 if sp.get('album_id') and sp.get('album_cover'): 364 if sp.get('album_id') and sp.get('album_cover'):
336 tasks['album_cover'] = lambda: _safe_transfer( 365 tasks['album_cover'] = lambda: _safe_transfer(
...@@ -348,7 +377,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing ...@@ -348,7 +377,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing
348 _require_primary_assets(assets, _required_import_assets()) 377 _require_primary_assets(assets, _required_import_assets())
349 audio_url, audio_md5 = assets['audio'] 378 audio_url, audio_md5 = assets['audio']
350 cover_url = assets['cover'] 379 cover_url = assets['cover']
351 lyric_url = assets['lyric'] 380 lyric_url, lyric_text = assets['lyric']
352 album_cover = assets.get('album_cover') or cover_url 381 album_cover = assets.get('album_cover') or cover_url
353 382
354 singer_rows = [{ 383 singer_rows = [{
...@@ -422,7 +451,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing ...@@ -422,7 +451,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing
422 'version': version, 451 'version': version,
423 'name': hk_row['name'], 452 'name': hk_row['name'],
424 'duration': sp.get('duration') or hk_row.get('song_time') or 0, 453 'duration': sp.get('duration') or hk_row.get('song_time') or 0,
425 'lyric': ensure_newlines(raw_lyric), 454 'lyric': ensure_newlines(lyric_text),
426 'composer_name': sp.get('composer_name') or hk_row.get('composer'), 455 'composer_name': sp.get('composer_name') or hk_row.get('composer'),
427 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), 456 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
428 'url': audio_url, 457 'url': audio_url,
...@@ -460,7 +489,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s ...@@ -460,7 +489,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s
460 build_oss_key('kugou', 'cover', str(song_id) + '.jpg'), 489 build_oss_key('kugou', 'cover', str(song_id) + '.jpg'),
461 bucket, base_url, 490 bucket, base_url,
462 ), 491 ),
463 'lyric': lambda: _safe_upload_lyric('kugou', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), 492 'lyric': lambda: _safe_prepare_lyric('kugou', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url),
464 } 493 }
465 if sp.get('album_id') and sp.get('album_cover'): 494 if sp.get('album_id') and sp.get('album_cover'):
466 tasks['album_cover'] = lambda: _safe_transfer( 495 tasks['album_cover'] = lambda: _safe_transfer(
...@@ -478,7 +507,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s ...@@ -478,7 +507,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s
478 _require_primary_assets(assets, _required_import_assets()) 507 _require_primary_assets(assets, _required_import_assets())
479 audio_url, audio_md5 = assets['audio'] 508 audio_url, audio_md5 = assets['audio']
480 cover_url = assets['cover'] 509 cover_url = assets['cover']
481 lyric_url = assets['lyric'] 510 lyric_url, lyric_text = assets['lyric']
482 album_cover = assets.get('album_cover') or cover_url 511 album_cover = assets.get('album_cover') or cover_url
483 512
484 singer_rows = [{ 513 singer_rows = [{
...@@ -535,7 +564,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s ...@@ -535,7 +564,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s
535 'version': version, 564 'version': version,
536 'name': hk_row['name'], 565 'name': hk_row['name'],
537 'duration': sp.get('duration') or hk_row.get('song_time') or 0, 566 'duration': sp.get('duration') or hk_row.get('song_time') or 0,
538 'lyric': ensure_newlines(raw_lyric), 567 'lyric': ensure_newlines(lyric_text),
539 'composer_name': sp.get('composer_name') or hk_row.get('composer'), 568 'composer_name': sp.get('composer_name') or hk_row.get('composer'),
540 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), 569 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
541 'url': audio_url, 570 'url': audio_url,
...@@ -572,7 +601,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, ...@@ -572,7 +601,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict,
572 build_oss_key('netease', 'cover', str(song_id) + '.jpg'), 601 build_oss_key('netease', 'cover', str(song_id) + '.jpg'),
573 bucket, base_url, 602 bucket, base_url,
574 ), 603 ),
575 'lyric': lambda: _safe_upload_lyric('netease', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), 604 'lyric': lambda: _safe_prepare_lyric('netease', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url),
576 } 605 }
577 if sp.get('album_id') and sp.get('album_cover'): 606 if sp.get('album_id') and sp.get('album_cover'):
578 tasks['album_cover'] = lambda: _safe_transfer( 607 tasks['album_cover'] = lambda: _safe_transfer(
...@@ -590,7 +619,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, ...@@ -590,7 +619,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict,
590 _require_primary_assets(assets, _required_import_assets()) 619 _require_primary_assets(assets, _required_import_assets())
591 audio_url, audio_md5 = assets['audio'] 620 audio_url, audio_md5 = assets['audio']
592 cover_url = assets['cover'] 621 cover_url = assets['cover']
593 lyric_url = assets['lyric'] 622 lyric_url, lyric_text = assets['lyric']
594 album_cover = assets.get('album_cover') or cover_url 623 album_cover = assets.get('album_cover') or cover_url
595 624
596 singer_rows = [{ 625 singer_rows = [{
...@@ -659,7 +688,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, ...@@ -659,7 +688,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict,
659 'version': version, 688 'version': version,
660 'name': hk_row['name'], 689 'name': hk_row['name'],
661 'duration': sp.get('duration') or hk_row.get('song_time') or 0, 690 'duration': sp.get('duration') or hk_row.get('song_time') or 0,
662 'lyric': ensure_newlines(raw_lyric), 691 'lyric': ensure_newlines(lyric_text),
663 'composer_name': sp.get('composer_name') or hk_row.get('composer'), 692 'composer_name': sp.get('composer_name') or hk_row.get('composer'),
664 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), 693 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
665 'url': audio_url, 694 'url': audio_url,
...@@ -870,7 +899,7 @@ def _process_qq( ...@@ -870,7 +899,7 @@ def _process_qq(
870 build_oss_key('qq', 'cover', str(song_id_int) + '.jpg'), 899 build_oss_key('qq', 'cover', str(song_id_int) + '.jpg'),
871 bucket, base_url 900 bucket, base_url
872 ), 901 ),
873 'lyric': lambda: _safe_upload_lyric('qq', mid, raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), 902 'lyric': lambda: _safe_prepare_lyric('qq', mid, raw_lyric, hk_row.get('lyrics_url'), bucket, base_url),
874 } 903 }
875 if sp.get('album_id') and sp.get('album_cover'): 904 if sp.get('album_id') and sp.get('album_cover'):
876 tasks['album_cover'] = lambda: _safe_transfer( 905 tasks['album_cover'] = lambda: _safe_transfer(
...@@ -890,7 +919,7 @@ def _process_qq( ...@@ -890,7 +919,7 @@ def _process_qq(
890 _require_primary_assets(assets) 919 _require_primary_assets(assets)
891 audio_url, audio_md5 = assets['audio'] 920 audio_url, audio_md5 = assets['audio']
892 cover_url = assets['cover'] 921 cover_url = assets['cover']
893 lyric_url = assets['lyric'] 922 lyric_url, lyric_text = assets['lyric']
894 album_cover = assets.get('album_cover') or cover_url 923 album_cover = assets.get('album_cover') or cover_url
895 924
896 # 歌手头像转移 + 写入 singers 925 # 歌手头像转移 + 写入 singers
...@@ -967,7 +996,7 @@ def _process_qq( ...@@ -967,7 +996,7 @@ def _process_qq(
967 'version': version, 996 'version': version,
968 'name': hk_row['name'], 997 'name': hk_row['name'],
969 'duration': sp.get('duration') or hk_row.get('song_time') or 0, 998 'duration': sp.get('duration') or hk_row.get('song_time') or 0,
970 'lyric': ensure_newlines(raw_lyric), 999 'lyric': ensure_newlines(lyric_text),
971 'composer_name': sp.get('composer_name') or hk_row.get('composer'), 1000 'composer_name': sp.get('composer_name') or hk_row.get('composer'),
972 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), 1001 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
973 'url': audio_url, 1002 'url': audio_url,
...@@ -1026,7 +1055,7 @@ def _process_kugou( ...@@ -1026,7 +1055,7 @@ def _process_kugou(
1026 build_oss_key('kugou', 'cover', str(song_id) + '.jpg'), 1055 build_oss_key('kugou', 'cover', str(song_id) + '.jpg'),
1027 bucket, base_url 1056 bucket, base_url
1028 ), 1057 ),
1029 'lyric': lambda: _safe_upload_lyric('kugou', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), 1058 'lyric': lambda: _safe_prepare_lyric('kugou', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url),
1030 } 1059 }
1031 if sp.get('album_id') and sp.get('album_cover'): 1060 if sp.get('album_id') and sp.get('album_cover'):
1032 tasks['album_cover'] = lambda: _safe_transfer( 1061 tasks['album_cover'] = lambda: _safe_transfer(
...@@ -1046,7 +1075,7 @@ def _process_kugou( ...@@ -1046,7 +1075,7 @@ def _process_kugou(
1046 _require_primary_assets(assets) 1075 _require_primary_assets(assets)
1047 audio_url, audio_md5 = assets['audio'] 1076 audio_url, audio_md5 = assets['audio']
1048 cover_url = assets['cover'] 1077 cover_url = assets['cover']
1049 lyric_url = assets['lyric'] 1078 lyric_url, lyric_text = assets['lyric']
1050 album_cover = assets.get('album_cover') or cover_url 1079 album_cover = assets.get('album_cover') or cover_url
1051 1080
1052 singer_rows = [] 1081 singer_rows = []
...@@ -1103,7 +1132,7 @@ def _process_kugou( ...@@ -1103,7 +1132,7 @@ def _process_kugou(
1103 'version': version, 1132 'version': version,
1104 'name': hk_row['name'], 1133 'name': hk_row['name'],
1105 'duration': sp.get('duration') or hk_row.get('song_time') or 0, 1134 'duration': sp.get('duration') or hk_row.get('song_time') or 0,
1106 'lyric': ensure_newlines(raw_lyric), 1135 'lyric': ensure_newlines(lyric_text),
1107 'composer_name': sp.get('composer_name') or hk_row.get('composer'), 1136 'composer_name': sp.get('composer_name') or hk_row.get('composer'),
1108 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), 1137 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
1109 'url': audio_url, 1138 'url': audio_url,
...@@ -1161,7 +1190,7 @@ def _process_netease( ...@@ -1161,7 +1190,7 @@ def _process_netease(
1161 build_oss_key('netease', 'cover', str(song_id) + '.jpg'), 1190 build_oss_key('netease', 'cover', str(song_id) + '.jpg'),
1162 bucket, base_url 1191 bucket, base_url
1163 ), 1192 ),
1164 'lyric': lambda: _safe_upload_lyric('netease', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), 1193 'lyric': lambda: _safe_prepare_lyric('netease', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url),
1165 } 1194 }
1166 if sp.get('album_id') and sp.get('album_cover'): 1195 if sp.get('album_id') and sp.get('album_cover'):
1167 tasks['album_cover'] = lambda: _safe_transfer( 1196 tasks['album_cover'] = lambda: _safe_transfer(
...@@ -1181,7 +1210,7 @@ def _process_netease( ...@@ -1181,7 +1210,7 @@ def _process_netease(
1181 _require_primary_assets(assets) 1210 _require_primary_assets(assets)
1182 audio_url, audio_md5 = assets['audio'] 1211 audio_url, audio_md5 = assets['audio']
1183 cover_url = assets['cover'] 1212 cover_url = assets['cover']
1184 lyric_url = assets['lyric'] 1213 lyric_url, lyric_text = assets['lyric']
1185 album_cover = assets.get('album_cover') or cover_url 1214 album_cover = assets.get('album_cover') or cover_url
1186 1215
1187 singer_rows = [] 1216 singer_rows = []
...@@ -1250,7 +1279,7 @@ def _process_netease( ...@@ -1250,7 +1279,7 @@ def _process_netease(
1250 'version': version, 1279 'version': version,
1251 'name': hk_row['name'], 1280 'name': hk_row['name'],
1252 'duration': sp.get('duration') or hk_row.get('song_time') or 0, 1281 'duration': sp.get('duration') or hk_row.get('song_time') or 0,
1253 'lyric': ensure_newlines(raw_lyric), 1282 'lyric': ensure_newlines(lyric_text),
1254 'composer_name': sp.get('composer_name') or hk_row.get('composer'), 1283 'composer_name': sp.get('composer_name') or hk_row.get('composer'),
1255 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), 1284 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
1256 'url': audio_url, 1285 'url': audio_url,
......
...@@ -442,7 +442,14 @@ def upsert_qq_songs(cur, songs: list[dict]) -> None: ...@@ -442,7 +442,14 @@ def upsert_qq_songs(cur, songs: list[dict]) -> None:
442 platform_index_url, published_at, album, singers, status, created_at, updated_at, 442 platform_index_url, published_at, album, singers, status, created_at, updated_at,
443 version, audio_md5, provider_name, crawler_source_data) 443 version, audio_md5, provider_name, crawler_source_data)
444 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json) 444 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json)
445 ON CONFLICT (platform_song_id) DO NOTHING 445 ON CONFLICT (platform_song_id) DO UPDATE SET
446 cover = CASE WHEN BTRIM(COALESCE(crawler_qqmusic_songs.cover, '')) = ''
447 THEN EXCLUDED.cover ELSE crawler_qqmusic_songs.cover END,
448 lyric = CASE WHEN BTRIM(COALESCE(crawler_qqmusic_songs.lyric, '')) = ''
449 THEN EXCLUDED.lyric ELSE crawler_qqmusic_songs.lyric END,
450 lyric_url = CASE WHEN BTRIM(COALESCE(crawler_qqmusic_songs.lyric_url, '')) = ''
451 THEN EXCLUDED.lyric_url ELSE crawler_qqmusic_songs.lyric_url END,
452 updated_at = NOW()
446 """ 453 """
447 rows = [( 454 rows = [(
448 s['song_uuid'], s['platform_song_id'], s['mid'], s.get('album_id'), 455 s['song_uuid'], s['platform_song_id'], s['mid'], s.get('album_id'),
...@@ -534,7 +541,14 @@ def upsert_kugou_songs(cur, songs: list[dict]) -> None: ...@@ -534,7 +541,14 @@ def upsert_kugou_songs(cur, songs: list[dict]) -> None:
534 platform_index_url, published_at, singers, status, created_at, updated_at, 541 platform_index_url, published_at, singers, status, created_at, updated_at,
535 version, audio_md5, provider_name, crawler_source_data) 542 version, audio_md5, provider_name, crawler_source_data)
536 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json) 543 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json)
537 ON CONFLICT (platform_song_id) DO NOTHING 544 ON CONFLICT (platform_song_id) DO UPDATE SET
545 cover = CASE WHEN BTRIM(COALESCE(crawler_kugou_songs.cover, '')) = ''
546 THEN EXCLUDED.cover ELSE crawler_kugou_songs.cover END,
547 lyric = CASE WHEN BTRIM(COALESCE(crawler_kugou_songs.lyric, '')) = ''
548 THEN EXCLUDED.lyric ELSE crawler_kugou_songs.lyric END,
549 lyric_url = CASE WHEN BTRIM(COALESCE(crawler_kugou_songs.lyric_url, '')) = ''
550 THEN EXCLUDED.lyric_url ELSE crawler_kugou_songs.lyric_url END,
551 updated_at = NOW()
538 """ 552 """
539 rows = [( 553 rows = [(
540 s['song_uuid'], s['platform_song_id'], s.get('hash', ''), 554 s['song_uuid'], s['platform_song_id'], s.get('hash', ''),
...@@ -623,7 +637,14 @@ def upsert_netease_songs(cur, songs: list[dict]) -> None: ...@@ -623,7 +637,14 @@ def upsert_netease_songs(cur, songs: list[dict]) -> None:
623 platform_index_url, published_at, album, singers, status, created_at, updated_at, 637 platform_index_url, published_at, album, singers, status, created_at, updated_at,
624 version, audio_md5, provider_name, crawler_source_data) 638 version, audio_md5, provider_name, crawler_source_data)
625 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json) 639 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json)
626 ON CONFLICT (platform_song_id) DO NOTHING 640 ON CONFLICT (platform_song_id) DO UPDATE SET
641 cover = CASE WHEN BTRIM(COALESCE(crawler_netease_songs.cover, '')) = ''
642 THEN EXCLUDED.cover ELSE crawler_netease_songs.cover END,
643 lyric = CASE WHEN BTRIM(COALESCE(crawler_netease_songs.lyric, '')) = ''
644 THEN EXCLUDED.lyric ELSE crawler_netease_songs.lyric END,
645 lyric_url = CASE WHEN BTRIM(COALESCE(crawler_netease_songs.lyric_url, '')) = ''
646 THEN EXCLUDED.lyric_url ELSE crawler_netease_songs.lyric_url END,
647 updated_at = NOW()
627 """ 648 """
628 rows = [( 649 rows = [(
629 s['song_uuid'], s['platform_song_id'], s.get('album_id'), 650 s['song_uuid'], s['platform_song_id'], s.get('album_id'),
......
...@@ -105,32 +105,35 @@ def test_records2_import_allows_empty_cover_and_lyric(monkeypatch): ...@@ -105,32 +105,35 @@ def test_records2_import_allows_empty_cover_and_lyric(monkeypatch):
105 }, runner._required_import_assets()) 105 }, runner._required_import_assets())
106 106
107 107
108 def test_lyric_upload_prefers_hk_url_and_forces_oss_transfer(monkeypatch): 108 def test_lyric_prepare_prefers_hk_text_for_both_url_and_text_column(monkeypatch):
109 download = MagicMock(return_value='[00:01.00]HK lyric') 109 download = MagicMock(return_value='[00:01.00]HK lyric')
110 monkeypatch.setattr(runner, 'download_text_url', download) 110 monkeypatch.setattr(runner, 'download_text_url', download)
111 upload = MagicMock(return_value='https://archive.example/crawler/lyric/qq/mid.txt') 111 upload = MagicMock(return_value='https://archive.example/crawler/lyric/qq/mid.txt')
112 monkeypatch.setattr(runner, 'upload_plain_lyric_to_bucket', upload) 112 monkeypatch.setattr(runner, 'upload_plain_lyric_to_bucket', upload)
113 bucket = object() 113 bucket = object()
114 114
115 result = runner._safe_upload_lyric( 115 result = runner._safe_prepare_lyric(
116 'qq', 'mid', 'spider lyric', 'https://source.example/lyric.txt', bucket, 'https://archive.example', 116 'qq', 'mid', 'spider lyric', 'https://source.example/lyric.txt', bucket, 'https://archive.example',
117 ) 117 )
118 118
119 assert result == 'https://archive.example/crawler/lyric/qq/mid.txt' 119 assert result == (
120 'https://archive.example/crawler/lyric/qq/mid.txt',
121 '[00:01.00]HK lyric',
122 )
120 download.assert_called_once_with('https://source.example/lyric.txt', 'https://archive.example') 123 download.assert_called_once_with('https://source.example/lyric.txt', 'https://archive.example')
121 upload.assert_called_once_with( 124 upload.assert_called_once_with(
122 'qq', 'mid', '[00:01.00]HK lyric', bucket, 'https://archive.example', 125 'qq', 'mid', '[00:01.00]HK lyric', bucket, 'https://archive.example',
123 ) 126 )
124 127
125 128
126 def test_lyric_upload_falls_back_to_spider_text_when_hk_transfer_fails(monkeypatch): 129 def test_lyric_prepare_falls_back_to_spider_text_when_hk_download_fails(monkeypatch):
127 monkeypatch.setattr(runner, 'download_text_url', MagicMock(side_effect=TimeoutError('timeout'))) 130 monkeypatch.setattr(runner, 'download_text_url', MagicMock(side_effect=TimeoutError('timeout')))
128 upload = MagicMock(return_value='https://archive.example/crawler/lyric/qq/mid.txt') 131 upload = MagicMock(return_value='https://archive.example/crawler/lyric/qq/mid.txt')
129 monkeypatch.setattr(runner, 'upload_plain_lyric_to_bucket', upload) 132 monkeypatch.setattr(runner, 'upload_plain_lyric_to_bucket', upload)
130 133
131 assert runner._safe_upload_lyric( 134 assert runner._safe_prepare_lyric(
132 'qq', 'mid', 'spider lyric', 'https://source.example/lyric.txt', object(), 'https://archive.example', 135 'qq', 'mid', 'spider lyric', 'https://source.example/lyric.txt', object(), 'https://archive.example',
133 ) == 'https://archive.example/crawler/lyric/qq/mid.txt' 136 ) == ('https://archive.example/crawler/lyric/qq/mid.txt', 'spider lyric')
134 assert upload.call_args.args[2] == 'spider lyric' 137 assert upload.call_args.args[2] == 'spider lyric'
135 138
136 139
...@@ -519,7 +522,10 @@ def test_process_qq_builds_album_json_for_song_insert(monkeypatch): ...@@ -519,7 +522,10 @@ def test_process_qq_builds_album_json_for_song_insert(monkeypatch):
519 monkeypatch.setattr(runner, 'fetch_qq_singers', lambda conn, song_ids: {}) 522 monkeypatch.setattr(runner, 'fetch_qq_singers', lambda conn, song_ids: {})
520 monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url) 523 monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url)
521 monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5')) 524 monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5'))
522 monkeypatch.setattr(runner, '_safe_upload_lyric', lambda *args: 'https://bucket.example.com/lyric.txt') 525 monkeypatch.setattr(
526 runner, '_safe_prepare_lyric',
527 lambda *args: ('https://bucket.example.com/lyric.txt', '[00:01.00]HK歌词'),
528 )
523 monkeypatch.setattr(runner, 'upsert_qq_singers', lambda cur, singers: None) 529 monkeypatch.setattr(runner, 'upsert_qq_singers', lambda cur, singers: None)
524 monkeypatch.setattr(runner, 'upsert_qq_albums', lambda cur, albums: None) 530 monkeypatch.setattr(runner, 'upsert_qq_albums', lambda cur, albums: None)
525 monkeypatch.setattr(runner, 'upsert_qq_songs', lambda cur, songs: inserted_songs.extend(songs)) 531 monkeypatch.setattr(runner, 'upsert_qq_songs', lambda cur, songs: inserted_songs.extend(songs))
...@@ -550,6 +556,8 @@ def test_process_qq_builds_album_json_for_song_insert(monkeypatch): ...@@ -550,6 +556,8 @@ def test_process_qq_builds_album_json_for_song_insert(monkeypatch):
550 assert '"title": "专辑"' in inserted_songs[0]['album_json'] 556 assert '"title": "专辑"' in inserted_songs[0]['album_json']
551 assert inserted_songs[0]['title'] == '化风行万里' 557 assert inserted_songs[0]['title'] == '化风行万里'
552 assert inserted_songs[0]['version'] == 'DJ默涵版' 558 assert inserted_songs[0]['version'] == 'DJ默涵版'
559 assert inserted_songs[0]['lyric'] == '[00:01.00]HK歌词'
560 assert inserted_songs[0]['lyric_url'] == 'https://bucket.example.com/lyric.txt'
553 561
554 562
555 def test_process_netease_builds_album_json_for_song_insert(monkeypatch): 563 def test_process_netease_builds_album_json_for_song_insert(monkeypatch):
...@@ -582,7 +590,10 @@ def test_process_netease_builds_album_json_for_song_insert(monkeypatch): ...@@ -582,7 +590,10 @@ def test_process_netease_builds_album_json_for_song_insert(monkeypatch):
582 monkeypatch.setattr(runner, 'fetch_netease_singers', lambda conn, song_ids: {}) 590 monkeypatch.setattr(runner, 'fetch_netease_singers', lambda conn, song_ids: {})
583 monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url) 591 monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url)
584 monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5')) 592 monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5'))
585 monkeypatch.setattr(runner, '_safe_upload_lyric', lambda *args: 'https://bucket.example.com/lyric.txt') 593 monkeypatch.setattr(
594 runner, '_safe_prepare_lyric',
595 lambda *args: ('https://bucket.example.com/lyric.txt', '[00:01.00]HK歌词'),
596 )
586 monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None) 597 monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None)
587 monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None) 598 monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None)
588 monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs)) 599 monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs))
...@@ -625,7 +636,10 @@ def test_process_netease_uses_prefetched_song_and_singers(monkeypatch): ...@@ -625,7 +636,10 @@ def test_process_netease_uses_prefetched_song_and_singers(monkeypatch):
625 monkeypatch.setattr(runner, 'fetch_netease_singers', fetch_singers) 636 monkeypatch.setattr(runner, 'fetch_netease_singers', fetch_singers)
626 monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url) 637 monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url)
627 monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5')) 638 monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5'))
628 monkeypatch.setattr(runner, '_safe_upload_lyric', lambda *args: 'https://bucket.example.com/lyric.txt') 639 monkeypatch.setattr(
640 runner, '_safe_prepare_lyric',
641 lambda *args: ('https://bucket.example.com/lyric.txt', '[00:01.00]HK歌词'),
642 )
629 monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None) 643 monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None)
630 monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None) 644 monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None)
631 monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs)) 645 monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs))
......
1 from unittest.mock import MagicMock, call 1 from unittest.mock import MagicMock, call
2 import pytest
3
2 import etl_to_crawler.writer as writer 4 import etl_to_crawler.writer as writer
3 from etl_to_crawler.writer import ( 5 from etl_to_crawler.writer import (
4 fetch_pending_yinyan_song_records, 6 fetch_pending_yinyan_song_records,
...@@ -327,6 +329,34 @@ def test_upsert_kugou_songs_writes_provider_and_source_data(): ...@@ -327,6 +329,34 @@ def test_upsert_kugou_songs_writes_provider_and_source_data():
327 assert rows[0][-4:] == ('DJ默涵版', 'md5-200', 'yinyan', '{"id": 200}') 329 assert rows[0][-4:] == ('DJ默涵版', 'md5-200', 'yinyan', '{"id": 200}')
328 330
329 331
332 @pytest.mark.parametrize(
333 ('upsert', 'table_name', 'extra'),
334 [
335 (upsert_qq_songs, 'crawler_qqmusic_songs', {'mid': 'song-mid'}),
336 (upsert_kugou_songs, 'crawler_kugou_songs', {'hash': 'hash'}),
337 (upsert_netease_songs, 'crawler_netease_songs', {}),
338 ],
339 )
340 def test_song_conflict_only_fills_empty_cover_and_lyric_fields(upsert, table_name, extra):
341 cur = MagicMock()
342 upsert(cur, [{
343 'song_uuid': 'song-uuid',
344 'platform_song_id': 200,
345 'name': '歌名',
346 'cover': 'https://example.com/cover.jpg',
347 'lyric': '歌词文本',
348 'lyric_url': 'https://example.com/lyric.txt',
349 **extra,
350 }])
351
352 sql = cur.executemany.call_args.args[0]
353 assert 'ON CONFLICT (platform_song_id) DO UPDATE SET' in sql
354 assert f"BTRIM(COALESCE({table_name}.cover, '')) = ''" in sql
355 assert f"BTRIM(COALESCE({table_name}.lyric, '')) = ''" in sql
356 assert f"BTRIM(COALESCE({table_name}.lyric_url, '')) = ''" in sql
357 assert 'THEN EXCLUDED.lyric' in sql
358
359
330 def test_upsert_kugou_singers_writes_provider_and_source_data(): 360 def test_upsert_kugou_singers_writes_provider_and_source_data():
331 cur = MagicMock() 361 cur = MagicMock()
332 upsert_kugou_singers(cur, [{ 362 upsert_kugou_singers(cur, [{
......