refactor(lyric): 优化歌词转存逻辑及更新冲突处理
Showing
4 changed files
with
135 additions
and
41 deletions
| ... | @@ -100,28 +100,55 @@ def _safe_transfer_audio(url, oss_key, bucket, base_url) -> tuple[str, str]: | ... | @@ -100,28 +100,55 @@ def _safe_transfer_audio(url, oss_key, bucket, base_url) -> tuple[str, str]: |
| 100 | return '', '' # 不将未转存的音频外链写入 crawler | 100 | return '', '' # 不将未转存的音频外链写入 crawler |
| 101 | 101 | ||
| 102 | 102 | ||
| 103 | def _safe_upload_lyric( | 103 | def _safe_prepare_lyric( |
| 104 | platform: str, | 104 | platform: str, |
| 105 | unique_id: str, | 105 | unique_id: str, |
| 106 | spider_lyric: str | None, | 106 | spider_lyric: str | None, |
| 107 | hk_lyrics_url: str | None, | 107 | hk_lyrics_url: str | None, |
| 108 | bucket, | 108 | bucket, |
| 109 | base_url: str, | 109 | base_url: str, |
| 110 | ) -> str: | 110 | ) -> tuple[str, str]: |
| 111 | """优先将音眼歌词 URL 转存到 crawler OSS;spider 歌词文本仅作回退。""" | 111 | """返回实际写入的 (lyric_url, lyric_text),音眼歌词优先、spider 回退。""" |
| 112 | candidates: list[tuple[str, str]] = [] | ||
| 112 | if str(hk_lyrics_url or '').strip(): | 113 | if str(hk_lyrics_url or '').strip(): |
| 113 | try: | 114 | try: |
| 114 | hk_lyric = download_text_url(hk_lyrics_url, base_url) | 115 | hk_lyric = download_text_url(hk_lyrics_url, base_url) |
| 115 | uploaded_url = upload_plain_lyric_to_bucket(platform, unique_id, hk_lyric, bucket, base_url) | 116 | if str(hk_lyric or '').strip(): |
| 116 | if uploaded_url: | 117 | candidates.append(('HK', hk_lyric)) |
| 117 | return uploaded_url | ||
| 118 | except Exception as e: | 118 | except Exception as e: |
| 119 | log.warning('HK lyric transfer failed for %s/%s: %s', platform, unique_id, e) | 119 | log.warning('HK lyric download failed for %s/%s: %s', platform, unique_id, e) |
| 120 | |||
| 121 | spider_text = spider_lyric or '' | ||
| 122 | if str(spider_text).strip() and all(text != spider_text for _, text in candidates): | ||
| 123 | candidates.append(('spider', spider_text)) | ||
| 124 | |||
| 125 | for source, lyric_text in candidates: | ||
| 120 | try: | 126 | try: |
| 121 | return upload_plain_lyric_to_bucket(platform, unique_id, spider_lyric or '', bucket, base_url) | 127 | uploaded_url = upload_plain_lyric_to_bucket( |
| 128 | platform, unique_id, lyric_text, bucket, base_url, | ||
| 129 | ) | ||
| 130 | if uploaded_url: | ||
| 131 | return uploaded_url, lyric_text | ||
| 122 | except Exception as e: | 132 | except Exception as e: |
| 123 | log.warning("Lyric upload failed for %s/%s: %s", platform, unique_id, e) | 133 | log.warning('%s lyric upload failed for %s/%s: %s', source, platform, unique_id, e) |
| 124 | return '' | 134 | |
| 135 | # records2 允许 URL 转存失败时入库;若文本已成功获取,仍尽量保留文本。 | ||
| 136 | return '', candidates[0][1] if candidates else '' | ||
| 137 | |||
| 138 | |||
| 139 | def _safe_upload_lyric( | ||
| 140 | platform: str, | ||
| 141 | unique_id: str, | ||
| 142 | spider_lyric: str | None, | ||
| 143 | hk_lyrics_url: str | None, | ||
| 144 | bucket, | ||
| 145 | base_url: str, | ||
| 146 | ) -> str: | ||
| 147 | """兼容仅需要歌词 URL 的调用方。""" | ||
| 148 | lyric_url, _ = _safe_prepare_lyric( | ||
| 149 | platform, unique_id, spider_lyric, hk_lyrics_url, bucket, base_url, | ||
| 150 | ) | ||
| 151 | return lyric_url | ||
| 125 | 152 | ||
| 126 | 153 | ||
| 127 | def _required_import_assets() -> tuple[str, ...]: | 154 | def _required_import_assets() -> tuple[str, ...]: |
| ... | @@ -134,7 +161,9 @@ def _required_import_assets() -> tuple[str, ...]: | ... | @@ -134,7 +161,9 @@ def _required_import_assets() -> tuple[str, ...]: |
| 134 | def _require_primary_assets(assets: dict, required: tuple[str, ...] | None = None) -> None: | 161 | def _require_primary_assets(assets: dict, required: tuple[str, ...] | None = None) -> None: |
| 135 | """校验当前导入支线要求的资源,允许非必需资源以空值入库。""" | 162 | """校验当前导入支线要求的资源,允许非必需资源以空值入库。""" |
| 136 | audio_url, _ = assets.get('audio', ('', '')) | 163 | audio_url, _ = assets.get('audio', ('', '')) |
| 137 | values = {'audio': audio_url, 'cover': assets.get('cover'), 'lyric': assets.get('lyric')} | 164 | lyric_asset = assets.get('lyric') |
| 165 | lyric_url = lyric_asset[0] if isinstance(lyric_asset, tuple) else lyric_asset | ||
| 166 | values = {'audio': audio_url, 'cover': assets.get('cover'), 'lyric': lyric_url} | ||
| 138 | required = required or ('audio', 'cover', 'lyric') | 167 | required = required or ('audio', 'cover', 'lyric') |
| 139 | missing = [name for name in required if not values.get(name)] | 168 | missing = [name for name in required if not values.get(name)] |
| 140 | if missing: | 169 | if missing: |
| ... | @@ -330,7 +359,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing | ... | @@ -330,7 +359,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing |
| 330 | build_oss_key('qq', 'cover', str(song_id_int) + '.jpg'), | 359 | build_oss_key('qq', 'cover', str(song_id_int) + '.jpg'), |
| 331 | bucket, base_url, | 360 | bucket, base_url, |
| 332 | ), | 361 | ), |
| 333 | 'lyric': lambda: _safe_upload_lyric('qq', mid, raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), | 362 | 'lyric': lambda: _safe_prepare_lyric('qq', mid, raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), |
| 334 | } | 363 | } |
| 335 | if sp.get('album_id') and sp.get('album_cover'): | 364 | if sp.get('album_id') and sp.get('album_cover'): |
| 336 | tasks['album_cover'] = lambda: _safe_transfer( | 365 | tasks['album_cover'] = lambda: _safe_transfer( |
| ... | @@ -348,7 +377,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing | ... | @@ -348,7 +377,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing |
| 348 | _require_primary_assets(assets, _required_import_assets()) | 377 | _require_primary_assets(assets, _required_import_assets()) |
| 349 | audio_url, audio_md5 = assets['audio'] | 378 | audio_url, audio_md5 = assets['audio'] |
| 350 | cover_url = assets['cover'] | 379 | cover_url = assets['cover'] |
| 351 | lyric_url = assets['lyric'] | 380 | lyric_url, lyric_text = assets['lyric'] |
| 352 | album_cover = assets.get('album_cover') or cover_url | 381 | album_cover = assets.get('album_cover') or cover_url |
| 353 | 382 | ||
| 354 | singer_rows = [{ | 383 | singer_rows = [{ |
| ... | @@ -422,7 +451,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing | ... | @@ -422,7 +451,7 @@ def _prepare_qq_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, sing |
| 422 | 'version': version, | 451 | 'version': version, |
| 423 | 'name': hk_row['name'], | 452 | 'name': hk_row['name'], |
| 424 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, | 453 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, |
| 425 | 'lyric': ensure_newlines(raw_lyric), | 454 | 'lyric': ensure_newlines(lyric_text), |
| 426 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), | 455 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), |
| 427 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), | 456 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), |
| 428 | 'url': audio_url, | 457 | 'url': audio_url, |
| ... | @@ -460,7 +489,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s | ... | @@ -460,7 +489,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s |
| 460 | build_oss_key('kugou', 'cover', str(song_id) + '.jpg'), | 489 | build_oss_key('kugou', 'cover', str(song_id) + '.jpg'), |
| 461 | bucket, base_url, | 490 | bucket, base_url, |
| 462 | ), | 491 | ), |
| 463 | 'lyric': lambda: _safe_upload_lyric('kugou', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), | 492 | 'lyric': lambda: _safe_prepare_lyric('kugou', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), |
| 464 | } | 493 | } |
| 465 | if sp.get('album_id') and sp.get('album_cover'): | 494 | if sp.get('album_id') and sp.get('album_cover'): |
| 466 | tasks['album_cover'] = lambda: _safe_transfer( | 495 | tasks['album_cover'] = lambda: _safe_transfer( |
| ... | @@ -478,7 +507,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s | ... | @@ -478,7 +507,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s |
| 478 | _require_primary_assets(assets, _required_import_assets()) | 507 | _require_primary_assets(assets, _required_import_assets()) |
| 479 | audio_url, audio_md5 = assets['audio'] | 508 | audio_url, audio_md5 = assets['audio'] |
| 480 | cover_url = assets['cover'] | 509 | cover_url = assets['cover'] |
| 481 | lyric_url = assets['lyric'] | 510 | lyric_url, lyric_text = assets['lyric'] |
| 482 | album_cover = assets.get('album_cover') or cover_url | 511 | album_cover = assets.get('album_cover') or cover_url |
| 483 | 512 | ||
| 484 | singer_rows = [{ | 513 | singer_rows = [{ |
| ... | @@ -535,7 +564,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s | ... | @@ -535,7 +564,7 @@ def _prepare_kugou_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, s |
| 535 | 'version': version, | 564 | 'version': version, |
| 536 | 'name': hk_row['name'], | 565 | 'name': hk_row['name'], |
| 537 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, | 566 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, |
| 538 | 'lyric': ensure_newlines(raw_lyric), | 567 | 'lyric': ensure_newlines(lyric_text), |
| 539 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), | 568 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), |
| 540 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), | 569 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), |
| 541 | 'url': audio_url, | 570 | 'url': audio_url, |
| ... | @@ -572,7 +601,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, | ... | @@ -572,7 +601,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, |
| 572 | build_oss_key('netease', 'cover', str(song_id) + '.jpg'), | 601 | build_oss_key('netease', 'cover', str(song_id) + '.jpg'), |
| 573 | bucket, base_url, | 602 | bucket, base_url, |
| 574 | ), | 603 | ), |
| 575 | 'lyric': lambda: _safe_upload_lyric('netease', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), | 604 | 'lyric': lambda: _safe_prepare_lyric('netease', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), |
| 576 | } | 605 | } |
| 577 | if sp.get('album_id') and sp.get('album_cover'): | 606 | if sp.get('album_id') and sp.get('album_cover'): |
| 578 | tasks['album_cover'] = lambda: _safe_transfer( | 607 | tasks['album_cover'] = lambda: _safe_transfer( |
| ... | @@ -590,7 +619,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, | ... | @@ -590,7 +619,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, |
| 590 | _require_primary_assets(assets, _required_import_assets()) | 619 | _require_primary_assets(assets, _required_import_assets()) |
| 591 | audio_url, audio_md5 = assets['audio'] | 620 | audio_url, audio_md5 = assets['audio'] |
| 592 | cover_url = assets['cover'] | 621 | cover_url = assets['cover'] |
| 593 | lyric_url = assets['lyric'] | 622 | lyric_url, lyric_text = assets['lyric'] |
| 594 | album_cover = assets.get('album_cover') or cover_url | 623 | album_cover = assets.get('album_cover') or cover_url |
| 595 | 624 | ||
| 596 | singer_rows = [{ | 625 | singer_rows = [{ |
| ... | @@ -659,7 +688,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, | ... | @@ -659,7 +688,7 @@ def _prepare_netease_payload(hk_row: dict, pr: dict, bucket, base_url, sp: dict, |
| 659 | 'version': version, | 688 | 'version': version, |
| 660 | 'name': hk_row['name'], | 689 | 'name': hk_row['name'], |
| 661 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, | 690 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, |
| 662 | 'lyric': ensure_newlines(raw_lyric), | 691 | 'lyric': ensure_newlines(lyric_text), |
| 663 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), | 692 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), |
| 664 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), | 693 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), |
| 665 | 'url': audio_url, | 694 | 'url': audio_url, |
| ... | @@ -870,7 +899,7 @@ def _process_qq( | ... | @@ -870,7 +899,7 @@ def _process_qq( |
| 870 | build_oss_key('qq', 'cover', str(song_id_int) + '.jpg'), | 899 | build_oss_key('qq', 'cover', str(song_id_int) + '.jpg'), |
| 871 | bucket, base_url | 900 | bucket, base_url |
| 872 | ), | 901 | ), |
| 873 | 'lyric': lambda: _safe_upload_lyric('qq', mid, raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), | 902 | 'lyric': lambda: _safe_prepare_lyric('qq', mid, raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), |
| 874 | } | 903 | } |
| 875 | if sp.get('album_id') and sp.get('album_cover'): | 904 | if sp.get('album_id') and sp.get('album_cover'): |
| 876 | tasks['album_cover'] = lambda: _safe_transfer( | 905 | tasks['album_cover'] = lambda: _safe_transfer( |
| ... | @@ -890,7 +919,7 @@ def _process_qq( | ... | @@ -890,7 +919,7 @@ def _process_qq( |
| 890 | _require_primary_assets(assets) | 919 | _require_primary_assets(assets) |
| 891 | audio_url, audio_md5 = assets['audio'] | 920 | audio_url, audio_md5 = assets['audio'] |
| 892 | cover_url = assets['cover'] | 921 | cover_url = assets['cover'] |
| 893 | lyric_url = assets['lyric'] | 922 | lyric_url, lyric_text = assets['lyric'] |
| 894 | album_cover = assets.get('album_cover') or cover_url | 923 | album_cover = assets.get('album_cover') or cover_url |
| 895 | 924 | ||
| 896 | # 歌手头像转移 + 写入 singers | 925 | # 歌手头像转移 + 写入 singers |
| ... | @@ -967,7 +996,7 @@ def _process_qq( | ... | @@ -967,7 +996,7 @@ def _process_qq( |
| 967 | 'version': version, | 996 | 'version': version, |
| 968 | 'name': hk_row['name'], | 997 | 'name': hk_row['name'], |
| 969 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, | 998 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, |
| 970 | 'lyric': ensure_newlines(raw_lyric), | 999 | 'lyric': ensure_newlines(lyric_text), |
| 971 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), | 1000 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), |
| 972 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), | 1001 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), |
| 973 | 'url': audio_url, | 1002 | 'url': audio_url, |
| ... | @@ -1026,7 +1055,7 @@ def _process_kugou( | ... | @@ -1026,7 +1055,7 @@ def _process_kugou( |
| 1026 | build_oss_key('kugou', 'cover', str(song_id) + '.jpg'), | 1055 | build_oss_key('kugou', 'cover', str(song_id) + '.jpg'), |
| 1027 | bucket, base_url | 1056 | bucket, base_url |
| 1028 | ), | 1057 | ), |
| 1029 | 'lyric': lambda: _safe_upload_lyric('kugou', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), | 1058 | 'lyric': lambda: _safe_prepare_lyric('kugou', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), |
| 1030 | } | 1059 | } |
| 1031 | if sp.get('album_id') and sp.get('album_cover'): | 1060 | if sp.get('album_id') and sp.get('album_cover'): |
| 1032 | tasks['album_cover'] = lambda: _safe_transfer( | 1061 | tasks['album_cover'] = lambda: _safe_transfer( |
| ... | @@ -1046,7 +1075,7 @@ def _process_kugou( | ... | @@ -1046,7 +1075,7 @@ def _process_kugou( |
| 1046 | _require_primary_assets(assets) | 1075 | _require_primary_assets(assets) |
| 1047 | audio_url, audio_md5 = assets['audio'] | 1076 | audio_url, audio_md5 = assets['audio'] |
| 1048 | cover_url = assets['cover'] | 1077 | cover_url = assets['cover'] |
| 1049 | lyric_url = assets['lyric'] | 1078 | lyric_url, lyric_text = assets['lyric'] |
| 1050 | album_cover = assets.get('album_cover') or cover_url | 1079 | album_cover = assets.get('album_cover') or cover_url |
| 1051 | 1080 | ||
| 1052 | singer_rows = [] | 1081 | singer_rows = [] |
| ... | @@ -1103,7 +1132,7 @@ def _process_kugou( | ... | @@ -1103,7 +1132,7 @@ def _process_kugou( |
| 1103 | 'version': version, | 1132 | 'version': version, |
| 1104 | 'name': hk_row['name'], | 1133 | 'name': hk_row['name'], |
| 1105 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, | 1134 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, |
| 1106 | 'lyric': ensure_newlines(raw_lyric), | 1135 | 'lyric': ensure_newlines(lyric_text), |
| 1107 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), | 1136 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), |
| 1108 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), | 1137 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), |
| 1109 | 'url': audio_url, | 1138 | 'url': audio_url, |
| ... | @@ -1161,7 +1190,7 @@ def _process_netease( | ... | @@ -1161,7 +1190,7 @@ def _process_netease( |
| 1161 | build_oss_key('netease', 'cover', str(song_id) + '.jpg'), | 1190 | build_oss_key('netease', 'cover', str(song_id) + '.jpg'), |
| 1162 | bucket, base_url | 1191 | bucket, base_url |
| 1163 | ), | 1192 | ), |
| 1164 | 'lyric': lambda: _safe_upload_lyric('netease', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), | 1193 | 'lyric': lambda: _safe_prepare_lyric('netease', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url), |
| 1165 | } | 1194 | } |
| 1166 | if sp.get('album_id') and sp.get('album_cover'): | 1195 | if sp.get('album_id') and sp.get('album_cover'): |
| 1167 | tasks['album_cover'] = lambda: _safe_transfer( | 1196 | tasks['album_cover'] = lambda: _safe_transfer( |
| ... | @@ -1181,7 +1210,7 @@ def _process_netease( | ... | @@ -1181,7 +1210,7 @@ def _process_netease( |
| 1181 | _require_primary_assets(assets) | 1210 | _require_primary_assets(assets) |
| 1182 | audio_url, audio_md5 = assets['audio'] | 1211 | audio_url, audio_md5 = assets['audio'] |
| 1183 | cover_url = assets['cover'] | 1212 | cover_url = assets['cover'] |
| 1184 | lyric_url = assets['lyric'] | 1213 | lyric_url, lyric_text = assets['lyric'] |
| 1185 | album_cover = assets.get('album_cover') or cover_url | 1214 | album_cover = assets.get('album_cover') or cover_url |
| 1186 | 1215 | ||
| 1187 | singer_rows = [] | 1216 | singer_rows = [] |
| ... | @@ -1250,7 +1279,7 @@ def _process_netease( | ... | @@ -1250,7 +1279,7 @@ def _process_netease( |
| 1250 | 'version': version, | 1279 | 'version': version, |
| 1251 | 'name': hk_row['name'], | 1280 | 'name': hk_row['name'], |
| 1252 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, | 1281 | 'duration': sp.get('duration') or hk_row.get('song_time') or 0, |
| 1253 | 'lyric': ensure_newlines(raw_lyric), | 1282 | 'lyric': ensure_newlines(lyric_text), |
| 1254 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), | 1283 | 'composer_name': sp.get('composer_name') or hk_row.get('composer'), |
| 1255 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), | 1284 | 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), |
| 1256 | 'url': audio_url, | 1285 | 'url': audio_url, | ... | ... |
| ... | @@ -442,7 +442,14 @@ def upsert_qq_songs(cur, songs: list[dict]) -> None: | ... | @@ -442,7 +442,14 @@ def upsert_qq_songs(cur, songs: list[dict]) -> None: |
| 442 | platform_index_url, published_at, album, singers, status, created_at, updated_at, | 442 | platform_index_url, published_at, album, singers, status, created_at, updated_at, |
| 443 | version, audio_md5, provider_name, crawler_source_data) | 443 | version, audio_md5, provider_name, crawler_source_data) |
| 444 | VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json) | 444 | VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json) |
| 445 | ON CONFLICT (platform_song_id) DO NOTHING | 445 | ON CONFLICT (platform_song_id) DO UPDATE SET |
| 446 | cover = CASE WHEN BTRIM(COALESCE(crawler_qqmusic_songs.cover, '')) = '' | ||
| 447 | THEN EXCLUDED.cover ELSE crawler_qqmusic_songs.cover END, | ||
| 448 | lyric = CASE WHEN BTRIM(COALESCE(crawler_qqmusic_songs.lyric, '')) = '' | ||
| 449 | THEN EXCLUDED.lyric ELSE crawler_qqmusic_songs.lyric END, | ||
| 450 | lyric_url = CASE WHEN BTRIM(COALESCE(crawler_qqmusic_songs.lyric_url, '')) = '' | ||
| 451 | THEN EXCLUDED.lyric_url ELSE crawler_qqmusic_songs.lyric_url END, | ||
| 452 | updated_at = NOW() | ||
| 446 | """ | 453 | """ |
| 447 | rows = [( | 454 | rows = [( |
| 448 | s['song_uuid'], s['platform_song_id'], s['mid'], s.get('album_id'), | 455 | s['song_uuid'], s['platform_song_id'], s['mid'], s.get('album_id'), |
| ... | @@ -534,7 +541,14 @@ def upsert_kugou_songs(cur, songs: list[dict]) -> None: | ... | @@ -534,7 +541,14 @@ def upsert_kugou_songs(cur, songs: list[dict]) -> None: |
| 534 | platform_index_url, published_at, singers, status, created_at, updated_at, | 541 | platform_index_url, published_at, singers, status, created_at, updated_at, |
| 535 | version, audio_md5, provider_name, crawler_source_data) | 542 | version, audio_md5, provider_name, crawler_source_data) |
| 536 | VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json) | 543 | VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json) |
| 537 | ON CONFLICT (platform_song_id) DO NOTHING | 544 | ON CONFLICT (platform_song_id) DO UPDATE SET |
| 545 | cover = CASE WHEN BTRIM(COALESCE(crawler_kugou_songs.cover, '')) = '' | ||
| 546 | THEN EXCLUDED.cover ELSE crawler_kugou_songs.cover END, | ||
| 547 | lyric = CASE WHEN BTRIM(COALESCE(crawler_kugou_songs.lyric, '')) = '' | ||
| 548 | THEN EXCLUDED.lyric ELSE crawler_kugou_songs.lyric END, | ||
| 549 | lyric_url = CASE WHEN BTRIM(COALESCE(crawler_kugou_songs.lyric_url, '')) = '' | ||
| 550 | THEN EXCLUDED.lyric_url ELSE crawler_kugou_songs.lyric_url END, | ||
| 551 | updated_at = NOW() | ||
| 538 | """ | 552 | """ |
| 539 | rows = [( | 553 | rows = [( |
| 540 | s['song_uuid'], s['platform_song_id'], s.get('hash', ''), | 554 | s['song_uuid'], s['platform_song_id'], s.get('hash', ''), |
| ... | @@ -623,7 +637,14 @@ def upsert_netease_songs(cur, songs: list[dict]) -> None: | ... | @@ -623,7 +637,14 @@ def upsert_netease_songs(cur, songs: list[dict]) -> None: |
| 623 | platform_index_url, published_at, album, singers, status, created_at, updated_at, | 637 | platform_index_url, published_at, album, singers, status, created_at, updated_at, |
| 624 | version, audio_md5, provider_name, crawler_source_data) | 638 | version, audio_md5, provider_name, crawler_source_data) |
| 625 | VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json) | 639 | VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s, %s::json) |
| 626 | ON CONFLICT (platform_song_id) DO NOTHING | 640 | ON CONFLICT (platform_song_id) DO UPDATE SET |
| 641 | cover = CASE WHEN BTRIM(COALESCE(crawler_netease_songs.cover, '')) = '' | ||
| 642 | THEN EXCLUDED.cover ELSE crawler_netease_songs.cover END, | ||
| 643 | lyric = CASE WHEN BTRIM(COALESCE(crawler_netease_songs.lyric, '')) = '' | ||
| 644 | THEN EXCLUDED.lyric ELSE crawler_netease_songs.lyric END, | ||
| 645 | lyric_url = CASE WHEN BTRIM(COALESCE(crawler_netease_songs.lyric_url, '')) = '' | ||
| 646 | THEN EXCLUDED.lyric_url ELSE crawler_netease_songs.lyric_url END, | ||
| 647 | updated_at = NOW() | ||
| 627 | """ | 648 | """ |
| 628 | rows = [( | 649 | rows = [( |
| 629 | s['song_uuid'], s['platform_song_id'], s.get('album_id'), | 650 | s['song_uuid'], s['platform_song_id'], s.get('album_id'), | ... | ... |
| ... | @@ -105,32 +105,35 @@ def test_records2_import_allows_empty_cover_and_lyric(monkeypatch): | ... | @@ -105,32 +105,35 @@ def test_records2_import_allows_empty_cover_and_lyric(monkeypatch): |
| 105 | }, runner._required_import_assets()) | 105 | }, runner._required_import_assets()) |
| 106 | 106 | ||
| 107 | 107 | ||
| 108 | def test_lyric_upload_prefers_hk_url_and_forces_oss_transfer(monkeypatch): | 108 | def test_lyric_prepare_prefers_hk_text_for_both_url_and_text_column(monkeypatch): |
| 109 | download = MagicMock(return_value='[00:01.00]HK lyric') | 109 | download = MagicMock(return_value='[00:01.00]HK lyric') |
| 110 | monkeypatch.setattr(runner, 'download_text_url', download) | 110 | monkeypatch.setattr(runner, 'download_text_url', download) |
| 111 | upload = MagicMock(return_value='https://archive.example/crawler/lyric/qq/mid.txt') | 111 | upload = MagicMock(return_value='https://archive.example/crawler/lyric/qq/mid.txt') |
| 112 | monkeypatch.setattr(runner, 'upload_plain_lyric_to_bucket', upload) | 112 | monkeypatch.setattr(runner, 'upload_plain_lyric_to_bucket', upload) |
| 113 | bucket = object() | 113 | bucket = object() |
| 114 | 114 | ||
| 115 | result = runner._safe_upload_lyric( | 115 | result = runner._safe_prepare_lyric( |
| 116 | 'qq', 'mid', 'spider lyric', 'https://source.example/lyric.txt', bucket, 'https://archive.example', | 116 | 'qq', 'mid', 'spider lyric', 'https://source.example/lyric.txt', bucket, 'https://archive.example', |
| 117 | ) | 117 | ) |
| 118 | 118 | ||
| 119 | assert result == 'https://archive.example/crawler/lyric/qq/mid.txt' | 119 | assert result == ( |
| 120 | 'https://archive.example/crawler/lyric/qq/mid.txt', | ||
| 121 | '[00:01.00]HK lyric', | ||
| 122 | ) | ||
| 120 | download.assert_called_once_with('https://source.example/lyric.txt', 'https://archive.example') | 123 | download.assert_called_once_with('https://source.example/lyric.txt', 'https://archive.example') |
| 121 | upload.assert_called_once_with( | 124 | upload.assert_called_once_with( |
| 122 | 'qq', 'mid', '[00:01.00]HK lyric', bucket, 'https://archive.example', | 125 | 'qq', 'mid', '[00:01.00]HK lyric', bucket, 'https://archive.example', |
| 123 | ) | 126 | ) |
| 124 | 127 | ||
| 125 | 128 | ||
| 126 | def test_lyric_upload_falls_back_to_spider_text_when_hk_transfer_fails(monkeypatch): | 129 | def test_lyric_prepare_falls_back_to_spider_text_when_hk_download_fails(monkeypatch): |
| 127 | monkeypatch.setattr(runner, 'download_text_url', MagicMock(side_effect=TimeoutError('timeout'))) | 130 | monkeypatch.setattr(runner, 'download_text_url', MagicMock(side_effect=TimeoutError('timeout'))) |
| 128 | upload = MagicMock(return_value='https://archive.example/crawler/lyric/qq/mid.txt') | 131 | upload = MagicMock(return_value='https://archive.example/crawler/lyric/qq/mid.txt') |
| 129 | monkeypatch.setattr(runner, 'upload_plain_lyric_to_bucket', upload) | 132 | monkeypatch.setattr(runner, 'upload_plain_lyric_to_bucket', upload) |
| 130 | 133 | ||
| 131 | assert runner._safe_upload_lyric( | 134 | assert runner._safe_prepare_lyric( |
| 132 | 'qq', 'mid', 'spider lyric', 'https://source.example/lyric.txt', object(), 'https://archive.example', | 135 | 'qq', 'mid', 'spider lyric', 'https://source.example/lyric.txt', object(), 'https://archive.example', |
| 133 | ) == 'https://archive.example/crawler/lyric/qq/mid.txt' | 136 | ) == ('https://archive.example/crawler/lyric/qq/mid.txt', 'spider lyric') |
| 134 | assert upload.call_args.args[2] == 'spider lyric' | 137 | assert upload.call_args.args[2] == 'spider lyric' |
| 135 | 138 | ||
| 136 | 139 | ||
| ... | @@ -519,7 +522,10 @@ def test_process_qq_builds_album_json_for_song_insert(monkeypatch): | ... | @@ -519,7 +522,10 @@ def test_process_qq_builds_album_json_for_song_insert(monkeypatch): |
| 519 | monkeypatch.setattr(runner, 'fetch_qq_singers', lambda conn, song_ids: {}) | 522 | monkeypatch.setattr(runner, 'fetch_qq_singers', lambda conn, song_ids: {}) |
| 520 | monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url) | 523 | monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url) |
| 521 | monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5')) | 524 | monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5')) |
| 522 | monkeypatch.setattr(runner, '_safe_upload_lyric', lambda *args: 'https://bucket.example.com/lyric.txt') | 525 | monkeypatch.setattr( |
| 526 | runner, '_safe_prepare_lyric', | ||
| 527 | lambda *args: ('https://bucket.example.com/lyric.txt', '[00:01.00]HK歌词'), | ||
| 528 | ) | ||
| 523 | monkeypatch.setattr(runner, 'upsert_qq_singers', lambda cur, singers: None) | 529 | monkeypatch.setattr(runner, 'upsert_qq_singers', lambda cur, singers: None) |
| 524 | monkeypatch.setattr(runner, 'upsert_qq_albums', lambda cur, albums: None) | 530 | monkeypatch.setattr(runner, 'upsert_qq_albums', lambda cur, albums: None) |
| 525 | monkeypatch.setattr(runner, 'upsert_qq_songs', lambda cur, songs: inserted_songs.extend(songs)) | 531 | monkeypatch.setattr(runner, 'upsert_qq_songs', lambda cur, songs: inserted_songs.extend(songs)) |
| ... | @@ -550,6 +556,8 @@ def test_process_qq_builds_album_json_for_song_insert(monkeypatch): | ... | @@ -550,6 +556,8 @@ def test_process_qq_builds_album_json_for_song_insert(monkeypatch): |
| 550 | assert '"title": "专辑"' in inserted_songs[0]['album_json'] | 556 | assert '"title": "专辑"' in inserted_songs[0]['album_json'] |
| 551 | assert inserted_songs[0]['title'] == '化风行万里' | 557 | assert inserted_songs[0]['title'] == '化风行万里' |
| 552 | assert inserted_songs[0]['version'] == 'DJ默涵版' | 558 | assert inserted_songs[0]['version'] == 'DJ默涵版' |
| 559 | assert inserted_songs[0]['lyric'] == '[00:01.00]HK歌词' | ||
| 560 | assert inserted_songs[0]['lyric_url'] == 'https://bucket.example.com/lyric.txt' | ||
| 553 | 561 | ||
| 554 | 562 | ||
| 555 | def test_process_netease_builds_album_json_for_song_insert(monkeypatch): | 563 | def test_process_netease_builds_album_json_for_song_insert(monkeypatch): |
| ... | @@ -582,7 +590,10 @@ def test_process_netease_builds_album_json_for_song_insert(monkeypatch): | ... | @@ -582,7 +590,10 @@ def test_process_netease_builds_album_json_for_song_insert(monkeypatch): |
| 582 | monkeypatch.setattr(runner, 'fetch_netease_singers', lambda conn, song_ids: {}) | 590 | monkeypatch.setattr(runner, 'fetch_netease_singers', lambda conn, song_ids: {}) |
| 583 | monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url) | 591 | monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url) |
| 584 | monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5')) | 592 | monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5')) |
| 585 | monkeypatch.setattr(runner, '_safe_upload_lyric', lambda *args: 'https://bucket.example.com/lyric.txt') | 593 | monkeypatch.setattr( |
| 594 | runner, '_safe_prepare_lyric', | ||
| 595 | lambda *args: ('https://bucket.example.com/lyric.txt', '[00:01.00]HK歌词'), | ||
| 596 | ) | ||
| 586 | monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None) | 597 | monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None) |
| 587 | monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None) | 598 | monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None) |
| 588 | monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs)) | 599 | monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs)) |
| ... | @@ -625,7 +636,10 @@ def test_process_netease_uses_prefetched_song_and_singers(monkeypatch): | ... | @@ -625,7 +636,10 @@ def test_process_netease_uses_prefetched_song_and_singers(monkeypatch): |
| 625 | monkeypatch.setattr(runner, 'fetch_netease_singers', fetch_singers) | 636 | monkeypatch.setattr(runner, 'fetch_netease_singers', fetch_singers) |
| 626 | monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url) | 637 | monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url) |
| 627 | monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5')) | 638 | monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5')) |
| 628 | monkeypatch.setattr(runner, '_safe_upload_lyric', lambda *args: 'https://bucket.example.com/lyric.txt') | 639 | monkeypatch.setattr( |
| 640 | runner, '_safe_prepare_lyric', | ||
| 641 | lambda *args: ('https://bucket.example.com/lyric.txt', '[00:01.00]HK歌词'), | ||
| 642 | ) | ||
| 629 | monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None) | 643 | monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None) |
| 630 | monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None) | 644 | monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None) |
| 631 | monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs)) | 645 | monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs)) | ... | ... |
| 1 | from unittest.mock import MagicMock, call | 1 | from unittest.mock import MagicMock, call |
| 2 | import pytest | ||
| 3 | |||
| 2 | import etl_to_crawler.writer as writer | 4 | import etl_to_crawler.writer as writer |
| 3 | from etl_to_crawler.writer import ( | 5 | from etl_to_crawler.writer import ( |
| 4 | fetch_pending_yinyan_song_records, | 6 | fetch_pending_yinyan_song_records, |
| ... | @@ -327,6 +329,34 @@ def test_upsert_kugou_songs_writes_provider_and_source_data(): | ... | @@ -327,6 +329,34 @@ def test_upsert_kugou_songs_writes_provider_and_source_data(): |
| 327 | assert rows[0][-4:] == ('DJ默涵版', 'md5-200', 'yinyan', '{"id": 200}') | 329 | assert rows[0][-4:] == ('DJ默涵版', 'md5-200', 'yinyan', '{"id": 200}') |
| 328 | 330 | ||
| 329 | 331 | ||
| 332 | @pytest.mark.parametrize( | ||
| 333 | ('upsert', 'table_name', 'extra'), | ||
| 334 | [ | ||
| 335 | (upsert_qq_songs, 'crawler_qqmusic_songs', {'mid': 'song-mid'}), | ||
| 336 | (upsert_kugou_songs, 'crawler_kugou_songs', {'hash': 'hash'}), | ||
| 337 | (upsert_netease_songs, 'crawler_netease_songs', {}), | ||
| 338 | ], | ||
| 339 | ) | ||
| 340 | def test_song_conflict_only_fills_empty_cover_and_lyric_fields(upsert, table_name, extra): | ||
| 341 | cur = MagicMock() | ||
| 342 | upsert(cur, [{ | ||
| 343 | 'song_uuid': 'song-uuid', | ||
| 344 | 'platform_song_id': 200, | ||
| 345 | 'name': '歌名', | ||
| 346 | 'cover': 'https://example.com/cover.jpg', | ||
| 347 | 'lyric': '歌词文本', | ||
| 348 | 'lyric_url': 'https://example.com/lyric.txt', | ||
| 349 | **extra, | ||
| 350 | }]) | ||
| 351 | |||
| 352 | sql = cur.executemany.call_args.args[0] | ||
| 353 | assert 'ON CONFLICT (platform_song_id) DO UPDATE SET' in sql | ||
| 354 | assert f"BTRIM(COALESCE({table_name}.cover, '')) = ''" in sql | ||
| 355 | assert f"BTRIM(COALESCE({table_name}.lyric, '')) = ''" in sql | ||
| 356 | assert f"BTRIM(COALESCE({table_name}.lyric_url, '')) = ''" in sql | ||
| 357 | assert 'THEN EXCLUDED.lyric' in sql | ||
| 358 | |||
| 359 | |||
| 330 | def test_upsert_kugou_singers_writes_provider_and_source_data(): | 360 | def test_upsert_kugou_singers_writes_provider_and_source_data(): |
| 331 | cur = MagicMock() | 361 | cur = MagicMock() |
| 332 | upsert_kugou_singers(cur, [{ | 362 | upsert_kugou_singers(cur, [{ | ... | ... |
-
Please register or sign in to post a comment