Commit 37eefbba 37eefbba91b99fed15872cd465ce39f5713598cd by 沈秋雨

refactor(crawler): 调整oss路径结构以统一目录层级

- 修改路径顺序,将平台(platform)调整至类别(category)之后
1 parent 7ef2362f
......@@ -108,4 +108,4 @@ def build_oss_key(platform: str, category: str, filename: str) -> str:
category: 'audio' | 'cover' | 'singer' | 'album'
filename: 带扩展名的文件名
"""
return f"crawler/{platform}/{category}/{filename}"
return f"crawler/{category}/{platform}/{filename}"
......
......@@ -113,7 +113,7 @@ def upload_plain_lyric_to_bucket(platform: str, unique_id: str, lyric: str, buck
plain_lyric = extract_plain_lyric(lyric)
if not plain_lyric:
return ""
oss_key = f"crawler/{platform}/lyric/{unique_id}.txt"
oss_key = f"crawler/lyric/{platform}/{unique_id}.txt"
bucket.put_object(oss_key, plain_lyric.encode("utf-8"), headers={'Content-Type': 'text/plain; charset=utf-8'})
return f"{base_url.rstrip('/')}/{oss_key}"
......
......@@ -42,9 +42,9 @@ def test_external_url_downloads_and_uploads():
with patch('etl_to_crawler.oss._http_get') as mock_get:
mock_get.return_value.content = fake_content
mock_get.return_value.raise_for_status = MagicMock()
result = transfer_url(OTHER_URL, "crawler/qq/audio/abc.mp3", bucket, BASE_URL)
bucket.put_object.assert_called_once_with("crawler/qq/audio/abc.mp3", fake_content)
assert result == f"{BASE_URL}/crawler/qq/audio/abc.mp3"
result = transfer_url(OTHER_URL, "crawler/audio/qq/abc.mp3", bucket, BASE_URL)
bucket.put_object.assert_called_once_with("crawler/audio/qq/abc.mp3", fake_content)
assert result == f"{BASE_URL}/crawler/audio/qq/abc.mp3"
def test_external_url_can_rewrite_download_base_to_internal_endpoint():
......@@ -59,11 +59,11 @@ def test_external_url_can_rewrite_download_base_to_internal_endpoint():
with patch('etl_to_crawler.oss._http_get') as mock_get:
mock_get.return_value.content = fake_content
mock_get.return_value.raise_for_status = MagicMock()
result = transfer_url(public_source, "crawler/qq/audio/abc.mp3", bucket, BASE_URL)
result = transfer_url(public_source, "crawler/audio/qq/abc.mp3", bucket, BASE_URL)
mock_get.assert_called_once_with(f"{internal_source_base}/audio/abc.mp3", timeout=30)
bucket.put_object.assert_called_once_with("crawler/qq/audio/abc.mp3", fake_content)
assert result == f"{BASE_URL}/crawler/qq/audio/abc.mp3"
bucket.put_object.assert_called_once_with("crawler/audio/qq/abc.mp3", fake_content)
assert result == f"{BASE_URL}/crawler/audio/qq/abc.mp3"
def test_transfer_url_with_md5_hashes_downloaded_content_before_upload():
......@@ -72,9 +72,9 @@ def test_transfer_url_with_md5_hashes_downloaded_content_before_upload():
with patch('etl_to_crawler.oss._http_get') as mock_get:
mock_get.return_value.content = fake_content
mock_get.return_value.raise_for_status = MagicMock()
result, audio_md5 = transfer_url_with_md5(OTHER_URL, "crawler/qq/audio/abc.mp3", bucket, BASE_URL)
bucket.put_object.assert_called_once_with("crawler/qq/audio/abc.mp3", fake_content)
assert result == f"{BASE_URL}/crawler/qq/audio/abc.mp3"
result, audio_md5 = transfer_url_with_md5(OTHER_URL, "crawler/audio/qq/abc.mp3", bucket, BASE_URL)
bucket.put_object.assert_called_once_with("crawler/audio/qq/abc.mp3", fake_content)
assert result == f"{BASE_URL}/crawler/audio/qq/abc.mp3"
assert audio_md5 == "04d43544b267629d9089eaed3b847a99"
......@@ -84,11 +84,11 @@ def test_audio_transfer_retries_timeout_then_uploads():
response.raise_for_status.return_value = None
with patch('etl_to_crawler.oss._http_get', side_effect=[requests.ReadTimeout('slow'), response]) as mock_get:
with patch('etl_to_crawler.oss.time.sleep') as mock_sleep:
result, _ = transfer_url_with_md5(OTHER_URL, 'crawler/qq/audio/abc.mp3', bucket, BASE_URL)
result, _ = transfer_url_with_md5(OTHER_URL, 'crawler/audio/qq/abc.mp3', bucket, BASE_URL)
assert mock_get.call_count == 2
mock_sleep.assert_called_once()
assert result == f"{BASE_URL}/crawler/qq/audio/abc.mp3"
assert result == f"{BASE_URL}/crawler/audio/qq/abc.mp3"
def test_transfer_does_not_retry_not_found_response():
......@@ -98,7 +98,7 @@ def test_transfer_does_not_retry_not_found_response():
with patch('etl_to_crawler.oss._http_get', side_effect=error) as mock_get:
with patch('etl_to_crawler.oss.time.sleep') as mock_sleep:
try:
transfer_url_with_md5(OTHER_URL, 'crawler/qq/audio/abc.mp3', bucket, BASE_URL)
transfer_url_with_md5(OTHER_URL, 'crawler/audio/qq/abc.mp3', bucket, BASE_URL)
except requests.HTTPError:
pass
else:
......
......@@ -522,5 +522,5 @@ def test_process_netease_keeps_timestamped_lyric_and_uploads_plain_lyric(monkeyp
assert inserted_songs[0]['lyric'] == '[ti:歌名]\n[00:01.00]第一句\n[00:02.00]第二句'
assert inserted_songs[0]['audio_md5'] == 'audio-md5'
assert inserted_songs[0]['lyric_url'] == 'https://bucket.example.com/crawler/netease/lyric/300.txt'
assert inserted_songs[0]['lyric_url'] == 'https://bucket.example.com/crawler/lyric/netease/300.txt'
assert uploaded['body'].decode('utf-8') == '第一句\n第二句'
......