Commit e2246e03 e2246e03d4c1ae183c1ffce91e6866e32b491b7b by 沈秋雨

feat(etl): 实现 OSS URL 转移工具

1 parent d17b5206
1 import requests
2 import oss2
3 from urllib.parse import urlparse
4
5 ARCHIVE_DEV_HOST = 'archive-dev.oss-cn-beijing.aliyuncs.com'
6
7
8 def transfer_url(url: str | None, oss_key: str, bucket: oss2.Bucket, base_url: str) -> str:
9 """
10 将 url 指向的文件转移到 archive-dev OSS 的 oss_key 路径。
11 若 url 为空或已在 archive-dev,直接返回原 url(不上传)。
12 返回新的 archive-dev URL。
13 """
14 if not url:
15 return ''
16 if ARCHIVE_DEV_HOST in url:
17 return url
18 resp = requests.get(url, timeout=30)
19 resp.raise_for_status()
20 bucket.put_object(oss_key, resp.content)
21 return f"{base_url.rstrip('/')}/{oss_key}"
22
23
24 def build_oss_key(platform: str, category: str, filename: str) -> str:
25 """
26 构造 archive-dev 内的存储路径。
27 platform: 'qq' | 'kugou' | 'netease'
28 category: 'audio' | 'cover' | 'singer' | 'album'
29 filename: 带扩展名的文件名
30 """
31 return f"crawler/{platform}/{category}/{filename}"
1 from unittest.mock import MagicMock, patch
2 from etl_to_crawler.oss import transfer_url
3
4 ARCHIVE_URL = "https://archive-dev.oss-cn-beijing.aliyuncs.com/some/path.mp3"
5 OTHER_URL = "https://hikoon-data-platform.oss-cn-beijing.aliyuncs.com/qq-audio/abc.mp3"
6 BASE_URL = "https://archive-dev.oss-cn-beijing.aliyuncs.com"
7
8 def test_already_archive_dev_returns_as_is():
9 bucket = MagicMock()
10 result = transfer_url(ARCHIVE_URL, "any/key.mp3", bucket, BASE_URL)
11 assert result == ARCHIVE_URL
12 bucket.put_object.assert_not_called()
13
14 def test_empty_url_returns_empty():
15 bucket = MagicMock()
16 result = transfer_url('', "any/key.mp3", bucket, BASE_URL)
17 assert result == ''
18 bucket.put_object.assert_not_called()
19
20 def test_none_url_returns_empty():
21 bucket = MagicMock()
22 result = transfer_url(None, "any/key.mp3", bucket, BASE_URL)
23 assert result == ''
24
25 def test_external_url_downloads_and_uploads():
26 bucket = MagicMock()
27 fake_content = b"audio_bytes"
28 with patch('etl_to_crawler.oss.requests.get') as mock_get:
29 mock_get.return_value.content = fake_content
30 mock_get.return_value.raise_for_status = MagicMock()
31 result = transfer_url(OTHER_URL, "crawler/qq/audio/abc.mp3", bucket, BASE_URL)
32 bucket.put_object.assert_called_once_with("crawler/qq/audio/abc.mp3", fake_content)
33 assert result == f"{BASE_URL}/crawler/qq/audio/abc.mp3"