feat(etl): 实现 OSS URL 转移工具
Showing
2 changed files
with
64 additions
and
0 deletions
etl_to_crawler/oss.py
0 → 100644
| 1 | import requests | ||
| 2 | import oss2 | ||
| 3 | from urllib.parse import urlparse | ||
| 4 | |||
| 5 | ARCHIVE_DEV_HOST = 'archive-dev.oss-cn-beijing.aliyuncs.com' | ||
| 6 | |||
| 7 | |||
| 8 | def transfer_url(url: str | None, oss_key: str, bucket: oss2.Bucket, base_url: str) -> str: | ||
| 9 | """ | ||
| 10 | 将 url 指向的文件转移到 archive-dev OSS 的 oss_key 路径。 | ||
| 11 | 若 url 为空或已在 archive-dev,直接返回原 url(不上传)。 | ||
| 12 | 返回新的 archive-dev URL。 | ||
| 13 | """ | ||
| 14 | if not url: | ||
| 15 | return '' | ||
| 16 | if ARCHIVE_DEV_HOST in url: | ||
| 17 | return url | ||
| 18 | resp = requests.get(url, timeout=30) | ||
| 19 | resp.raise_for_status() | ||
| 20 | bucket.put_object(oss_key, resp.content) | ||
| 21 | return f"{base_url.rstrip('/')}/{oss_key}" | ||
| 22 | |||
| 23 | |||
| 24 | def build_oss_key(platform: str, category: str, filename: str) -> str: | ||
| 25 | """ | ||
| 26 | 构造 archive-dev 内的存储路径。 | ||
| 27 | platform: 'qq' | 'kugou' | 'netease' | ||
| 28 | category: 'audio' | 'cover' | 'singer' | 'album' | ||
| 29 | filename: 带扩展名的文件名 | ||
| 30 | """ | ||
| 31 | return f"crawler/{platform}/{category}/{filename}" |
tests/test_oss.py
0 → 100644
| 1 | from unittest.mock import MagicMock, patch | ||
| 2 | from etl_to_crawler.oss import transfer_url | ||
| 3 | |||
| 4 | ARCHIVE_URL = "https://archive-dev.oss-cn-beijing.aliyuncs.com/some/path.mp3" | ||
| 5 | OTHER_URL = "https://hikoon-data-platform.oss-cn-beijing.aliyuncs.com/qq-audio/abc.mp3" | ||
| 6 | BASE_URL = "https://archive-dev.oss-cn-beijing.aliyuncs.com" | ||
| 7 | |||
| 8 | def test_already_archive_dev_returns_as_is(): | ||
| 9 | bucket = MagicMock() | ||
| 10 | result = transfer_url(ARCHIVE_URL, "any/key.mp3", bucket, BASE_URL) | ||
| 11 | assert result == ARCHIVE_URL | ||
| 12 | bucket.put_object.assert_not_called() | ||
| 13 | |||
| 14 | def test_empty_url_returns_empty(): | ||
| 15 | bucket = MagicMock() | ||
| 16 | result = transfer_url('', "any/key.mp3", bucket, BASE_URL) | ||
| 17 | assert result == '' | ||
| 18 | bucket.put_object.assert_not_called() | ||
| 19 | |||
| 20 | def test_none_url_returns_empty(): | ||
| 21 | bucket = MagicMock() | ||
| 22 | result = transfer_url(None, "any/key.mp3", bucket, BASE_URL) | ||
| 23 | assert result == '' | ||
| 24 | |||
| 25 | def test_external_url_downloads_and_uploads(): | ||
| 26 | bucket = MagicMock() | ||
| 27 | fake_content = b"audio_bytes" | ||
| 28 | with patch('etl_to_crawler.oss.requests.get') as mock_get: | ||
| 29 | mock_get.return_value.content = fake_content | ||
| 30 | mock_get.return_value.raise_for_status = MagicMock() | ||
| 31 | result = transfer_url(OTHER_URL, "crawler/qq/audio/abc.mp3", bucket, BASE_URL) | ||
| 32 | bucket.put_object.assert_called_once_with("crawler/qq/audio/abc.mp3", fake_content) | ||
| 33 | assert result == f"{BASE_URL}/crawler/qq/audio/abc.mp3" |
-
Please register or sign in to post a comment