Skip to content
Toggle navigation
Toggle navigation
This project
Loading...
Sign in
沈秋雨
/
import_hk_songs
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Network
Create a new issue
Builds
Commits
Issue Boards
Files
Commits
Network
Compare
Branches
Tags
Commit
37eefbba
...
37eefbba91b99fed15872cd465ce39f5713598cd
authored
2026-07-10 13:18:50 +0800
by
沈秋雨
Browse Files
Options
Browse Files
Tag
Download
Email Patches
Plain Diff
refactor(crawler): 调整oss路径结构以统一目录层级
- 修改路径顺序,将平台(platform)调整至类别(category)之后
1 parent
7ef2362f
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
15 additions
and
15 deletions
etl_to_crawler/oss.py
etl_to_crawler/utils.py
tests/test_oss.py
tests/test_runner.py
etl_to_crawler/oss.py
View file @
37eefbb
...
...
@@ -108,4 +108,4 @@ def build_oss_key(platform: str, category: str, filename: str) -> str:
category: 'audio' | 'cover' | 'singer' | 'album'
filename: 带扩展名的文件名
"""
return
f
"crawler/{
platform}/{category
}/{filename}"
return
f
"crawler/{
category}/{platform
}/{filename}"
...
...
etl_to_crawler/utils.py
View file @
37eefbb
...
...
@@ -113,7 +113,7 @@ def upload_plain_lyric_to_bucket(platform: str, unique_id: str, lyric: str, buck
plain_lyric
=
extract_plain_lyric
(
lyric
)
if
not
plain_lyric
:
return
""
oss_key
=
f
"crawler/
{platform}/lyric
/{unique_id}.txt"
oss_key
=
f
"crawler/
lyric/{platform}
/{unique_id}.txt"
bucket
.
put_object
(
oss_key
,
plain_lyric
.
encode
(
"utf-8"
),
headers
=
{
'Content-Type'
:
'text/plain; charset=utf-8'
})
return
f
"{base_url.rstrip('/')}/{oss_key}"
...
...
tests/test_oss.py
View file @
37eefbb
...
...
@@ -42,9 +42,9 @@ def test_external_url_downloads_and_uploads():
with
patch
(
'etl_to_crawler.oss._http_get'
)
as
mock_get
:
mock_get
.
return_value
.
content
=
fake_content
mock_get
.
return_value
.
raise_for_status
=
MagicMock
()
result
=
transfer_url
(
OTHER_URL
,
"crawler/
qq/audio
/abc.mp3"
,
bucket
,
BASE_URL
)
bucket
.
put_object
.
assert_called_once_with
(
"crawler/
qq/audio
/abc.mp3"
,
fake_content
)
assert
result
==
f
"{BASE_URL}/crawler/
qq/audio
/abc.mp3"
result
=
transfer_url
(
OTHER_URL
,
"crawler/
audio/qq
/abc.mp3"
,
bucket
,
BASE_URL
)
bucket
.
put_object
.
assert_called_once_with
(
"crawler/
audio/qq
/abc.mp3"
,
fake_content
)
assert
result
==
f
"{BASE_URL}/crawler/
audio/qq
/abc.mp3"
def
test_external_url_can_rewrite_download_base_to_internal_endpoint
():
...
...
@@ -59,11 +59,11 @@ def test_external_url_can_rewrite_download_base_to_internal_endpoint():
with
patch
(
'etl_to_crawler.oss._http_get'
)
as
mock_get
:
mock_get
.
return_value
.
content
=
fake_content
mock_get
.
return_value
.
raise_for_status
=
MagicMock
()
result
=
transfer_url
(
public_source
,
"crawler/
qq/audio
/abc.mp3"
,
bucket
,
BASE_URL
)
result
=
transfer_url
(
public_source
,
"crawler/
audio/qq
/abc.mp3"
,
bucket
,
BASE_URL
)
mock_get
.
assert_called_once_with
(
f
"{internal_source_base}/audio/abc.mp3"
,
timeout
=
30
)
bucket
.
put_object
.
assert_called_once_with
(
"crawler/
qq/audio
/abc.mp3"
,
fake_content
)
assert
result
==
f
"{BASE_URL}/crawler/
qq/audio
/abc.mp3"
bucket
.
put_object
.
assert_called_once_with
(
"crawler/
audio/qq
/abc.mp3"
,
fake_content
)
assert
result
==
f
"{BASE_URL}/crawler/
audio/qq
/abc.mp3"
def
test_transfer_url_with_md5_hashes_downloaded_content_before_upload
():
...
...
@@ -72,9 +72,9 @@ def test_transfer_url_with_md5_hashes_downloaded_content_before_upload():
with
patch
(
'etl_to_crawler.oss._http_get'
)
as
mock_get
:
mock_get
.
return_value
.
content
=
fake_content
mock_get
.
return_value
.
raise_for_status
=
MagicMock
()
result
,
audio_md5
=
transfer_url_with_md5
(
OTHER_URL
,
"crawler/
qq/audio
/abc.mp3"
,
bucket
,
BASE_URL
)
bucket
.
put_object
.
assert_called_once_with
(
"crawler/
qq/audio
/abc.mp3"
,
fake_content
)
assert
result
==
f
"{BASE_URL}/crawler/
qq/audio
/abc.mp3"
result
,
audio_md5
=
transfer_url_with_md5
(
OTHER_URL
,
"crawler/
audio/qq
/abc.mp3"
,
bucket
,
BASE_URL
)
bucket
.
put_object
.
assert_called_once_with
(
"crawler/
audio/qq
/abc.mp3"
,
fake_content
)
assert
result
==
f
"{BASE_URL}/crawler/
audio/qq
/abc.mp3"
assert
audio_md5
==
"04d43544b267629d9089eaed3b847a99"
...
...
@@ -84,11 +84,11 @@ def test_audio_transfer_retries_timeout_then_uploads():
response
.
raise_for_status
.
return_value
=
None
with
patch
(
'etl_to_crawler.oss._http_get'
,
side_effect
=
[
requests
.
ReadTimeout
(
'slow'
),
response
])
as
mock_get
:
with
patch
(
'etl_to_crawler.oss.time.sleep'
)
as
mock_sleep
:
result
,
_
=
transfer_url_with_md5
(
OTHER_URL
,
'crawler/
qq/audio
/abc.mp3'
,
bucket
,
BASE_URL
)
result
,
_
=
transfer_url_with_md5
(
OTHER_URL
,
'crawler/
audio/qq
/abc.mp3'
,
bucket
,
BASE_URL
)
assert
mock_get
.
call_count
==
2
mock_sleep
.
assert_called_once
()
assert
result
==
f
"{BASE_URL}/crawler/
qq/audio
/abc.mp3"
assert
result
==
f
"{BASE_URL}/crawler/
audio/qq
/abc.mp3"
def
test_transfer_does_not_retry_not_found_response
():
...
...
@@ -98,7 +98,7 @@ def test_transfer_does_not_retry_not_found_response():
with
patch
(
'etl_to_crawler.oss._http_get'
,
side_effect
=
error
)
as
mock_get
:
with
patch
(
'etl_to_crawler.oss.time.sleep'
)
as
mock_sleep
:
try
:
transfer_url_with_md5
(
OTHER_URL
,
'crawler/
qq/audio
/abc.mp3'
,
bucket
,
BASE_URL
)
transfer_url_with_md5
(
OTHER_URL
,
'crawler/
audio/qq
/abc.mp3'
,
bucket
,
BASE_URL
)
except
requests
.
HTTPError
:
pass
else
:
...
...
tests/test_runner.py
View file @
37eefbb
...
...
@@ -522,5 +522,5 @@ def test_process_netease_keeps_timestamped_lyric_and_uploads_plain_lyric(monkeyp
assert
inserted_songs
[
0
][
'lyric'
]
==
'[ti:歌名]
\n
[00:01.00]第一句
\n
[00:02.00]第二句'
assert
inserted_songs
[
0
][
'audio_md5'
]
==
'audio-md5'
assert
inserted_songs
[
0
][
'lyric_url'
]
==
'https://bucket.example.com/crawler/
netease/lyric
/300.txt'
assert
inserted_songs
[
0
][
'lyric_url'
]
==
'https://bucket.example.com/crawler/
lyric/netease
/300.txt'
assert
uploaded
[
'body'
]
.
decode
(
'utf-8'
)
==
'第一句
\n
第二句'
...
...
Please
register
or
sign in
to post a comment