Skip to content
Toggle navigation
Toggle navigation
This project
Loading...
Sign in
沈秋雨
/
import_hk_songs
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Network
Create a new issue
Builds
Commits
Issue Boards
Files
Commits
Network
Compare
Branches
Tags
Commit
d260999d
...
d260999daac14f044e1a33ca964f0eaef6b50843
authored
2026-07-08 15:55:44 +0800
by
沈秋雨
Browse Files
Options
Browse Files
Tag
Download
Email Patches
Plain Diff
feat(etl): 实现 hk_songs_test 读取与 hikoon-data 关联
1 parent
e2246e03
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
66 additions
and
0 deletions
etl_to_crawler/reader.py
etl_to_crawler/reader.py
0 → 100644
View file @
d260999
from
typing
import
Iterator
import
pymysql
from
.config
import
PLATFORMS
_HK_SONGS_QUERY
=
"""
SELECT id, name, lyricist, composer, audio_url, lyrics_url,
cover_url, singer, issue_time, source_song_id
FROM hk_songs_test
WHERE deleted = '0'
AND name IS NOT NULL AND name != ''
AND audio_url IS NOT NULL AND audio_url != ''
AND singer IS NOT NULL AND singer != ''
ORDER BY id
LIMIT
%
s OFFSET
%
s
"""
_PLATFORM_QUERY
=
"""
SELECT
sar.song_id AS source_song_id,
mr.platform,
mr.platform_unique_key,
mr.platform_mid,
mr.album_audio_id,
sar.is_main_version
FROM hk_song_and_record sar
JOIN hk_music_record mr ON mr.id = sar.record_id
WHERE sar.song_id IN ({placeholders})
AND mr.platform IN ('1','2','4')
AND mr.platform_unique_key IS NOT NULL
AND mr.platform_unique_key != ''
AND mr.deleted = 0
ORDER BY sar.song_id, mr.platform, sar.is_main_version DESC
"""
def
iter_hk_songs_batches
(
conn
:
pymysql
.
Connection
,
batch_size
:
int
)
->
Iterator
[
list
[
dict
]]:
offset
=
0
with
conn
.
cursor
()
as
cur
:
while
True
:
cur
.
execute
(
_HK_SONGS_QUERY
,
(
batch_size
,
offset
))
rows
=
cur
.
fetchall
()
if
not
rows
:
break
yield
rows
if
len
(
rows
)
<
batch_size
:
break
offset
+=
batch_size
def
fetch_platform_records
(
source_conn
:
pymysql
.
Connection
,
song_ids
:
list
[
int
])
->
list
[
dict
]:
if
not
song_ids
:
return
[]
placeholders
=
','
.
join
([
'
%
s'
]
*
len
(
song_ids
))
query
=
_PLATFORM_QUERY
.
format
(
placeholders
=
placeholders
)
with
source_conn
.
cursor
()
as
cur
:
cur
.
execute
(
query
,
song_ids
)
rows
=
cur
.
fetchall
()
# 每个 (source_song_id, platform) 只保留 is_main_version 最高的一条
seen
=
{}
for
row
in
rows
:
key
=
(
row
[
'source_song_id'
],
row
[
'platform'
])
if
key
not
in
seen
:
seen
[
key
]
=
row
return
list
(
seen
.
values
())
Please
register
or
sign in
to post a comment