Skip to content
Toggle navigation
Toggle navigation
This project
Loading...
Sign in
沈秋雨
/
import_hk_songs
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Network
Create a new issue
Builds
Commits
Issue Boards
Files
Commits
Network
Compare
Branches
Tags
Commit
0e50b4c2
...
0e50b4c2a0f2a2f2b7973396f65c797f1c1038f5
authored
2026-07-08 23:08:00 +0800
by
沈秋雨
Browse Files
Options
Browse Files
Tag
Download
Email Patches
Plain Diff
feat(etl): duration fallback to song_time,platform_index_url 缺失时按平台规则构造
1 parent
bc4f610a
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
7 additions
and
7 deletions
etl_to_crawler/reader.py
etl_to_crawler/runner.py
etl_to_crawler/reader.py
View file @
0e50b4c
...
...
@@ -5,7 +5,7 @@ from .config import PLATFORMS
_HK_SONGS_QUERY
=
"""
SELECT id, name, lyricist, composer, audio_url, lyrics_url,
cover_url, singer, issue_time, source_song_id
cover_url, singer, issue_time, source_song_id
, song_time
FROM hk_songs_test
WHERE deleted = '0'
AND name IS NOT NULL AND name != ''
...
...
etl_to_crawler/runner.py
View file @
0e50b4c
...
...
@@ -105,13 +105,13 @@ def _process_qq(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url):
'cover'
:
cover_url
,
'title'
:
sp
.
get
(
'title'
,
hk_row
[
'name'
]),
'name'
:
hk_row
[
'name'
],
'duration'
:
sp
.
get
(
'duration'
,
0
)
or
0
,
'duration'
:
sp
.
get
(
'duration'
)
or
hk_row
.
get
(
'song_time'
)
or
0
,
'lyric'
:
strip_timestamps
(
sp
.
get
(
'lyric'
)),
'composer_name'
:
sp
.
get
(
'composer_name'
)
or
hk_row
.
get
(
'composer'
),
'lyricist_name'
:
sp
.
get
(
'lyricist_name'
)
or
hk_row
.
get
(
'lyricist'
),
'url'
:
audio_url
,
'lyric_url'
:
hk_row
.
get
(
'lyrics_url'
),
'platform_index_url'
:
sp
.
get
(
'platform_index_url'
),
'platform_index_url'
:
sp
.
get
(
'platform_index_url'
)
or
f
'https://y.qq.com/n/ryqq/songDetail/{mid}'
,
'published_at'
:
sp
.
get
(
'published_at'
)
or
hk_row
.
get
(
'issue_time'
),
'singers_json'
:
singers_json
,
}])
...
...
@@ -193,13 +193,13 @@ def _process_kugou(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url
'cover'
:
cover_url
,
'title'
:
sp
.
get
(
'title'
,
hk_row
[
'name'
]),
'name'
:
hk_row
[
'name'
],
'duration'
:
sp
.
get
(
'duration'
,
0
)
or
0
,
'duration'
:
sp
.
get
(
'duration'
)
or
hk_row
.
get
(
'song_time'
)
or
0
,
'lyric'
:
strip_timestamps
(
sp
.
get
(
'lyric'
)),
'composer_name'
:
sp
.
get
(
'composer_name'
)
or
hk_row
.
get
(
'composer'
),
'lyricist_name'
:
sp
.
get
(
'lyricist_name'
)
or
hk_row
.
get
(
'lyricist'
),
'url'
:
audio_url
,
'lyric_url'
:
hk_row
.
get
(
'lyrics_url'
),
'platform_index_url'
:
sp
.
get
(
'platform_index_url'
),
'platform_index_url'
:
sp
.
get
(
'platform_index_url'
)
or
f
'http://www.kugou.com/song/#hash={sp.get("hid") or pr.get("platform_mid", "")}'
,
'published_at'
:
sp
.
get
(
'published_at'
)
or
hk_row
.
get
(
'issue_time'
),
'singers_json'
:
singers_json
,
}])
...
...
@@ -278,13 +278,13 @@ def _process_netease(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_u
'cover'
:
cover_url
,
'title'
:
sp
.
get
(
'title'
,
hk_row
[
'name'
]),
'name'
:
hk_row
[
'name'
],
'duration'
:
sp
.
get
(
'duration'
,
0
)
or
0
,
'duration'
:
sp
.
get
(
'duration'
)
or
hk_row
.
get
(
'song_time'
)
or
0
,
'lyric'
:
strip_timestamps
(
sp
.
get
(
'lyric'
)),
'composer_name'
:
sp
.
get
(
'composer_name'
)
or
hk_row
.
get
(
'composer'
),
'lyricist_name'
:
sp
.
get
(
'lyricist_name'
)
or
hk_row
.
get
(
'lyricist'
),
'url'
:
audio_url
,
'lyric_url'
:
hk_row
.
get
(
'lyrics_url'
),
'platform_index_url'
:
sp
.
get
(
'platform_index_url'
),
'platform_index_url'
:
sp
.
get
(
'platform_index_url'
)
or
f
'https://music.163.com/#/song?id={song_id}'
,
'published_at'
:
sp
.
get
(
'published_at'
)
or
hk_row
.
get
(
'issue_time'
),
'singers_json'
:
singers_json
,
}])
...
...
Please
register
or
sign in
to post a comment