Skip to content

沈秋雨 / import_hk_songs

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 0e50b4c2 0e50b4c2a0f2a2f2b7973396f65c797f1c1038f5 by 沈秋雨
Options

feat(etl): duration fallback to song_time，platform_index_url 缺失时按平台规则构造

1 parent bc4f610a
Inline Side-by-side
Showing 2 changed files with 7 additions and 7 deletions
...@@ -5,7 +5,7 @@ from .config import PLATFORMS ...@@ -5,7 +5,7 @@ from .config import PLATFORMS
5 5
6 _HK_SONGS_QUERY = """ 6 _HK_SONGS_QUERY = """
7 SELECT id, name, lyricist, composer, audio_url, lyrics_url, 7 SELECT id, name, lyricist, composer, audio_url, lyrics_url,
8 cover_url, singer, issue_time, source_song_id 8 cover_url, singer, issue_time, source_song_id, song_time
9 FROM hk_songs_test 9 FROM hk_songs_test
10 WHERE deleted = '0' 10 WHERE deleted = '0'
11 AND name IS NOT NULL AND name != '' 11 AND name IS NOT NULL AND name != ''
......
...@@ -105,13 +105,13 @@ def _process_qq(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url): ...@@ -105,13 +105,13 @@ def _process_qq(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url):
105 'cover': cover_url, 105 'cover': cover_url,
106 'title': sp.get('title', hk_row['name']), 106 'title': sp.get('title', hk_row['name']),
107 'name': hk_row['name'], 107 'name': hk_row['name'],
108 'duration': sp.get('duration', 0) or 0, 108 'duration': sp.get('duration') or hk_row.get('song_time') or 0,
109 'lyric': strip_timestamps(sp.get('lyric')), 109 'lyric': strip_timestamps(sp.get('lyric')),
110 'composer_name': sp.get('composer_name') or hk_row.get('composer'), 110 'composer_name': sp.get('composer_name') or hk_row.get('composer'),
111 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), 111 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
112 'url': audio_url, 112 'url': audio_url,
113 'lyric_url': hk_row.get('lyrics_url'), 113 'lyric_url': hk_row.get('lyrics_url'),
114 'platform_index_url': sp.get('platform_index_url'), 114 'platform_index_url': sp.get('platform_index_url') or f'https://y.qq.com/n/ryqq/songDetail/{mid}',
115 'published_at': sp.get('published_at') or hk_row.get('issue_time'), 115 'published_at': sp.get('published_at') or hk_row.get('issue_time'),
116 'singers_json': singers_json, 116 'singers_json': singers_json,
117 }]) 117 }])
...@@ -193,13 +193,13 @@ def _process_kugou(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url ...@@ -193,13 +193,13 @@ def _process_kugou(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url
193 'cover': cover_url, 193 'cover': cover_url,
194 'title': sp.get('title', hk_row['name']), 194 'title': sp.get('title', hk_row['name']),
195 'name': hk_row['name'], 195 'name': hk_row['name'],
196 'duration': sp.get('duration', 0) or 0, 196 'duration': sp.get('duration') or hk_row.get('song_time') or 0,
197 'lyric': strip_timestamps(sp.get('lyric')), 197 'lyric': strip_timestamps(sp.get('lyric')),
198 'composer_name': sp.get('composer_name') or hk_row.get('composer'), 198 'composer_name': sp.get('composer_name') or hk_row.get('composer'),
199 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), 199 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
200 'url': audio_url, 200 'url': audio_url,
201 'lyric_url': hk_row.get('lyrics_url'), 201 'lyric_url': hk_row.get('lyrics_url'),
202 'platform_index_url': sp.get('platform_index_url'), 202 'platform_index_url': sp.get('platform_index_url') or f'http://www.kugou.com/song/#hash={sp.get("hid") or pr.get("platform_mid", "")}',
203 'published_at': sp.get('published_at') or hk_row.get('issue_time'), 203 'published_at': sp.get('published_at') or hk_row.get('issue_time'),
204 'singers_json': singers_json, 204 'singers_json': singers_json,
205 }]) 205 }])
...@@ -278,13 +278,13 @@ def _process_netease(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_u ...@@ -278,13 +278,13 @@ def _process_netease(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_u
278 'cover': cover_url, 278 'cover': cover_url,
279 'title': sp.get('title', hk_row['name']), 279 'title': sp.get('title', hk_row['name']),
280 'name': hk_row['name'], 280 'name': hk_row['name'],
281 'duration': sp.get('duration', 0) or 0, 281 'duration': sp.get('duration') or hk_row.get('song_time') or 0,
282 'lyric': strip_timestamps(sp.get('lyric')), 282 'lyric': strip_timestamps(sp.get('lyric')),
283 'composer_name': sp.get('composer_name') or hk_row.get('composer'), 283 'composer_name': sp.get('composer_name') or hk_row.get('composer'),
284 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'), 284 'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
285 'url': audio_url, 285 'url': audio_url,
286 'lyric_url': hk_row.get('lyrics_url'), 286 'lyric_url': hk_row.get('lyrics_url'),
287 'platform_index_url': sp.get('platform_index_url'), 287 'platform_index_url': sp.get('platform_index_url') or f'https://music.163.com/#/song?id={song_id}',
288 'published_at': sp.get('published_at') or hk_row.get('issue_time'), 288 'published_at': sp.get('published_at') or hk_row.get('issue_time'),
289 'singers_json': singers_json, 289 'singers_json': singers_json,
290 }]) 290 }])
......