Commit 0e50b4c2 0e50b4c2a0f2a2f2b7973396f65c797f1c1038f5 by 沈秋雨

feat(etl): duration fallback to song_time,platform_index_url 缺失时按平台规则构造

1 parent bc4f610a
......@@ -5,7 +5,7 @@ from .config import PLATFORMS
_HK_SONGS_QUERY = """
SELECT id, name, lyricist, composer, audio_url, lyrics_url,
cover_url, singer, issue_time, source_song_id
cover_url, singer, issue_time, source_song_id, song_time
FROM hk_songs_test
WHERE deleted = '0'
AND name IS NOT NULL AND name != ''
......
......@@ -105,13 +105,13 @@ def _process_qq(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url):
'cover': cover_url,
'title': sp.get('title', hk_row['name']),
'name': hk_row['name'],
'duration': sp.get('duration', 0) or 0,
'duration': sp.get('duration') or hk_row.get('song_time') or 0,
'lyric': strip_timestamps(sp.get('lyric')),
'composer_name': sp.get('composer_name') or hk_row.get('composer'),
'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
'url': audio_url,
'lyric_url': hk_row.get('lyrics_url'),
'platform_index_url': sp.get('platform_index_url'),
'platform_index_url': sp.get('platform_index_url') or f'https://y.qq.com/n/ryqq/songDetail/{mid}',
'published_at': sp.get('published_at') or hk_row.get('issue_time'),
'singers_json': singers_json,
}])
......@@ -193,13 +193,13 @@ def _process_kugou(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url
'cover': cover_url,
'title': sp.get('title', hk_row['name']),
'name': hk_row['name'],
'duration': sp.get('duration', 0) or 0,
'duration': sp.get('duration') or hk_row.get('song_time') or 0,
'lyric': strip_timestamps(sp.get('lyric')),
'composer_name': sp.get('composer_name') or hk_row.get('composer'),
'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
'url': audio_url,
'lyric_url': hk_row.get('lyrics_url'),
'platform_index_url': sp.get('platform_index_url'),
'platform_index_url': sp.get('platform_index_url') or f'http://www.kugou.com/song/#hash={sp.get("hid") or pr.get("platform_mid", "")}',
'published_at': sp.get('published_at') or hk_row.get('issue_time'),
'singers_json': singers_json,
}])
......@@ -278,13 +278,13 @@ def _process_netease(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_u
'cover': cover_url,
'title': sp.get('title', hk_row['name']),
'name': hk_row['name'],
'duration': sp.get('duration', 0) or 0,
'duration': sp.get('duration') or hk_row.get('song_time') or 0,
'lyric': strip_timestamps(sp.get('lyric')),
'composer_name': sp.get('composer_name') or hk_row.get('composer'),
'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
'url': audio_url,
'lyric_url': hk_row.get('lyrics_url'),
'platform_index_url': sp.get('platform_index_url'),
'platform_index_url': sp.get('platform_index_url') or f'https://music.163.com/#/song?id={song_id}',
'published_at': sp.get('published_at') or hk_row.get('issue_time'),
'singers_json': singers_json,
}])
......