Skip to content
Toggle navigation
Toggle navigation
This project
Loading...
Sign in
沈秋雨
/
import_hk_songs
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Network
Create a new issue
Builds
Commits
Issue Boards
Files
Commits
Network
Compare
Branches
Tags
Commit
bc4f610a
...
bc4f610a0326dd65c093f4e0b858cec9a3fbd744
authored
2026-07-08 17:17:12 +0800
by
沈秋雨
Browse Files
Options
Browse Files
Tag
Download
Email Patches
Plain Diff
feat(etl): runner 运行后打印导入记录列表
1 parent
62fc7695
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
17 additions
and
1 deletions
etl_to_crawler/runner.py
etl_to_crawler/runner.py
View file @
bc4f610
...
...
@@ -125,6 +125,7 @@ def _process_qq(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url):
upsert_qq_singer_songs
(
pg_cur
,
[(
sg
[
'singer_id'
],
song_uuid
)
for
sg
in
singer_list
])
if
album_id
:
upsert_qq_singer_albums
(
pg_cur
,
[(
sg
[
'singer_id'
],
album_id
)
for
sg
in
singer_list
])
return
{
'platform'
:
'qq'
,
'platform_song_id'
:
platform_song_id
,
'mid'
:
mid
,
'title'
:
hk_row
[
'name'
]}
def
_process_kugou
(
hk_row
:
dict
,
pr
:
dict
,
spider_conn
,
pg_cur
,
bucket
,
base_url
):
...
...
@@ -211,6 +212,7 @@ def _process_kugou(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url
upsert_kugou_singer_songs
(
pg_cur
,
[(
sg
[
'singer_id'
],
song_uuid
)
for
sg
in
singer_list
])
if
album_id
:
upsert_kugou_singer_albums
(
pg_cur
,
[(
sg
[
'singer_id'
],
album_id
)
for
sg
in
singer_list
])
return
{
'platform'
:
'kugou'
,
'platform_song_id'
:
song_id
,
'hash'
:
sp
.
get
(
'hid'
,
''
),
'title'
:
hk_row
[
'name'
]}
def
_process_netease
(
hk_row
:
dict
,
pr
:
dict
,
spider_conn
,
pg_cur
,
bucket
,
base_url
):
...
...
@@ -295,6 +297,7 @@ def _process_netease(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_u
upsert_netease_singer_songs
(
pg_cur
,
[(
sg
[
'singer_id'
],
song_uuid
)
for
sg
in
singer_list
])
if
album_id
:
upsert_netease_singer_albums
(
pg_cur
,
[(
sg
[
'singer_id'
],
album_id
)
for
sg
in
singer_list
])
return
{
'platform'
:
'netease'
,
'platform_song_id'
:
song_id
,
'title'
:
hk_row
[
'name'
]}
_PROCESSORS
=
{
...
...
@@ -313,6 +316,7 @@ def run(platforms: list[str], max_batches: int | None = None) -> None:
base_url
=
OSS_CONFIG
[
'base_url'
]
total_ok
=
total_err
=
0
imported
:
list
[
dict
]
=
[]
try
:
for
i
,
batch
in
enumerate
(
tqdm
(
iter_hk_songs_batches
(
hk_conn
,
BATCH_SIZE
),
desc
=
'batches'
)):
...
...
@@ -338,9 +342,11 @@ def run(platforms: list[str], max_batches: int | None = None) -> None:
continue
pg_cur
.
execute
(
'SAVEPOINT sp_song'
)
try
:
processor
(
hk_row
,
pr
,
spider_conn
,
pg_cur
,
bucket
,
base_url
)
result
=
processor
(
hk_row
,
pr
,
spider_conn
,
pg_cur
,
bucket
,
base_url
)
pg_cur
.
execute
(
'RELEASE SAVEPOINT sp_song'
)
total_ok
+=
1
if
result
:
imported
.
append
(
result
)
except
Exception
as
e
:
pg_cur
.
execute
(
'ROLLBACK TO SAVEPOINT sp_song'
)
pg_cur
.
execute
(
'RELEASE SAVEPOINT sp_song'
)
...
...
@@ -356,3 +362,13 @@ def run(platforms: list[str], max_batches: int | None = None) -> None:
pg_conn
.
close
()
log
.
info
(
"Done. OK=
%
d ERR=
%
d"
,
total_ok
,
total_err
)
if
imported
:
print
(
f
"
\n
导入记录(共 {len(imported)} 条):"
)
for
r
in
imported
:
p
=
r
[
'platform'
]
if
p
==
'qq'
:
print
(
f
" [QQ] platform_song_id={r['platform_song_id']} mid={r['mid']} title={r['title']}"
)
elif
p
==
'kugou'
:
print
(
f
" [酷狗] platform_song_id={r['platform_song_id']} hash={r['hash']} title={r['title']}"
)
elif
p
==
'netease'
:
print
(
f
" [网易] platform_song_id={r['platform_song_id']} title={r['title']}"
)
...
...
Please
register
or
sign in
to post a comment