feat(etl_to_crawler): 在歌曲记录处理中添加平台字段
- 在runner.py中将platform字段添加至初始数据字典 - 在writer.py中修改SQL语句以插入platform信息 - 更新批量插入参数,包含platform字段 - 保持原有冲突处理逻辑不变,防止重复插入
Showing
2 changed files
with
4 additions
and
3 deletions
| ... | @@ -453,6 +453,7 @@ def initialize_yinyan_song_records(platforms: list[str], max_batches: int | None | ... | @@ -453,6 +453,7 @@ def initialize_yinyan_song_records(platforms: list[str], max_batches: int | None |
| 453 | init_rows.append({ | 453 | init_rows.append({ |
| 454 | 'song_id': src_id, | 454 | 'song_id': src_id, |
| 455 | 'record_id': int(primary_record['record_id']), | 455 | 'record_id': int(primary_record['record_id']), |
| 456 | 'platform': primary_record['platform'], | ||
| 456 | }) | 457 | }) |
| 457 | 458 | ||
| 458 | if init_rows: | 459 | if init_rows: | ... | ... |
| ... | @@ -7,11 +7,11 @@ def insert_yinyan_song_records(cur, records: list[dict]) -> None: | ... | @@ -7,11 +7,11 @@ def insert_yinyan_song_records(cur, records: list[dict]) -> None: |
| 7 | return | 7 | return |
| 8 | cur.executemany( | 8 | cur.executemany( |
| 9 | """ | 9 | """ |
| 10 | INSERT INTO yinyan_song_records (song_id, record_id, is_yinyan_push) | 10 | INSERT INTO yinyan_song_records (song_id, record_id, platform, is_yinyan_push) |
| 11 | VALUES (%s, %s, FALSE) | 11 | VALUES (%s, %s, %s, FALSE) |
| 12 | ON CONFLICT (song_id, record_id) DO NOTHING | 12 | ON CONFLICT (song_id, record_id) DO NOTHING |
| 13 | """, | 13 | """, |
| 14 | [(r['song_id'], r['record_id']) for r in records], | 14 | [(r['song_id'], r['record_id'], r['platform']) for r in records], |
| 15 | ) | 15 | ) |
| 16 | 16 | ||
| 17 | 17 | ... | ... |
-
Please register or sign in to post a comment