Skip to content

沈秋雨 / import_hk_songs

Go to a project
Toggle navigation
Toggle navigation pinning
Commit dc82bc10 dc82bc1084c4892bfcaccd24155b4c04b2acc8e5 by 沈秋雨
Options

refactor(etl): 优化音频转存与歌词上传流程，支持音频 MD5 计算和歌词换行处理 

- 移除 audit_hk_songs_duplicates.py 中删除零时长歌曲的逻辑
- 新增 utils.py 工具模块，提供音频 MD5 计算、歌词处理、OSS 上传等函数
- etl_to_crawler/oss.py 添加 transfer_url_with_md5 函数，用于下载音频、计算 MD5 并上传 OSS
- etl_to_crawler/lyric.py 增加 ensure_newlines 函数，确保歌词内容行间带换行符
- etl_to_crawler/runner.py 改进音频转存调用，返回音频 MD5 值
- 通过 safe_upload_lyric 函数上传去时间戳的纯文本歌词，失败时使用原始歌词 URL
- etl_to_crawler/writer.py 在歌曲数据写入中新增 audio_md5 字段支持
- 重构 yinyan_song_records 逻辑，新增初始化函数 initialize_yinyan_song_records
- run_etl.py 增加 --init-yinyan-records 命令行参数支持仅初始化状态表
- 优化 ETL 运行流程，从预标记状态表中拉取待处理歌曲逐批处理，提高效率
- 新增相关单元测试以覆盖音频 MD5 计算和任务处理逻辑修改
1 parent 8bd8c02b
Expand all
Inline Side-by-side
Showing 11 changed files with 555 additions and 111 deletions
...@@ -558,52 +558,9 @@ def main() -> int: ...@@ -558,52 +558,9 @@ def main() -> int:
558 help="补处理暂存表中 merge+skipped 记录：existing_into_new 入库+软删旧记录，new_into_existing 补作者合并", 558 help="补处理暂存表中 merge+skipped 记录：existing_into_new 入库+软删旧记录，new_into_existing 补作者合并",
559 ) 559 )
560 parser.add_argument("--skip-oss", action="store_true", help="--fix-merge 时跳过歌词 OSS 上传，保留原始文本") 560 parser.add_argument("--skip-oss", action="store_true", help="--fix-merge 时跳过歌词 OSS 上传，保留原始文本")
561 parser.add_argument("--dry-run", action="store_true", help="仅打印计划，不写库（配合 --fix-merge / --delete-zero-duration 使用）") 561 parser.add_argument("--dry-run", action="store_true", help="仅打印计划，不写库（配合 --fix-merge 使用）")
562 parser.add_argument(
563 "--delete-zero-duration",
564 action="store_true",
565 help="软删除 song_time=0 的记录（置 deleted='1'）",
566 )
567 args = parser.parse_args() 562 args = parser.parse_args()
568 563
569 # --delete-zero-duration 模式
570 if args.delete_zero_duration:
571 conn = pymysql.connect(**TARGET_DB_CONFIG)
572 try:
573 table = quote_identifier(DEFAULT_TARGET_TABLE)
574 with conn.cursor() as cursor:
575 cursor.execute(
576 f"SELECT id, name, lyricist, composer, song_time "
577 f"FROM {table} WHERE deleted = '0' AND song_time = 0"
578 )
579 rows = list(cursor.fetchall())
580 print(f"song_time=0 记录数: {len(rows)}")
581 if args.dry_run:
582 for row in rows[:20]:
583 print(row)
584 if len(rows) > 20:
585 print(f"... {len(rows) - 20} more rows")
586 return 0
587 if not rows:
588 print("无需处理")
589 return 0
590 ids = [row["id"] for row in rows]
591 placeholders = ",".join(["%s"] * len(ids))
592 with conn.cursor() as cursor:
593 cursor.execute(
594 f"UPDATE {table} "
595 f"SET deleted = '1', modify_time = NOW(), "
596 f" off_shelf_remark = 'song_time=0 soft-deleted by audit script' "
597 f"WHERE deleted = '0' AND id IN ({placeholders})",
598 ids,
599 )
600 affected = cursor.rowcount
601 conn.commit()
602 print(f"soft_deleted={affected}")
603 finally:
604 conn.close()
605 return 0
606
607 # --fix-merge 模式：连双库，执行合并方向后处理 564 # --fix-merge 模式：连双库，执行合并方向后处理
608 if args.fix_merge: 565 if args.fix_merge:
609 target_conn = pymysql.connect(**TARGET_DB_CONFIG) 566 target_conn = pymysql.connect(**TARGET_DB_CONFIG)
......
...@@ -4,6 +4,16 @@ _TIMESTAMP_RE = re.compile(r'\[\d{2}:\d{2}\.\d{2,3}\]') ...@@ -4,6 +4,16 @@ _TIMESTAMP_RE = re.compile(r'\[\d{2}:\d{2}\.\d{2,3}\]')
4 _META_TAG_RE = re.compile(r'\[[a-zA-Z]+:[^\]]*\]') 4 _META_TAG_RE = re.compile(r'\[[a-zA-Z]+:[^\]]*\]')
5 5
6 6
7 def ensure_newlines(text: str | None) -> str:
8 """确保歌词每行之间有换行符，保留时间戳等原始内容"""
9 if not text:
10 return ''
11 # 将字面量 \\n / \\r 转为真正换行符
12 text = text.replace('\\n', '\n').replace('\\r', '')
13 lines = [line.strip() for line in text.splitlines() if line.strip()]
14 return '\n'.join(lines)
15
16
7 def strip_timestamps(text: str | None) -> str: 17 def strip_timestamps(text: str | None) -> str:
8 if not text: 18 if not text:
9 return '' 19 return ''
......
1 import requests 1 import requests
2 import oss2 2 import oss2
3 from urllib.parse import urlparse 3 from urllib.parse import urlparse
4 from .utils import compute_audio_md5
4 5
5 ARCHIVE_DEV_HOST = 'archive-dev.oss-cn-beijing.aliyuncs.com' 6 ARCHIVE_DEV_HOST = 'archive-dev.oss-cn-beijing.aliyuncs.com'
6 7
...@@ -21,6 +22,22 @@ def transfer_url(url: str | None, oss_key: str, bucket: oss2.Bucket, base_url: s ...@@ -21,6 +22,22 @@ def transfer_url(url: str | None, oss_key: str, bucket: oss2.Bucket, base_url: s
21 return f"{base_url.rstrip('/')}/{oss_key}" 22 return f"{base_url.rstrip('/')}/{oss_key}"
22 23
23 24
25 def transfer_url_with_md5(url: str | None, oss_key: str, bucket: oss2.Bucket, base_url: str) -> tuple[str, str]:
26 """
27 将音频 URL 转存到 OSS，并基于下载到的音频字节计算 MD5。
28 已在目标 OSS 的 URL 无需重新下载，无法可靠计算 MD5，返回空 MD5。
29 """
30 if not url:
31 return '', ''
32 if ARCHIVE_DEV_HOST in url:
33 return url, ''
34 resp = requests.get(url, timeout=30)
35 resp.raise_for_status()
36 audio_md5 = compute_audio_md5(resp.content)
37 bucket.put_object(oss_key, resp.content)
38 return f"{base_url.rstrip('/')}/{oss_key}", audio_md5
39
40
24 def build_oss_key(platform: str, category: str, filename: str) -> str: 41 def build_oss_key(platform: str, category: str, filename: str) -> str:
25 """ 42 """
26 构造 archive-dev 内的存储路径。 43 构造 archive-dev 内的存储路径。
......
...@@ -16,6 +16,18 @@ ORDER BY id ...@@ -16,6 +16,18 @@ ORDER BY id
16 LIMIT %s 16 LIMIT %s
17 """ 17 """
18 18
19 _HK_SONGS_BY_SOURCE_IDS_QUERY = """
20 SELECT id, name, lyricist, composer, audio_url, lyrics_url,
21 cover_url, singer, issue_time, source_song_id, song_time
22 FROM hk_songs_test
23 WHERE deleted = '0'
24 AND source_song_id IN ({placeholders})
25 AND name IS NOT NULL AND name != ''
26 AND audio_url IS NOT NULL AND audio_url != ''
27 AND singer IS NOT NULL AND singer != ''
28 ORDER BY id
29 """
30
19 _PLATFORM_QUERY = """ 31 _PLATFORM_QUERY = """
20 SELECT 32 SELECT
21 sar.song_id AS source_song_id, 33 sar.song_id AS source_song_id,
...@@ -61,6 +73,21 @@ def iter_hk_songs_batches( ...@@ -61,6 +73,21 @@ def iter_hk_songs_batches(
61 break 73 break
62 74
63 75
76 def fetch_hk_songs_by_source_ids(conn: pymysql.Connection, source_song_ids: list[int]) -> dict[int, dict]:
77 if not source_song_ids:
78 return {}
79 placeholders = ','.join(['%s'] * len(source_song_ids))
80 query = _HK_SONGS_BY_SOURCE_IDS_QUERY.format(placeholders=placeholders)
81 with conn.cursor() as cur:
82 cur.execute(query, source_song_ids)
83 rows = cur.fetchall()
84 return {
85 int(row['source_song_id']): row
86 for row in rows
87 if row.get('source_song_id') is not None
88 }
89
90
64 def fetch_platform_records(source_conn: pymysql.Connection, song_ids: list[int]) -> list[dict]: 91 def fetch_platform_records(source_conn: pymysql.Connection, song_ids: list[int]) -> list[dict]:
65 if not song_ids: 92 if not song_ids:
66 return [] 93 return []
......
1 """工具函数模块"""
2 import asyncio
3 import hashlib
4 import logging
5 import re
6 from datetime import datetime
7 from urllib.parse import urlparse
8 from zoneinfo import ZoneInfo
9
10 try:
11 import httpx
12 except ModuleNotFoundError:
13 httpx = None
14
15 try:
16 from app.core.config import settings
17 from app.core.oss_client import oss_client
18 except ModuleNotFoundError:
19 settings = None
20 oss_client = None
21
22 CHINA_TZ = ZoneInfo("Asia/Shanghai")
23 logger = logging.getLogger(__name__)
24
25 # OSS 上传默认超时（秒），优先使用全局配置
26 _OSS_UPLOAD_TIMEOUT = settings.OSS_UPLOAD_TIMEOUT_SECONDS if settings else 30
27 # 音频下载默认超时（秒），优先使用全局配置
28 _AUDIO_DOWNLOAD_TIMEOUT = settings.AUDIO_DOWNLOAD_TIMEOUT_SECONDS if settings else 30
29 # 音频下载最大限制（字节），优先使用全局配置
30 _MAX_AUDIO_SIZE = settings.MAX_AUDIO_DOWNLOAD_SIZE if settings else 50 * 1024 * 1024
31
32 # 歌词中常见的词曲作者匹配规则
33 _LYRICIST_PATTERNS = [
34 r"作\s*词\s*[:：]\s*([^\r\n/／]+)",
35 r"词\s*[:：]\s*([^\r\n/／]+)",
36 r"Lyrics?\s*[:：]\s*([^\r\n/／]+)",
37 ]
38 _COMPOSER_PATTERNS = [
39 r"作\s*曲\s*[:：]\s*([^\r\n/／]+)",
40 r"曲\s*[:：]\s*([^\r\n/／]+)",
41 r"Composer\s*[:：]\s*([^\r\n/／]+)",
42 ]
43
44
45 def _search_with_patterns(patterns: list, text: str) -> str:
46 """根据多个正则匹配文本，返回首个命中分组"""
47 if not text:
48 return ""
49 for pattern in patterns:
50 match = re.search(pattern, text)
51 if match:
52 return match.group(1).strip()
53 return ""
54
55
56 def extract_lyricist_composer(lyric: str) -> tuple[str, str]:
57 """从歌词文本中提取词作者和曲作者
58
59 Returns:
60 (lyricist_name, composer_name)
61 """
62 lyricist = _search_with_patterns(_LYRICIST_PATTERNS, lyric)
63 composer = _search_with_patterns(_COMPOSER_PATTERNS, lyric)
64 return lyricist, composer
65
66
67 def now_cn() -> datetime:
68 """返回中国时区的当前时间（无时区信息），保留微秒"""
69 return datetime.now(CHINA_TZ).replace(tzinfo=None)
70
71
72 def extract_plain_lyric(lyric: str) -> str:
73 """去除 LRC 歌词中的时间戳和标签，保留纯文本
74
75 Args:
76 lyric: 原始歌词内容（可能含 [mm:ss.xx] 时间戳和 [ti:xxx] 等标签）
77
78 Returns:
79 纯文本歌词，行之间用换行符连接
80 """
81 if not lyric:
82 return ""
83 lines = []
84 for line in lyric.splitlines():
85 # 去除 LRC 时间戳 [mm:ss.xx] 或 [mm:ss.xxx]
86 cleaned = re.sub(r"\[\d{2}:\d{2}(?:\.\d{2,3})?\]", "", line)
87 # 去除标签 [xx:yy]
88 cleaned = re.sub(r"\[[a-zA-Z]+:[^\]]+\]", "", cleaned)
89 cleaned = cleaned.strip()
90 if cleaned:
91 lines.append(cleaned)
92 return "\n".join(lines)
93
94
95 def upload_plain_lyric_to_bucket(platform: str, unique_id: str, lyric: str, bucket, base_url: str) -> str:
96 """将歌词去时间戳后上传到当前 ETL 使用的 OSS bucket"""
97 plain_lyric = extract_plain_lyric(lyric)
98 if not plain_lyric:
99 return ""
100 oss_key = f"crawler/{platform}/lyric/{unique_id}.txt"
101 bucket.put_object(oss_key, plain_lyric.encode("utf-8"))
102 return f"{base_url.rstrip('/')}/{oss_key}"
103
104
105 async def upload_lyric_to_oss(platform: str, unique_id: str, lyric: str) -> str:
106 """将歌词去除时间戳后上传为纯文本文件到 OSS
107
108 Args:
109 platform: 平台标识，如 qqmusic/kugou/kuwo/netease/migu
110 unique_id: 平台唯一标识，如 song_mid/hash/rid 等
111 lyric: 原始歌词内容
112
113 Returns:
114 OSS 文件访问 URL，上传失败或歌词为空返回空字符串
115 """
116 if not lyric:
117 return ""
118 if oss_client is None:
119 logger.error("OSS client is not configured")
120 return ""
121 plain_lyric = extract_plain_lyric(lyric)
122 if not plain_lyric:
123 return ""
124 oss_key = f"lyrics/{platform}/{unique_id}.txt"
125 try:
126 file_url = await asyncio.wait_for(
127 asyncio.to_thread(
128 oss_client.upload_bytes,
129 plain_lyric.encode("utf-8"),
130 oss_key,
131 "public-read",
132 "text/plain; charset=utf-8",
133 ),
134 timeout=_OSS_UPLOAD_TIMEOUT,
135 )
136 return file_url or ""
137 except asyncio.TimeoutError:
138 logger.exception(f"上传歌词到 OSS 超时 {_OSS_UPLOAD_TIMEOUT}s {platform}/{unique_id}")
139 return ""
140 except Exception as e:
141 logger.exception(f"上传歌词到 OSS 失败 {platform}/{unique_id}: {e}")
142 return ""
143
144
145 async def download_and_upload_audio(
146 audio_url: str,
147 oss_key: str,
148 *,
149 headers: dict | None = None,
150 allow_external_oss_url: bool = False,
151 ) -> tuple[str, str]:
152 """流式下载音频并上传到 OSS
153
154 内置大小限制、下载超时、上传超时保护，防止大文件或慢网络拖垮 worker。
155
156 Args:
157 audio_url: 音频下载地址
158 oss_key: OSS 对象键，如 songs/qqmusic/xxxx.mp3
159 headers: 可选的下载请求头
160 allow_external_oss_url: 若 audio_url 已是当前 OSS 域名下的地址，是否直接透传
161
162 Returns:
163 (OSS 文件 URL, 音频 MD5)，失败返回 ("", "")
164 """
165 if not audio_url:
166 return "", ""
167
168 if allow_external_oss_url and settings.OSS_FILE_BASE_NAME and audio_url.startswith(settings.OSS_FILE_BASE_NAME):
169 return audio_url, ""
170
171 audio_bytes = bytearray()
172 try:
173 async with httpx.AsyncClient(timeout=_AUDIO_DOWNLOAD_TIMEOUT) as client:
174 async with client.stream("GET", audio_url, headers=headers or {}) as response:
175 response.raise_for_status()
176 content_length = response.headers.get("Content-Length")
177 if content_length and int(content_length) > _MAX_AUDIO_SIZE:
178 logger.warning(
179 f"音频文件过大，跳过: {oss_key}, "
180 f"size={int(content_length) / 1024 / 1024:.2f}MB"
181 )
182 return "", ""
183 async for chunk in response.aiter_bytes(chunk_size=64 * 1024):
184 audio_bytes.extend(chunk)
185 if len(audio_bytes) > _MAX_AUDIO_SIZE:
186 logger.warning(
187 f"音频下载超过大小限制，跳过: {oss_key}, "
188 f"size>{_MAX_AUDIO_SIZE / 1024 / 1024:.2f}MB"
189 )
190 return "", ""
191 except Exception as e:
192 logger.error(f"下载音频失败 {oss_key}: {e}")
193 return "", ""
194
195 audio_bytes = bytes(audio_bytes)
196 audio_md5 = compute_audio_md5(audio_bytes)
197 try:
198 file_url = await asyncio.wait_for(
199 asyncio.to_thread(oss_client.upload_bytes, audio_bytes, oss_key),
200 timeout=_OSS_UPLOAD_TIMEOUT,
201 )
202 except asyncio.TimeoutError:
203 logger.error(f"上传音频到 OSS 超时 {_OSS_UPLOAD_TIMEOUT}s: {oss_key}")
204 return "", ""
205 except Exception as e:
206 logger.error(f"上传音频到 OSS 失败 {oss_key}: {e}")
207 return "", ""
208 return file_url or "", audio_md5
209
210
211 _CONTENT_TYPE_EXT_MAP = {
212 "image/jpeg": "jpg",
213 "image/jpg": "jpg",
214 "image/png": "png",
215 "image/webp": "webp",
216 "image/gif": "gif",
217 }
218
219
220 def _guess_image_ext(content_type: str, url: str) -> str:
221 """根据 Content-Type 或 URL 路径推断图片扩展名，默认 jpg"""
222 if content_type:
223 mime = content_type.split(";")[0].strip().lower()
224 ext = _CONTENT_TYPE_EXT_MAP.get(mime)
225 if ext:
226 return ext
227 path = urlparse(url).path
228 suffix = path.rsplit(".", 1)[-1].lower()
229 if suffix in {"jpg", "jpeg", "png", "webp", "gif"}:
230 return "jpg" if suffix == "jpeg" else suffix
231 return "jpg"
232
233
234 async def upload_cover_to_oss(platform: str, unique_id: str, image_url: str) -> str:
235 """下载封面图片并上传到 OSS
236
237 Args:
238 platform: 平台标识，如 qqmusic/kugou/kuwo/netease/migu
239 unique_id: 平台唯一标识，如 song_mid/hash/rid 等
240 image_url: 封面图片原始 URL
241
242 Returns:
243 OSS 文件访问 URL，上传失败或 URL 为空返回空字符串
244 """
245 if not image_url:
246 return ""
247 try:
248 async with httpx.AsyncClient(timeout=10, follow_redirects=True) as client:
249 resp = await client.get(image_url, headers={
250 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
251 "Referer": "https://www.kugou.com/",
252 })
253 resp.raise_for_status()
254 image_bytes = resp.content
255 content_type = resp.headers.get("content-type", "")
256 except Exception as e:
257 logger.exception(f"下载封面图片失败 {platform}/{unique_id} {image_url}: {e}")
258 return ""
259
260 ext = _guess_image_ext(content_type, image_url)
261 oss_key = f"covers/{platform}/{unique_id}.{ext}"
262 mime = _CONTENT_TYPE_EXT_MAP.get(content_type.split(";")[0].strip().lower(), f"image/{ext}")
263 try:
264 file_url = await asyncio.wait_for(
265 asyncio.to_thread(
266 oss_client.upload_bytes,
267 image_bytes,
268 oss_key,
269 "public-read",
270 mime,
271 ),
272 timeout=_OSS_UPLOAD_TIMEOUT,
273 )
274 return file_url or ""
275 except asyncio.TimeoutError:
276 logger.exception(f"上传封面到 OSS 超时 {_OSS_UPLOAD_TIMEOUT}s {platform}/{unique_id}")
277 return ""
278 except Exception as e:
279 logger.exception(f"上传封面到 OSS 失败 {platform}/{unique_id}: {e}")
280 return ""
281
282
283 def compute_audio_md5(audio_bytes: bytes) -> str:
284 """计算音频字节流的 MD5 值（32 位小写十六进制字符串）
285
286 Args:
287 audio_bytes: 音频文件字节流
288
289 Returns:
290 MD5 字符串，输入为空时返回空字符串
291 """
292 if not audio_bytes:
293 return ""
294 return hashlib.md5(audio_bytes).hexdigest()
1 import uuid 1 import uuid
2 2
3 3
4 def upsert_yinyan_song_records(cur, pairs: list[tuple[int, int]]) -> None: 4 def insert_yinyan_song_records(cur, records: list[dict]) -> None:
5 """pairs: [(source_song_id, hk_music_record_id), ...]""" 5 """Initialize yinyan_song_records rows before crawler import."""
6 if not pairs: 6 if not records:
7 return 7 return
8 cur.executemany( 8 cur.executemany(
9 "DELETE FROM yinyan_song_records WHERE song_id = %s", 9 """
10 [(song_id,) for song_id, _ in pairs], 10 INSERT INTO yinyan_song_records (song_id, record_id, is_yinyan_push)
11 VALUES (%s, %s, FALSE)
12 ON CONFLICT (song_id, record_id) DO NOTHING
13 """,
14 [(r['song_id'], r['record_id']) for r in records],
11 ) 15 )
16
17
18 def fetch_pending_yinyan_song_records(cur, limit: int) -> list[dict]:
19 cur.execute(
20 """
21 SELECT song_id, record_id
22 FROM yinyan_song_records
23 WHERE is_yinyan_push = FALSE
24 ORDER BY song_id
25 LIMIT %s
26 """,
27 (limit,),
28 )
29 rows = cur.fetchall()
30 return [{'song_id': row[0], 'record_id': row[1]} for row in rows]
31
32
33 def upsert_yinyan_song_records(cur, records: list[dict]) -> None:
34 """Mark pre-initialized yinyan song-record rows as pushed to crawler."""
35 if not records:
36 return
12 cur.executemany( 37 cur.executemany(
13 """ 38 """
14 INSERT INTO yinyan_song_records (song_id, record_id) 39 UPDATE yinyan_song_records
15 VALUES (%s, %s) 40 SET platform = %s,
41 platform_song_id = %s,
42 is_yinyan_push = TRUE
43 WHERE song_id = %s AND record_id = %s
16 """, 44 """,
17 pairs, 45 [
46 (r['platform'], r['platform_song_id'], r['song_id'], r['record_id'])
47 for r in records
48 ],
18 ) 49 )
19 50
20 51
...@@ -67,8 +98,8 @@ def upsert_qq_songs(cur, songs: list[dict]) -> None: ...@@ -67,8 +98,8 @@ def upsert_qq_songs(cur, songs: list[dict]) -> None:
67 (id, platform_song_id, mid, album_id, cover, title, name, duration, 98 (id, platform_song_id, mid, album_id, cover, title, name, duration,
68 lyric, composer_name, lyricist_name, url, lyric_url, 99 lyric, composer_name, lyricist_name, url, lyric_url,
69 platform_index_url, published_at, singers, status, created_at, updated_at, 100 platform_index_url, published_at, singers, status, created_at, updated_at,
70 provider_name, crawler_source_data) 101 audio_md5, provider_name, crawler_source_data)
71 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s::json) 102 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s::json)
72 ON CONFLICT (platform_song_id) DO NOTHING 103 ON CONFLICT (platform_song_id) DO NOTHING
73 """ 104 """
74 rows = [( 105 rows = [(
...@@ -77,6 +108,7 @@ def upsert_qq_songs(cur, songs: list[dict]) -> None: ...@@ -77,6 +108,7 @@ def upsert_qq_songs(cur, songs: list[dict]) -> None:
77 s.get('lyric'), s.get('composer_name'), s.get('lyricist_name'), 108 s.get('lyric'), s.get('composer_name'), s.get('lyricist_name'),
78 s.get('url', ''), s.get('lyric_url'), 109 s.get('url', ''), s.get('lyric_url'),
79 s.get('platform_index_url'), s.get('published_at'), s.get('singers_json', '[]'), 110 s.get('platform_index_url'), s.get('published_at'), s.get('singers_json', '[]'),
111 s.get('audio_md5'),
80 s.get('provider_name'), s.get('crawler_source_data'), 112 s.get('provider_name'), s.get('crawler_source_data'),
81 ) for s in songs] 113 ) for s in songs]
82 cur.executemany(sql, rows) 114 cur.executemany(sql, rows)
...@@ -157,8 +189,8 @@ def upsert_kugou_songs(cur, songs: list[dict]) -> None: ...@@ -157,8 +189,8 @@ def upsert_kugou_songs(cur, songs: list[dict]) -> None:
157 (id, platform_song_id, hash, album_audio_id, album_id, cover, title, name, duration, 189 (id, platform_song_id, hash, album_audio_id, album_id, cover, title, name, duration,
158 lyric, composer_name, lyricist_name, url, lyric_url, 190 lyric, composer_name, lyricist_name, url, lyric_url,
159 platform_index_url, published_at, singers, status, created_at, updated_at, 191 platform_index_url, published_at, singers, status, created_at, updated_at,
160 provider_name, crawler_source_data) 192 audio_md5, provider_name, crawler_source_data)
161 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s::json) 193 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s::json)
162 ON CONFLICT (platform_song_id) DO NOTHING 194 ON CONFLICT (platform_song_id) DO NOTHING
163 """ 195 """
164 rows = [( 196 rows = [(
...@@ -168,6 +200,7 @@ def upsert_kugou_songs(cur, songs: list[dict]) -> None: ...@@ -168,6 +200,7 @@ def upsert_kugou_songs(cur, songs: list[dict]) -> None:
168 s.get('lyric'), s.get('composer_name'), s.get('lyricist_name'), 200 s.get('lyric'), s.get('composer_name'), s.get('lyricist_name'),
169 s.get('url', ''), s.get('lyric_url'), 201 s.get('url', ''), s.get('lyric_url'),
170 s.get('platform_index_url'), s.get('published_at'), s.get('singers_json', '[]'), 202 s.get('platform_index_url'), s.get('published_at'), s.get('singers_json', '[]'),
203 s.get('audio_md5'),
171 s.get('provider_name'), s.get('crawler_source_data'), 204 s.get('provider_name'), s.get('crawler_source_data'),
172 ) for s in songs] 205 ) for s in songs]
173 cur.executemany(sql, rows) 206 cur.executemany(sql, rows)
...@@ -244,8 +277,8 @@ def upsert_netease_songs(cur, songs: list[dict]) -> None: ...@@ -244,8 +277,8 @@ def upsert_netease_songs(cur, songs: list[dict]) -> None:
244 (id, platform_song_id, album_id, cover, title, name, duration, 277 (id, platform_song_id, album_id, cover, title, name, duration,
245 lyric, composer_name, lyricist_name, url, lyric_url, 278 lyric, composer_name, lyricist_name, url, lyric_url,
246 platform_index_url, published_at, album, singers, status, created_at, updated_at, 279 platform_index_url, published_at, album, singers, status, created_at, updated_at,
247 provider_name, crawler_source_data) 280 audio_md5, provider_name, crawler_source_data)
248 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s::json) 281 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s::json)
249 ON CONFLICT (platform_song_id) DO NOTHING 282 ON CONFLICT (platform_song_id) DO NOTHING
250 """ 283 """
251 rows = [( 284 rows = [(
...@@ -254,6 +287,7 @@ def upsert_netease_songs(cur, songs: list[dict]) -> None: ...@@ -254,6 +287,7 @@ def upsert_netease_songs(cur, songs: list[dict]) -> None:
254 s.get('lyric'), s.get('composer_name'), s.get('lyricist_name'), 287 s.get('lyric'), s.get('composer_name'), s.get('lyricist_name'),
255 s.get('url', ''), s.get('lyric_url'), 288 s.get('url', ''), s.get('lyric_url'),
256 s.get('platform_index_url'), s.get('published_at'), s.get('album_json'), s.get('singers_json', '[]'), 289 s.get('platform_index_url'), s.get('published_at'), s.get('album_json'), s.get('singers_json', '[]'),
290 s.get('audio_md5'),
257 s.get('provider_name'), s.get('crawler_source_data'), 291 s.get('provider_name'), s.get('crawler_source_data'),
258 ) for s in songs] 292 ) for s in songs]
259 cur.executemany(sql, rows) 293 cur.executemany(sql, rows)
......
1 #!/usr/bin/env python3 1 #!/usr/bin/env python3
2 import argparse 2 import argparse
3 from etl_to_crawler.config import PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE, PLATFORMS 3 from etl_to_crawler.config import PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE, PLATFORMS
4 from etl_to_crawler.runner import run 4 from etl_to_crawler.runner import initialize_yinyan_song_records, run
5 5
6 PLATFORM_MAP = { 6 PLATFORM_MAP = {
7 'qq': PLATFORM_QQ, 7 'qq': PLATFORM_QQ,
...@@ -16,14 +16,8 @@ if __name__ == '__main__': ...@@ -16,14 +16,8 @@ if __name__ == '__main__':
16 help='要导入的平台（默认 all）') 16 help='要导入的平台（默认 all）')
17 parser.add_argument('--max-batches', type=int, default=None, 17 parser.add_argument('--max-batches', type=int, default=None,
18 help='最多处理多少批次（冒烟测试用）') 18 help='最多处理多少批次（冒烟测试用）')
19 parser.add_argument('--resume', action='store_true', 19 parser.add_argument('--init-yinyan-records', action='store_true',
20 help='启用断点续传，按 hk_songs_test.id 从上次成功提交的批次后继续') 20 help='只初始化 yinyan_song_records 待导入状态表，不执行 crawler 导入')
21 parser.add_argument('--state-file', default='output/etl_to_crawler_state.json',
22 help='断点状态文件路径')
23 parser.add_argument('--start-after-id', type=int, default=None,
24 help='手动指定从哪个 hk_songs_test.id 之后开始读取')
25 parser.add_argument('--reset-state', action='store_true',
26 help='忽略已有断点状态，从头或 --start-after-id 指定位置重新开始')
27 args = parser.parse_args() 21 args = parser.parse_args()
28 22
29 if args.platform == 'all': 23 if args.platform == 'all':
...@@ -32,12 +26,7 @@ if __name__ == '__main__': ...@@ -32,12 +26,7 @@ if __name__ == '__main__':
32 platforms = [PLATFORM_MAP[args.platform]] 26 platforms = [PLATFORM_MAP[args.platform]]
33 27
34 print(f"Starting ETL for platforms: {platforms}") 28 print(f"Starting ETL for platforms: {platforms}")
35 run( 29 if args.init_yinyan_records:
36 platforms, 30 initialize_yinyan_song_records(platforms, max_batches=args.max_batches)
37 max_batches=args.max_batches, 31 else:
38 resume=args.resume, 32 run(platforms, max_batches=args.max_batches)
39 state_file=args.state_file,
40 state_key=args.platform,
41 start_after_id=args.start_after_id,
42 reset_state=args.reset_state,
43 )
......
1 from unittest.mock import MagicMock, patch 1 from unittest.mock import MagicMock, patch
2 from etl_to_crawler.oss import transfer_url 2 from etl_to_crawler.oss import transfer_url, transfer_url_with_md5
3 3
4 ARCHIVE_URL = "https://archive-dev.oss-cn-beijing.aliyuncs.com/some/path.mp3" 4 ARCHIVE_URL = "https://archive-dev.oss-cn-beijing.aliyuncs.com/some/path.mp3"
5 OTHER_URL = "https://hikoon-data-platform.oss-cn-beijing.aliyuncs.com/qq-audio/abc.mp3" 5 OTHER_URL = "https://hikoon-data-platform.oss-cn-beijing.aliyuncs.com/qq-audio/abc.mp3"
...@@ -31,3 +31,15 @@ def test_external_url_downloads_and_uploads(): ...@@ -31,3 +31,15 @@ def test_external_url_downloads_and_uploads():
31 result = transfer_url(OTHER_URL, "crawler/qq/audio/abc.mp3", bucket, BASE_URL) 31 result = transfer_url(OTHER_URL, "crawler/qq/audio/abc.mp3", bucket, BASE_URL)
32 bucket.put_object.assert_called_once_with("crawler/qq/audio/abc.mp3", fake_content) 32 bucket.put_object.assert_called_once_with("crawler/qq/audio/abc.mp3", fake_content)
33 assert result == f"{BASE_URL}/crawler/qq/audio/abc.mp3" 33 assert result == f"{BASE_URL}/crawler/qq/audio/abc.mp3"
34
35
36 def test_transfer_url_with_md5_hashes_downloaded_content_before_upload():
37 bucket = MagicMock()
38 fake_content = b"audio_bytes"
39 with patch('etl_to_crawler.oss.requests.get') as mock_get:
40 mock_get.return_value.content = fake_content
41 mock_get.return_value.raise_for_status = MagicMock()
42 result, audio_md5 = transfer_url_with_md5(OTHER_URL, "crawler/qq/audio/abc.mp3", bucket, BASE_URL)
43 bucket.put_object.assert_called_once_with("crawler/qq/audio/abc.mp3", fake_content)
44 assert result == f"{BASE_URL}/crawler/qq/audio/abc.mp3"
45 assert audio_md5 == "04d43544b267629d9089eaed3b847a99"
......
...@@ -48,12 +48,15 @@ def test_run_imports_all_platform_records_but_writes_one_primary_yinyan_relation ...@@ -48,12 +48,15 @@ def test_run_imports_all_platform_records_but_writes_one_primary_yinyan_relation
48 monkeypatch.setattr(runner, 'get_spider_conn', lambda: _Connection()) 48 monkeypatch.setattr(runner, 'get_spider_conn', lambda: _Connection())
49 monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn) 49 monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn)
50 monkeypatch.setattr(runner, 'get_oss_bucket', lambda: object()) 50 monkeypatch.setattr(runner, 'get_oss_bucket', lambda: object())
51 monkeypatch.setattr(runner, 'iter_hk_songs_batches', lambda conn, batch_size, start_after_id=0: [[{ 51 monkeypatch.setattr(runner, 'fetch_pending_yinyan_song_records', lambda cur, batch_size: [
52 {'song_id': 10, 'record_id': 200},
53 ] if pg_conn.commits == 0 else [])
54 monkeypatch.setattr(runner, 'fetch_hk_songs_by_source_ids', lambda conn, song_ids: {10: {
52 'source_song_id': 10, 55 'source_song_id': 10,
53 'name': '歌', 56 'name': '歌',
54 'audio_url': 'https://example.com/a.mp3', 57 'audio_url': 'https://example.com/a.mp3',
55 'singer': '歌手', 58 'singer': '歌手',
56 }]]) 59 }})
57 monkeypatch.setattr(runner, 'fetch_platform_records', lambda conn, song_ids: [ 60 monkeypatch.setattr(runner, 'fetch_platform_records', lambda conn, song_ids: [
58 { 61 {
59 'source_song_id': 10, 62 'source_song_id': 10,
...@@ -85,37 +88,56 @@ def test_run_imports_all_platform_records_but_writes_one_primary_yinyan_relation ...@@ -85,37 +88,56 @@ def test_run_imports_all_platform_records_but_writes_one_primary_yinyan_relation
85 88
86 processors['1'].assert_called_once() 89 processors['1'].assert_called_once()
87 processors['2'].assert_called_once() 90 processors['2'].assert_called_once()
88 yinyan_writer.assert_called_once_with(pg_conn.cur, [(10, 200)]) 91 yinyan_writer.assert_called_once_with(pg_conn.cur, [{
92 'song_id': 10,
93 'record_id': 200,
94 'platform': 'kugou',
95 'platform_song_id': 200,
96 }])
89 assert pg_conn.commits == 1 97 assert pg_conn.commits == 1
90 98
91 99
92 def test_run_resume_starts_after_saved_id_and_updates_state_after_commit(monkeypatch, tmp_path): 100 def test_initialize_yinyan_song_records_inserts_primary_records(monkeypatch):
93 pg_conn = _PgConnection() 101 pg_conn = _PgConnection()
94 state_file = tmp_path / 'etl_state.json' 102 inserted = []
95 state_file.write_text('{"all": {"last_hk_songs_id": 40}}', encoding='utf-8')
96 seen_start_ids = []
97 103
98 monkeypatch.setattr(runner, 'get_hk_songs_conn', lambda: _Connection()) 104 monkeypatch.setattr(runner, 'get_hk_songs_conn', lambda: _Connection())
99 monkeypatch.setattr(runner, 'get_source_conn', lambda: _Connection()) 105 monkeypatch.setattr(runner, 'get_source_conn', lambda: _Connection())
100 monkeypatch.setattr(runner, 'get_spider_conn', lambda: _Connection())
101 monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn) 106 monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn)
102 monkeypatch.setattr(runner, 'get_oss_bucket', lambda: object())
103
104 def fake_batches(conn, batch_size, start_after_id=0):
105 seen_start_ids.append(start_after_id)
106 return iter([[
107 {'id': 50, 'source_song_id': 10, 'name': '歌', 'audio_url': 'https://example.com/a.mp3', 'singer': '歌手'},
108 {'id': 60, 'source_song_id': 20, 'name': '歌2', 'audio_url': 'https://example.com/b.mp3', 'singer': '歌手2'},
109 ]])
110 107
111 monkeypatch.setattr(runner, 'iter_hk_songs_batches', fake_batches) 108 monkeypatch.setattr(runner, 'iter_hk_songs_batches', lambda conn, batch_size: [[
112 monkeypatch.setattr(runner, 'fetch_platform_records', lambda conn, song_ids: []) 109 {'id': 50, 'source_song_id': 10, 'name': '歌', 'audio_url': 'https://example.com/a.mp3', 'singer': '歌手'},
110 ]])
111 monkeypatch.setattr(runner, 'fetch_platform_records', lambda conn, song_ids: [
112 {
113 'source_song_id': 10,
114 'record_id': 100,
115 'platform': '1',
116 'platform_unique_key': 'qq-mid',
117 'platform_mid': '100',
118 'album_audio_id': None,
119 'is_main_version': 0,
120 'is_high': 0,
121 'pub_time': '2020-01-01',
122 },
123 {
124 'source_song_id': 10,
125 'record_id': 200,
126 'platform': '2',
127 'platform_unique_key': '200',
128 'platform_mid': 'kg-hash',
129 'album_audio_id': None,
130 'is_main_version': 1,
131 'is_high': 0,
132 'pub_time': '2021-01-01',
133 },
134 ])
135 monkeypatch.setattr(runner, 'insert_yinyan_song_records', lambda cur, rows: inserted.extend(rows))
113 136
114 runner.run(['1', '2', '4'], resume=True, state_file=state_file, state_key='all') 137 runner.initialize_yinyan_song_records(['1', '2'])
115 138
116 assert seen_start_ids == [40] 139 assert inserted == [{'song_id': 10, 'record_id': 200}]
117 assert pg_conn.commits == 1 140 assert pg_conn.commits == 1
118 assert '"last_hk_songs_id": 60' in state_file.read_text(encoding='utf-8')
119 141
120 142
121 def test_process_netease_builds_album_json_for_song_insert(monkeypatch): 143 def test_process_netease_builds_album_json_for_song_insert(monkeypatch):
...@@ -147,6 +169,7 @@ def test_process_netease_builds_album_json_for_song_insert(monkeypatch): ...@@ -147,6 +169,7 @@ def test_process_netease_builds_album_json_for_song_insert(monkeypatch):
147 }) 169 })
148 monkeypatch.setattr(runner, 'fetch_netease_singers', lambda conn, song_ids: {}) 170 monkeypatch.setattr(runner, 'fetch_netease_singers', lambda conn, song_ids: {})
149 monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url) 171 monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url)
172 monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5'))
150 monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None) 173 monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None)
151 monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None) 174 monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None)
152 monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs)) 175 monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs))
...@@ -174,3 +197,60 @@ def test_process_netease_builds_album_json_for_song_insert(monkeypatch): ...@@ -174,3 +197,60 @@ def test_process_netease_builds_album_json_for_song_insert(monkeypatch):
174 assert inserted_songs[0]['album_json'] 197 assert inserted_songs[0]['album_json']
175 assert '"id": 20' in inserted_songs[0]['album_json'] 198 assert '"id": 20' in inserted_songs[0]['album_json']
176 assert '"title": "专辑"' in inserted_songs[0]['album_json'] 199 assert '"title": "专辑"' in inserted_songs[0]['album_json']
200
201
202 def test_process_netease_keeps_timestamped_lyric_and_uploads_plain_lyric(monkeypatch):
203 pg_cur = MagicMock()
204 pg_cur.fetchone.return_value = ('song-uuid',)
205 inserted_songs = []
206 uploaded = {}
207
208 class Bucket:
209 def put_object(self, key, body):
210 uploaded['key'] = key
211 uploaded['body'] = body
212
213 monkeypatch.setattr(runner, 'fetch_netease_songs', lambda conn, song_ids: {
214 300: {
215 'id': 300,
216 'album_id': None,
217 'cover': 'https://example.com/cover.jpg',
218 'title': '录音标题',
219 'duration': 180,
220 'lyric': '[ti:歌名]\n[00:01.00]第一句\n[00:02.00]第二句',
221 'composer_name': '曲作者',
222 'lyricist_name': '词作者',
223 'platform_index_url': None,
224 'published_at': '2020-01-02',
225 },
226 })
227 monkeypatch.setattr(runner, 'fetch_netease_singers', lambda conn, song_ids: {})
228 monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url)
229 monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5'))
230 monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None)
231 monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None)
232 monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs))
233 monkeypatch.setattr(runner, 'upsert_netease_singer_songs', lambda cur, pairs: None)
234
235 runner._process_netease(
236 {
237 'name': '词曲名',
238 'audio_url': 'https://example.com/audio.mp3',
239 'lyrics_url': 'https://example.com/original.lrc',
240 'cover_url': '',
241 'composer': '词曲曲作者',
242 'lyricist': '词曲词作者',
243 'issue_time': '2019-01-01',
244 'song_time': 120,
245 },
246 {'platform_unique_key': '300'},
247 spider_conn=object(),
248 pg_cur=pg_cur,
249 bucket=Bucket(),
250 base_url='https://bucket.example.com',
251 )
252
253 assert inserted_songs[0]['lyric'] == '[ti:歌名]\n[00:01.00]第一句\n[00:02.00]第二句'
254 assert inserted_songs[0]['audio_md5'] == 'audio-md5'
255 assert inserted_songs[0]['lyric_url'] == 'https://bucket.example.com/crawler/netease/lyric/300.txt'
256 assert uploaded['body'].decode('utf-8') == '第一句\n第二句'
......
1 from unittest.mock import MagicMock, call 1 from unittest.mock import MagicMock, call
2 from etl_to_crawler.writer import ( 2 from etl_to_crawler.writer import (
3 insert_yinyan_song_records,
3 upsert_kugou_albums, 4 upsert_kugou_albums,
4 upsert_kugou_singers, 5 upsert_kugou_singers,
5 upsert_kugou_songs, 6 upsert_kugou_songs,
...@@ -41,18 +42,35 @@ def test_upsert_qq_singer_songs(): ...@@ -41,18 +42,35 @@ def test_upsert_qq_singer_songs():
41 assert 'crawler_qqmusic_singer_songs' in sql 42 assert 'crawler_qqmusic_singer_songs' in sql
42 43
43 44
44 def test_upsert_yinyan_song_records_replaces_existing_song_relation(): 45 def test_upsert_yinyan_song_records_marks_existing_relation_as_pushed():
45 cur = MagicMock() 46 cur = MagicMock()
46 upsert_yinyan_song_records(cur, [(10, 100), (11, 101)]) 47 upsert_yinyan_song_records(cur, [
48 {'song_id': 10, 'record_id': 100, 'platform': 'qq', 'platform_song_id': 1000},
49 {'song_id': 11, 'record_id': 101, 'platform': 'kugou', 'platform_song_id': 1001},
50 ])
47 51
48 assert cur.executemany.call_count == 2 52 sql, rows = cur.executemany.call_args[0]
49 delete_sql, delete_rows = cur.executemany.call_args_list[0][0] 53 assert 'UPDATE yinyan_song_records' in sql
50 insert_sql, insert_rows = cur.executemany.call_args_list[1][0] 54 assert 'platform = %s' in sql
51 assert 'DELETE FROM yinyan_song_records' in delete_sql 55 assert 'platform_song_id = %s' in sql
52 assert 'WHERE song_id = %s' in delete_sql 56 assert 'is_yinyan_push = TRUE' in sql
53 assert delete_rows == [(10,), (11,)] 57 assert 'WHERE song_id = %s AND record_id = %s' in sql
54 assert 'INSERT INTO yinyan_song_records' in insert_sql 58 assert rows == [('qq', 1000, 10, 100), ('kugou', 1001, 11, 101)]
55 assert insert_rows == [(10, 100), (11, 101)] 59
60
61 def test_insert_yinyan_song_records_initializes_unpushed_rows():
62 cur = MagicMock()
63 insert_yinyan_song_records(cur, [
64 {'song_id': 10, 'record_id': 100},
65 {'song_id': 11, 'record_id': 101},
66 ])
67
68 sql, rows = cur.executemany.call_args[0]
69 assert 'INSERT INTO yinyan_song_records' in sql
70 assert 'is_yinyan_push' in sql
71 assert 'FALSE' in sql
72 assert 'ON CONFLICT (song_id, record_id) DO NOTHING' in sql
73 assert rows == [(10, 100), (11, 101)]
56 74
57 75
58 def test_upsert_netease_songs_writes_album_json_column(): 76 def test_upsert_netease_songs_writes_album_json_column():
...@@ -104,12 +122,14 @@ def test_upsert_kugou_songs_writes_provider_and_source_data(): ...@@ -104,12 +122,14 @@ def test_upsert_kugou_songs_writes_provider_and_source_data():
104 'singers_json': '[]', 122 'singers_json': '[]',
105 'provider_name': 'yinyan', 123 'provider_name': 'yinyan',
106 'crawler_source_data': '{"id": 200}', 124 'crawler_source_data': '{"id": 200}',
125 'audio_md5': 'md5-200',
107 }]) 126 }])
108 127
109 sql, rows = cur.executemany.call_args[0] 128 sql, rows = cur.executemany.call_args[0]
129 assert 'audio_md5' in sql
110 assert 'provider_name, crawler_source_data' in sql 130 assert 'provider_name, crawler_source_data' in sql
111 assert '%s::json' in sql 131 assert '%s::json' in sql
112 assert rows[0][-2:] == ('yinyan', '{"id": 200}') 132 assert rows[0][-3:] == ('md5-200', 'yinyan', '{"id": 200}')
113 133
114 134
115 def test_upsert_kugou_singers_writes_provider_and_source_data(): 135 def test_upsert_kugou_singers_writes_provider_and_source_data():
...@@ -211,10 +231,12 @@ def test_upsert_qq_entities_write_provider_and_source_data(): ...@@ -211,10 +231,12 @@ def test_upsert_qq_entities_write_provider_and_source_data():
211 'singers_json': '[]', 231 'singers_json': '[]',
212 'provider_name': 'yinyan', 232 'provider_name': 'yinyan',
213 'crawler_source_data': '{"id": 200}', 233 'crawler_source_data': '{"id": 200}',
234 'audio_md5': 'md5-200',
214 }]) 235 }])
215 sql, rows = cur.executemany.call_args[0] 236 sql, rows = cur.executemany.call_args[0]
237 assert 'audio_md5' in sql
216 assert 'provider_name, crawler_source_data' in sql 238 assert 'provider_name, crawler_source_data' in sql
217 assert rows[0][-2:] == ('yinyan', '{"id": 200}') 239 assert rows[0][-3:] == ('md5-200', 'yinyan', '{"id": 200}')
218 240
219 241
220 def test_upsert_netease_entities_write_provider_and_source_data(): 242 def test_upsert_netease_entities_write_provider_and_source_data():
...@@ -273,7 +295,9 @@ def test_upsert_netease_entities_write_provider_and_source_data(): ...@@ -273,7 +295,9 @@ def test_upsert_netease_entities_write_provider_and_source_data():
273 'singers_json': '[]', 295 'singers_json': '[]',
274 'provider_name': 'yinyan', 296 'provider_name': 'yinyan',
275 'crawler_source_data': '{"id": 300}', 297 'crawler_source_data': '{"id": 300}',
298 'audio_md5': 'md5-300',
276 }]) 299 }])
277 sql, rows = cur.executemany.call_args[0] 300 sql, rows = cur.executemany.call_args[0]
301 assert 'audio_md5' in sql
278 assert 'provider_name, crawler_source_data' in sql 302 assert 'provider_name, crawler_source_data' in sql
279 assert rows[0][-2:] == ('yinyan', '{"id": 300}') 303 assert rows[0][-3:] == ('md5-300', 'yinyan', '{"id": 300}')
......