Skip to content

沈秋雨 / lyric_rhyme

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 5ae39abf 5ae39abf1a4228bb3797e195d471c99ea4ee7581 by 沈秋雨
Options

更新从数据库查询歌曲上传dna库，修正测试集生成逻辑

1 parent 7c12359a
Inline Side-by-side
Showing 3 changed files with 184 additions and 68 deletions
...@@ -333,11 +333,9 @@ def main() -> None: ...@@ -333,11 +333,9 @@ def main() -> None:
333 "expected": "duplicate", 333 "expected": "duplicate",
334 }) 334 })
335 335
336 # ---- 不应去重（expected=not_duplicate）：负样本 + 破坏指纹的变换 ---- 336 # ---- 不应去重（expected=not_duplicate）：composition_drop 原始文件 ----
337 for wav in _tqdm(negative_selected, desc="生成负样本变体", total=len(negative_selected)): 337 for wav in _tqdm(negative_selected, desc="生成负样本原始", total=len(negative_selected)):
338 song_id = _song_id(wav) 338 song_id = _song_id(wav)
339
340 # 负样本原始
341 query_rows.append({ 339 query_rows.append({
342 "song_id": song_id, 340 "song_id": song_id,
343 "audio_path": str(wav.resolve()), 341 "audio_path": str(wav.resolve()),
...@@ -347,59 +345,62 @@ def main() -> None: ...@@ -347,59 +345,62 @@ def main() -> None:
347 "expected": "not_duplicate", 345 "expected": "not_duplicate",
348 }) 346 })
349 347
350 if not args.negative_variants: 348 # ---- 不应去重（expected=not_duplicate）：参照歌 + 破坏指纹的变换 ----
351 continue 349 # 用已入库的参照歌做破坏性变换，测试 ACRCloud 在何种变换幅度下无法识别
352 350 if args.negative_variants:
353 for variant_name, af in NEGATIVE_VARIANTS: 351 for wav in _tqdm(selected, desc="生成参照歌破坏性变体（负样本）", total=len(selected)):
354 dst = variants_dir / f"{song_id}_{variant_name}.wav" 352 song_id = _song_id(wav)
355 if variant_name == "short_clip": 353
356 # 截取 5 秒极短片段，不足以提取有效指纹 354 for variant_name, af in NEGATIVE_VARIANTS:
357 duration = _probe_duration(wav) 355 dst = variants_dir / f"{song_id}_{variant_name}.wav"
358 if duration is not None: 356 if variant_name == "short_clip":
359 usable_end = max(0.0, duration - 5.0) 357 # 截取 5 秒极短片段，不足以提取有效指纹
360 start_min = min(duration * 0.30, usable_end) 358 duration = _probe_duration(wav)
361 start_max = min(duration * 0.50, usable_end) 359 if duration is not None:
362 ss = random.uniform(start_min, start_max) if start_max > start_min else start_min 360 usable_end = max(0.0, duration - 5.0)
363 ok = _run_ffmpeg([ 361 start_min = min(duration * 0.30, usable_end)
364 "ffmpeg", "-y", "-i", str(wav), 362 start_max = min(duration * 0.50, usable_end)
365 "-ss", f"{ss:.3f}", "-t", "5.0", 363 ss = random.uniform(start_min, start_max) if start_max > start_min else start_min
366 "-ar", "22050", "-ac", "1", 364 ok = _run_ffmpeg([
367 str(dst), 365 "ffmpeg", "-y", "-i", str(wav),
368 ]) 366 "-ss", f"{ss:.3f}", "-t", "5.0",
367 "-ar", "22050", "-ac", "1",
368 str(dst),
369 ])
370 else:
371 ok = False
369 else: 372 else:
370 ok = False 373 ok = _ffmpeg_variant(wav, dst, af)
371 else: 374 if not ok:
372 ok = _ffmpeg_variant(wav, dst, af) 375 logger.warning("破坏性变换失败，跳过: %s %s", wav.name, variant_name)
373 if not ok: 376 continue
374 logger.warning("负样本变换失败，跳过: %s %s", wav.name, variant_name) 377 query_rows.append({
375 continue 378 "song_id": song_id,
376 query_rows.append({ 379 "audio_path": str(dst.resolve()),
377 "song_id": song_id, 380 "variant": variant_name,
378 "audio_path": str(dst.resolve()), 381 "sample_class": "negative",
379 "variant": variant_name, 382 "expected_song_id": song_id,
380 "sample_class": "negative", 383 "expected": "not_duplicate",
381 "expected_song_id": "", 384 })
382 "expected": "not_duplicate", 385
383 }) 386 # 片段拼接负样本：从参照歌中各取一段拼接，混合内容不应归属任何单首歌
384 387 if len(selected) >= SPLICE_SONGS_PER_SAMPLE:
385 # 片段拼接负样本：每个样本从几首不同歌曲各取一段拼接 388 for i in _tqdm(range(len(selected)), desc="生成片段拼接负样本", total=len(selected)):
386 if args.negative_variants and len(negative_selected) >= SPLICE_SONGS_PER_SAMPLE: 389 srcs = random.sample(selected, SPLICE_SONGS_PER_SAMPLE)
387 for i in _tqdm(range(len(negative_selected)), desc="生成片段拼接负样本", total=len(negative_selected)): 390 splice_id = f"splice_{i:04d}"
388 srcs = random.sample(negative_selected, SPLICE_SONGS_PER_SAMPLE) 391 dst = variants_dir / f"{splice_id}_negative_splice.wav"
389 splice_id = f"splice_{i:04d}" 392 ok = _ffmpeg_splice(srcs, dst)
390 dst = variants_dir / f"{splice_id}_negative_splice.wav" 393 if not ok:
391 ok = _ffmpeg_splice(srcs, dst) 394 logger.warning("片段拼接生成失败，跳过: splice %d", i)
392 if not ok: 395 continue
393 logger.warning("片段拼接生成失败，跳过: splice %d", i) 396 query_rows.append({
394 continue 397 "song_id": splice_id,
395 query_rows.append({ 398 "audio_path": str(dst.resolve()),
396 "song_id": splice_id, 399 "variant": "negative_splice",
397 "audio_path": str(dst.resolve()), 400 "sample_class": "negative",
398 "variant": "negative_splice", 401 "expected_song_id": "",
399 "sample_class": "negative", 402 "expected": "not_duplicate",
400 "expected_song_id": "", 403 })
401 "expected": "not_duplicate",
402 })
403 404
404 # ---- 输出 CSV ---- 405 # ---- 输出 CSV ----
405 ref_path = out_dir / "reference.csv" 406 ref_path = out_dir / "reference.csv"
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
3 """批量上传音频到阿里云 OSS 并提交 DNA 入库作业。 3 """批量上传音频到阿里云 OSS 并提交 DNA 入库作业。
4 4
5 流程： 5 流程：
6 1. 扫描音频文件 6 1. 扫描音频文件（来源：本地目录 / CSV / 数据库）
7 2. 降采样（可选，默认 16kHz 单声道） 7 2. 降采样（可选，默认 16kHz 单声道）
8 3. 上传到 OSS 8 3. 上传到 OSS
9 4. 调用 SubmitDNAJob 提交入库作业（SaveType=save, MediaType=audio） 9 4. 调用 SubmitDNAJob 提交入库作业（SaveType=save, MediaType=audio）
...@@ -21,6 +21,13 @@ python scripts/aliyun_dna/upload_dna_library.py --retry-failed ...@@ -21,6 +21,13 @@ python scripts/aliyun_dna/upload_dna_library.py --retry-failed
21 21
22 # 不降采样（上传原始文件） 22 # 不降采样（上传原始文件）
23 python scripts/aliyun_dna/upload_dna_library.py --ref-csv ... --no-resample 23 python scripts/aliyun_dna/upload_dna_library.py --ref-csv ... --no-resample
24
25 # 从数据库读取（从指定表的 id/url 列拉取音频 URL 后下载入库）
26 python scripts/aliyun_dna/upload_dna_library.py \
27 --db-table crawler_kugou_songs \
28 --db-id-column id \
29 --db-url-column url \
30 --state-file upload_dna_state.json
24 """ 31 """
25 32
26 import argparse 33 import argparse
...@@ -31,6 +38,8 @@ import os ...@@ -31,6 +38,8 @@ import os
31 import sys 38 import sys
32 import tempfile 39 import tempfile
33 import time 40 import time
41 import urllib.request
42 import urllib.error
34 from pathlib import Path 43 from pathlib import Path
35 44
36 sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) 45 sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
...@@ -180,6 +189,68 @@ def discover_audio_files(ref_csv: str | None, audio_dir: str | None) -> list[tup ...@@ -180,6 +189,68 @@ def discover_audio_files(ref_csv: str | None, audio_dir: str | None) -> list[tup
180 return results 189 return results
181 190
182 191
192 def discover_audio_from_db(
193 db_url: str,
194 table: str,
195 id_column: str,
196 url_column: str,
197 where: str | None = None,
198 limit: int | None = None,
199 ) -> list[tuple[str, str]]:
200 """从数据库查询歌曲，返回 [(song_id, audio_url), ...]。
201
202 audio_url 是远端 HTTP(S) 地址，后续由 download_audio_url 下载到本地临时文件。
203 """
204 try:
205 import psycopg
206 except ImportError:
207 logger.error("缺少 psycopg 驱动，请执行: pip install psycopg")
208 sys.exit(1)
209
210 # DATABASE_URL 格式为 postgresql+asyncpg://...，需转为 psycopg 格式
211 dsn = db_url.replace("postgresql+asyncpg://", "postgresql://").replace("postgresql+psycopg2://", "postgresql://")
212
213 sql = f'SELECT "{id_column}", "{url_column}" FROM "{table}"'
214 if where:
215 sql += f" WHERE {where}"
216 if limit:
217 sql += f" LIMIT {limit}"
218
219 logger.info("查询数据库: %s", sql)
220 results = []
221 with psycopg.connect(dsn) as conn:
222 with conn.cursor() as cur:
223 cur.execute(sql)
224 for row in cur.fetchall():
225 song_id = str(row[0]).strip()
226 audio_url = (row[1] or "").strip()
227 if not audio_url:
228 logger.warning(" 跳过 song_id=%s: url 为空", song_id)
229 continue
230 results.append((song_id, audio_url))
231
232 logger.info("数据库查询到 %d 条记录", len(results))
233 return results
234
235
236 def download_audio_url(audio_url: str, song_id: str, timeout: int = 60) -> str | None:
237 """从 HTTP(S) URL 下载音频到临时文件，返回本地路径。失败返回 None。"""
238 ext = Path(audio_url.split("?")[0]).suffix or ".mp3"
239 tmp = tempfile.NamedTemporaryFile(suffix=ext, delete=False)
240 tmp.close()
241 try:
242 req = urllib.request.Request(audio_url, headers={"User-Agent": "Mozilla/5.0"})
243 with urllib.request.urlopen(req, timeout=timeout) as resp:
244 data = resp.read()
245 Path(tmp.name).write_bytes(data)
246 logger.info(" 下载完成: %s (%s)", song_id, _human_size(len(data)))
247 return tmp.name
248 except Exception as e:
249 logger.warning(" 下载失败 song_id=%s: %s", song_id, e)
250 Path(tmp.name).unlink(missing_ok=True)
251 return None
252
253
183 def load_state(state_file: str) -> dict: 254 def load_state(state_file: str) -> dict:
184 if not Path(state_file).exists(): 255 if not Path(state_file).exists():
185 return {} 256 return {}
...@@ -211,6 +282,16 @@ def main() -> None: ...@@ -211,6 +282,16 @@ def main() -> None:
211 help=f"降采样目标采样率（默认 {DEFAULT_SAMPLE_RATE}Hz）") 282 help=f"降采样目标采样率（默认 {DEFAULT_SAMPLE_RATE}Hz）")
212 parser.add_argument("--no-resample", action="store_true", 283 parser.add_argument("--no-resample", action="store_true",
213 help="不降采样，上传原始文件") 284 help="不降采样，上传原始文件")
285 # 数据库模式参数
286 db_group = parser.add_argument_group("数据库模式（与 --audio-dir/--ref-csv 互斥）")
287 db_group.add_argument("--db-table", help="要查询的数据库表名，例如 crawler_kugou_songs")
288 db_group.add_argument("--db-id-column", default="id", help="主键列名（默认 id）")
289 db_group.add_argument("--db-url-column", default="url", help="音频 URL 列名（默认 url）")
290 db_group.add_argument("--db-where", help="可选的 WHERE 子句，例如 \"url IS NOT NULL\"")
291 db_group.add_argument("--db-limit", type=int, help="最多查询的行数限制")
292 db_group.add_argument("--db-url", help="数据库连接串（默认读取 DATABASE_URL 环境变量）")
293 db_group.add_argument("--download-timeout", type=int, default=60,
294 help="下载音频 URL 的超时秒数（默认 60）")
214 args = parser.parse_args() 295 args = parser.parse_args()
215 296
216 # 验证配置 297 # 验证配置
...@@ -221,15 +302,37 @@ def main() -> None: ...@@ -221,15 +302,37 @@ def main() -> None:
221 logger.error("缺少环境变量: %s", key) 302 logger.error("缺少环境变量: %s", key)
222 sys.exit(1) 303 sys.exit(1)
223 304
224 audio_files = discover_audio_files(args.ref_csv, args.audio_dir) 305 use_db_mode = bool(args.db_table)
225 logger.info("发现 %d 个音频文件", len(audio_files)) 306
307 if use_db_mode:
308 db_url = args.db_url or os.environ.get("DATABASE_URL", "")
309 if not db_url:
310 logger.error("数据库模式需要 --db-url 参数或 DATABASE_URL 环境变量")
311 sys.exit(1)
312 # 返回 [(song_id, audio_url), ...] 其中 audio_url 为远端地址
313 audio_files = discover_audio_from_db(
314 db_url=db_url,
315 table=args.db_table,
316 id_column=args.db_id_column,
317 url_column=args.db_url_column,
318 where=args.db_where,
319 limit=args.db_limit,
320 )
321 else:
322 audio_files = discover_audio_files(args.ref_csv, args.audio_dir)
323
324 logger.info("发现 %d 个音频条目", len(audio_files))
226 325
227 state = load_state(args.state_file) 326 state = load_state(args.state_file)
228 327
229 # 过滤已成功入库的文件 328 # 过滤已成功入库的条目（以 song_id 作为 key 避免 URL 变化导致重复）
329 def _state_key(song_id: str, audio_path: str) -> str:
330 return f"db:{song_id}" if use_db_mode else audio_path
331
230 pending = [] 332 pending = []
231 for song_id, audio_path in audio_files: 333 for song_id, audio_path in audio_files:
232 status = state.get(audio_path) 334 key = _state_key(song_id, audio_path)
335 status = state.get(key)
233 if status == "ok": 336 if status == "ok":
234 continue 337 continue
235 if status == "failed" and not args.retry_failed: 338 if status == "failed" and not args.retry_failed:
...@@ -256,15 +359,27 @@ def main() -> None: ...@@ -256,15 +359,27 @@ def main() -> None:
256 tmp_files = [] # 跟踪临时文件，结束时清理 359 tmp_files = [] # 跟踪临时文件，结束时清理
257 360
258 for i, (song_id, audio_path) in enumerate(pending, 1): 361 for i, (song_id, audio_path) in enumerate(pending, 1):
259 logger.info("[%d/%d] 处理: %s (song_id=%s)", i, len(pending), Path(audio_path).name, song_id) 362 logger.info("[%d/%d] 处理: %s (song_id=%s)", i, len(pending),
363 audio_path if not use_db_mode else f"url={audio_path[:80]}...", song_id)
260 364
365 key = _state_key(song_id, audio_path)
261 upload_path = audio_path 366 upload_path = audio_path
367 downloaded_tmp = None
368
262 try: 369 try:
263 t0 = time.perf_counter() 370 t0 = time.perf_counter()
264 371
372 # 数据库模式：先从 URL 下载到本地临时文件
373 if use_db_mode:
374 downloaded_tmp = download_audio_url(audio_path, song_id, timeout=args.download_timeout)
375 if downloaded_tmp is None:
376 raise RuntimeError("音频 URL 下载失败")
377 upload_path = downloaded_tmp
378 tmp_files.append(downloaded_tmp)
379
265 # 1. 降采样 380 # 1. 降采样
266 if not args.no_resample: 381 if not args.no_resample:
267 resampled = _resample_audio(audio_path, target_sr=args.sample_rate) 382 resampled = _resample_audio(upload_path, target_sr=args.sample_rate)
268 if resampled: 383 if resampled:
269 upload_path = resampled 384 upload_path = resampled
270 tmp_files.append(resampled) 385 tmp_files.append(resampled)
...@@ -278,15 +393,15 @@ def main() -> None: ...@@ -278,15 +393,15 @@ def main() -> None:
278 elapsed_ms = (time.perf_counter() - t0) * 1000 393 elapsed_ms = (time.perf_counter() - t0) * 1000
279 394
280 logger.info(" 入库作业已提交: JobId=%s (%.0fms)", job_id, elapsed_ms) 395 logger.info(" 入库作业已提交: JobId=%s (%.0fms)", job_id, elapsed_ms)
281 state[audio_path] = "ok" 396 state[key] = "ok"
282 state[f"{audio_path}::job_id"] = job_id 397 state[f"{key}::job_id"] = job_id
283 state[f"{audio_path}::oss_url"] = oss_url 398 state[f"{key}::oss_url"] = oss_url
284 success += 1 399 success += 1
285 400
286 except Exception as e: 401 except Exception as e:
287 logger.error(" 处理失败: %s", e) 402 logger.error(" 处理失败: %s", e)
288 state[audio_path] = "failed" 403 state[key] = "failed"
289 state[f"{audio_path}::error"] = str(e) 404 state[f"{key}::error"] = str(e)
290 fail += 1 405 fail += 1
291 406
292 save_state(args.state_file, state) 407 save_state(args.state_file, state)
......
This diff could not be displayed because it is too large.