app.py
17.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
"""FastAPI application for lyric duplicate checking and composition dedup."""
from __future__ import annotations
import logging
import os
import tempfile
import urllib.error
import urllib.request
from pathlib import Path
from typing import Any
from fastapi import FastAPI, File, Form, UploadFile
from fastapi.responses import HTMLResponse, JSONResponse
from pydantic import BaseModel, Field
from ._webui import HTML
from .config import ServerConfig
from .service import DedupService
from composition_dedup.service import CompositionConfig, CompositionDedupService
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# App lifecycle
# ---------------------------------------------------------------------------
app = FastAPI(title="Lyric & Composition Dedup API", version="0.1.0")
_config: ServerConfig | None = None
_service: DedupService | None = None
_composition_service: CompositionDedupService | None = None
@app.on_event("startup")
def _startup() -> None:
global _config, _service, _composition_service
_config = ServerConfig()
_service = DedupService(config=_config)
logger.info("Lyric Dedup API started (DSN=%s, trgm=%s)", _config.dsn, _config.enable_trgm)
composition_config = CompositionConfig(dsn=os.getenv("COMPOSITION_DEDUP_DSN", _config.dsn))
_composition_service = CompositionDedupService(config=composition_config)
logger.info("Composition Dedup API started (DSN=%s)", composition_config.dsn)
# ---------------------------------------------------------------------------
# Request / response models
# ---------------------------------------------------------------------------
class CheckRequest(BaseModel):
url: str = Field(..., description="URL of the LRC/TXT lyric file")
title: str | None = Field(None, description="Song title (optional)")
artist: str | None = Field(None, description="Artist name (optional)")
class CheckResponse(BaseModel):
duplicate: bool
decision: str | None = None
confidence: float | None = None
reason: str | None = None
record_ids: list[str] = []
class HealthResponse(BaseModel):
status: str
# ---------------------------------------------------------------------------
# Rhythm (composition dedup) request / response models
# ---------------------------------------------------------------------------
class RhythmIngestRequest(BaseModel):
song_id: int = Field(..., description="歌曲 ID")
url: str = Field(..., description="音频文件 URL(mp3/wav/flac 等 ffmpeg 支持的格式)")
class RhythmIngestResponse(BaseModel):
song_id: int
success: bool
message: str = ""
class RhythmCheckRequest(BaseModel):
url: str = Field(..., description="待检音频文件 URL")
top_k: int = Field(100, description="召回候选数量", ge=1, le=500)
song_id: int | None = Field(None, description="若提供且检测结果为不重复,自动将该音频特征写入曲库")
class RhythmCandidate(BaseModel):
song_id: int
similarity: float
source: str
class RhythmCheckResponse(BaseModel):
duplicate: bool
auto_ingested: bool = False
candidates: list[RhythmCandidate] = []
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@app.get("/ui", response_class=HTMLResponse, include_in_schema=False)
def webui() -> str:
return HTML
@app.get("/health", response_model=HealthResponse)
def health() -> dict[str, str]:
return {"status": "ok"}
@app.post("/api/v1/check", response_model=CheckResponse)
def check_lyric(req: CheckRequest) -> Any:
if _service is None:
return JSONResponse(
status_code=503,
content={"detail": "service not initialized"},
)
# 校验文件格式(仅接受 .txt / .lrc)
if not _is_valid_lyric_url(req.url):
return JSONResponse(
status_code=400,
content={"detail": "仅支持 .txt 或 .lrc 格式的歌词文件"},
)
try:
lyrics = _download_lyrics(req.url)
except ValueError as exc:
return JSONResponse(
status_code=400,
content={"detail": str(exc)},
)
except Exception as exc:
logger.exception("unexpected error during download")
return JSONResponse(
status_code=500,
content={"detail": f"下载歌词失败: {exc}"},
)
try:
result = _service.check(lyrics, title=req.title, artist=req.artist, source_url=req.url)
except Exception as exc:
logger.exception("unexpected error during dedup check")
return JSONResponse(
status_code=500,
content={"detail": f"歌词去重检测失败: {exc}"},
)
return CheckResponse(
duplicate=result.duplicate,
decision=result.decision,
confidence=result.confidence,
reason=result.reason,
record_ids=result.record_ids,
)
# ---------------------------------------------------------------------------
# Rhythm (composition dedup) endpoints
# ---------------------------------------------------------------------------
@app.post("/api/v1/rhythm/ingest", response_model=RhythmIngestResponse)
def rhythm_ingest(req: RhythmIngestRequest) -> Any:
"""下载音频并将特征写入曲库。"""
if _composition_service is None:
return JSONResponse(status_code=503, content={"detail": "service not initialized"})
try:
audio_path = _download_audio(req.url)
except ValueError as exc:
return JSONResponse(status_code=400, content={"detail": str(exc)})
except Exception as exc:
logger.exception("unexpected error during audio download")
return JSONResponse(status_code=500, content={"detail": f"下载音频失败: {exc}"})
try:
_composition_service.ingest(req.song_id, audio_path)
_save_audio_url(req.song_id, req.url)
except Exception as exc:
logger.exception("unexpected error during composition ingest")
return JSONResponse(status_code=500, content={"detail": f"音频入库失败: {exc}"})
finally:
_cleanup_temp(audio_path)
return RhythmIngestResponse(song_id=req.song_id, success=True, message="入库成功")
@app.post("/api/v1/rhythm/check", response_model=RhythmCheckResponse)
def rhythm_check(req: RhythmCheckRequest) -> Any:
"""下载音频并查询是否与曲库中的曲目重复。不重复且提供了 song_id 时自动入库。"""
if _composition_service is None:
return JSONResponse(status_code=503, content={"detail": "service not initialized"})
try:
audio_path = _download_audio(req.url)
except ValueError as exc:
return JSONResponse(status_code=400, content={"detail": str(exc)})
except Exception as exc:
logger.exception("unexpected error during audio download")
return JSONResponse(status_code=500, content={"detail": f"下载音频失败: {exc}"})
try:
candidates = _composition_service.query(audio_path, top_k=req.top_k)
duplicate = _composition_service.candidates_indicate_duplicate(candidates)
auto_ingested = False
if not duplicate and req.song_id is not None:
_composition_service.ingest(req.song_id, audio_path)
_save_audio_url(req.song_id, req.url)
auto_ingested = True
logger.info("曲去重检测结果为新曲,已自动入库: song_id=%s", req.song_id)
except Exception as exc:
logger.exception("unexpected error during composition query")
return JSONResponse(status_code=500, content={"detail": f"曲去重检测失败: {exc}"})
finally:
_cleanup_temp(audio_path)
return RhythmCheckResponse(
duplicate=duplicate,
auto_ingested=auto_ingested,
candidates=[
RhythmCandidate(song_id=c.song_id, similarity=c.similarity, source=c.source)
for c in candidates
],
)
# ---------------------------------------------------------------------------
# Upload endpoints(供 WebUI 直接上传文件使用)
# ---------------------------------------------------------------------------
@app.post("/api/v1/lyric/check-upload", response_model=CheckResponse)
async def check_lyric_upload(
file: UploadFile = File(...),
title: str | None = Form(None),
artist: str | None = Form(None),
) -> Any:
"""直接上传歌词文件进行去重检测。"""
if _service is None:
return JSONResponse(status_code=503, content={"detail": "service not initialized"})
content = await file.read()
lyrics: str | None = None
for encoding in _ENCODING_CHAIN:
try:
lyrics = content.decode(encoding)
break
except UnicodeDecodeError:
continue
if lyrics is None:
return JSONResponse(status_code=400, content={"detail": "无法解析文件编码,支持: utf-8-sig / utf-8 / gb18030 / big5"})
try:
result = _service.check(lyrics, title=title or None, artist=artist or None, source_url=file.filename)
except Exception as exc:
logger.exception("unexpected error during dedup check (upload)")
return JSONResponse(status_code=500, content={"detail": f"歌词去重检测失败: {exc}"})
return CheckResponse(
duplicate=result.duplicate,
decision=result.decision,
confidence=result.confidence,
reason=result.reason,
record_ids=result.record_ids,
)
@app.post("/api/v1/rhythm/ingest-upload", response_model=RhythmIngestResponse)
async def rhythm_ingest_upload(
song_id: int = Form(...),
file: UploadFile = File(...),
) -> Any:
"""直接上传音频文件写入曲库。"""
if _composition_service is None:
return JSONResponse(status_code=503, content={"detail": "service not initialized"})
content = await file.read()
suffix = Path(file.filename or "audio").suffix.lower() or ".audio"
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=suffix)
try:
tmp.write(content)
tmp.flush()
tmp.close()
_composition_service.ingest(song_id, tmp.name)
except Exception as exc:
logger.exception("unexpected error during composition ingest (upload)")
return JSONResponse(status_code=500, content={"detail": f"音频入库失败: {exc}"})
finally:
_cleanup_temp(tmp.name)
return RhythmIngestResponse(song_id=song_id, success=True, message="入库成功")
@app.post("/api/v1/rhythm/check-upload", response_model=RhythmCheckResponse)
async def rhythm_check_upload(
file: UploadFile = File(...),
top_k: int = Form(100),
song_id: int | None = Form(None),
) -> Any:
"""直接上传音频文件查询是否与曲库重复。不重复且提供了 song_id 时自动入库。"""
if _composition_service is None:
return JSONResponse(status_code=503, content={"detail": "service not initialized"})
content = await file.read()
suffix = Path(file.filename or "audio").suffix.lower() or ".audio"
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=suffix)
try:
tmp.write(content)
tmp.flush()
tmp.close()
candidates = _composition_service.query(tmp.name, top_k=top_k)
duplicate = _composition_service.candidates_indicate_duplicate(candidates)
auto_ingested = False
if not duplicate and song_id is not None:
_composition_service.ingest(song_id, tmp.name)
auto_ingested = True
logger.info("曲去重检测结果为新曲,已自动入库: song_id=%s", song_id)
except Exception as exc:
logger.exception("unexpected error during composition query (upload)")
return JSONResponse(status_code=500, content={"detail": f"曲去重检测失败: {exc}"})
finally:
_cleanup_temp(tmp.name)
return RhythmCheckResponse(
duplicate=duplicate,
auto_ingested=auto_ingested,
candidates=[
RhythmCandidate(song_id=c.song_id, similarity=c.similarity, source=c.source)
for c in candidates
],
)
# ---------------------------------------------------------------------------
# Preview endpoints
# ---------------------------------------------------------------------------
@app.get("/api/v1/lyric/preview")
def lyric_preview(record_id: str) -> Any:
"""按 record_id 返回歌词原文及元数据,供 WebUI 预览。"""
if _service is None:
return JSONResponse(status_code=503, content={"detail": "service not initialized"})
try:
import psycopg
with psycopg.connect(_config.dsn) as conn:
with conn.cursor() as cursor:
cursor.execute(
"SELECT title, artist, raw_text FROM lyrics WHERE record_id = %s AND deleted_at IS NULL",
(record_id,),
)
row = cursor.fetchone()
except Exception as exc:
logger.exception("lyric preview query failed")
return JSONResponse(status_code=500, content={"detail": f"查询失败: {exc}"})
if row is None:
return JSONResponse(status_code=404, content={"detail": "记录不存在"})
title, artist, raw_text = row
return {"record_id": record_id, "title": title, "artist": artist, "text": raw_text}
@app.get("/api/v1/rhythm/preview")
def rhythm_preview(song_id: int) -> Any:
"""按 song_id 返回音频 URL,供 WebUI 预览播放。"""
if _composition_service is None:
return JSONResponse(status_code=503, content={"detail": "service not initialized"})
try:
import psycopg
with psycopg.connect(_composition_service.config.dsn) as conn:
with conn.cursor() as cursor:
cursor.execute(
"SELECT audio_url FROM composition_feature WHERE song_id = %s",
(song_id,),
)
row = cursor.fetchone()
except Exception as exc:
logger.exception("rhythm preview query failed")
return JSONResponse(status_code=500, content={"detail": f"查询失败: {exc}"})
if row is None:
return JSONResponse(status_code=404, content={"detail": "记录不存在"})
return {"song_id": song_id, "audio_url": row[0]}
_ENCODING_CHAIN = ("utf-8-sig", "utf-8", "gb18030", "big5")
_ALLOWED_EXTENSIONS = {".txt", ".lrc"}
def _is_valid_lyric_url(url: str) -> bool:
"""Check if URL points to a .txt or .lrc file."""
from urllib.parse import urlparse
ext = Path(urlparse(url).path).suffix.lower()
return ext in _ALLOWED_EXTENSIONS
def _download_lyrics(url: str) -> str:
"""Download a lyric file and decode with encoding fallback chain."""
try:
with urllib.request.urlopen(url, timeout=_config.download_timeout if _config else 10) as resp:
data = resp.read()
except urllib.error.HTTPError as exc:
raise ValueError(f"下载失败: HTTP {exc.code}") from exc
except urllib.error.URLError as exc:
raise ValueError(f"下载失败: {exc.reason}") from exc
except TimeoutError as exc:
raise ValueError("下载超时") from exc
except Exception as exc:
raise ValueError(f"下载失败: {exc}") from exc
for encoding in _ENCODING_CHAIN:
try:
return data.decode(encoding)
except UnicodeDecodeError:
continue
raise ValueError("无法解析文件编码,支持: utf-8-sig / utf-8 / gb18030 / big5")
_AUDIO_EXTENSIONS = {".mp3", ".wav", ".flac", ".aac", ".ogg", ".m4a", ".opus", ".wma"}
_DOWNLOAD_TIMEOUT: int = 60
def _download_audio(url: str) -> str:
"""下载音频文件到临时目录,返回临时文件路径。
调用方负责在使用完毕后调用 _cleanup_temp() 删除临时文件。
Raises:
ValueError: URL 不合法、格式不支持或下载失败。
"""
from urllib.parse import urlparse
path_suffix = Path(urlparse(url).path).suffix.lower()
if path_suffix not in _AUDIO_EXTENSIONS:
raise ValueError(
f"不支持的音频格式 {path_suffix!r},支持: {', '.join(sorted(_AUDIO_EXTENSIONS))}"
)
try:
with urllib.request.urlopen(url, timeout=_DOWNLOAD_TIMEOUT) as resp:
data = resp.read()
except urllib.error.HTTPError as exc:
raise ValueError(f"下载失败: HTTP {exc.code}") from exc
except urllib.error.URLError as exc:
raise ValueError(f"下载失败: {exc.reason}") from exc
except TimeoutError as exc:
raise ValueError("下载超时") from exc
except Exception as exc:
raise ValueError(f"下载失败: {exc}") from exc
suffix = path_suffix or ".audio"
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=suffix)
try:
tmp.write(data)
tmp.flush()
finally:
tmp.close()
return tmp.name
def _cleanup_temp(path: str) -> None:
"""删除临时文件(忽略不存在等错误)。"""
try:
Path(path).unlink(missing_ok=True)
except Exception:
pass
def _save_audio_url(song_id: int, audio_url: str | None) -> None:
"""将 audio_url 写入 composition_feature 表,供预览使用。失败时仅记录警告,不抛出。"""
if _composition_service is None:
return
try:
import psycopg
with psycopg.connect(_composition_service.config.dsn) as conn:
with conn.cursor() as cursor:
cursor.execute(
"UPDATE composition_feature SET audio_url = %s WHERE song_id = %s",
(audio_url, song_id),
)
conn.commit()
except Exception as exc:
logger.warning("保存 audio_url 失败: song_id=%s, err=%s", song_id, exc)