test_lyric_dedup.py
12.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
import csv
from lyric_dedup import DuplicateChecker
from lyric_dedup import DuplicateDecision
from lyric_dedup import LyricRecord
from lyric_dedup.cli import evaluate_csv
from lyric_dedup.eval_dataset import generate_eval_set
from lyric_dedup.file_import import record_from_file
from lyric_dedup.normalization import normalize_lyrics
BASE_LYRIC = """
[00:01.00]作词:Someone
[00:02.00]我爱你在每个夜里
[00:03.00]听风说话也听见你
[00:04.00]城市的灯慢慢亮起
[00:05.00]我把回忆写进歌曲
[00:06.00]啦啦啦 我们不分离
[00:07.00]啦啦啦 我们不分离
[00:08.00]明天还会继续想你
"""
def test_normalization_removes_lyric_noise_and_simplifies() -> None:
normalized = normalize_lyrics("[00:01.20]我愛你!\nQQ音乐 www.example.com\n(副歌)\n聽風說話\n")
assert normalized.normalized_lines == ("我爱你", "听风说话")
assert normalized.normalized_full_text == "我爱你\n听风说话"
assert normalized.primary_lines == ("我爱你", "听风说话")
def test_exact_duplicate_handles_timestamps_punctuation_traditional_and_chorus_counts() -> None:
checker = DuplicateChecker()
checker.add_record(LyricRecord("song-1", BASE_LYRIC))
result = checker.check(
"""
我愛你,在每個夜裡!!!
聽風說話,也聽見你
城市的燈慢慢亮起
我把回憶寫進歌曲
啦啦啦 我們不分離
明天還會繼續想你
"""
)
assert result.decision == DuplicateDecision.DUPLICATE
assert result.confidence == 1.0
assert result.candidates[0].record_id == "song-1"
def test_short_shared_repeated_chorus_is_review_not_duplicate() -> None:
checker = DuplicateChecker()
checker.add_record(
LyricRecord(
"song-1",
"""
海边的风吹过旧信
你说夏天不会远去
啦啦啦 我们不分离
啦啦啦 我们不分离
转身以后各自旅行
""",
)
)
result = checker.check(
"""
山谷的雨落在清晨
我把名字交给星辰
啦啦啦 我们不分离
啦啦啦 我们不分离
世界安静等一个人
"""
)
assert result.decision == DuplicateDecision.REVIEW
assert result.candidates[0].reason == "重合内容主要集中在重复副歌行,不自动判重"
def test_substantial_line_overlap_is_duplicate_after_lsh_recall() -> None:
checker = DuplicateChecker()
checker.add_record(LyricRecord("song-1", BASE_LYRIC))
result = checker.check(
"""
我爱你在每个夜里
听风说话也听见你
城市灯火慢慢亮起
我把回忆写进歌曲
啦啦啦 我们不分离
明天还会继续想你
"""
)
assert result.decision == DuplicateDecision.DUPLICATE
assert result.candidates[0].jaccard >= 0.78
assert result.candidates[0].line_coverage >= 0.72
def test_fragment_of_full_song_is_not_duplicate() -> None:
checker = DuplicateChecker()
checker.add_record(LyricRecord("song-1", BASE_LYRIC))
result = checker.check(
"""
听风说话也听见你
城市的灯慢慢亮起
我把回忆写进歌曲
"""
)
assert result.decision != DuplicateDecision.DUPLICATE
assert result.candidates[0].primary_line_coverage < 0.72
def test_no_effective_lyrics_use_metadata_fallback_without_empty_hash_collision() -> None:
placeholder = """
作词:DJ金木
作曲:DJ金木
编曲:DJ金木
混音:DJ金木
【未经著作权人许可 不得翻唱 翻录或使用】
"""
checker = DuplicateChecker()
checker.add_record(LyricRecord("song-1", placeholder, title="Amnesia(House)", artist="DJ金木"))
checker.add_record(LyricRecord("song-2", placeholder, title="Angel(纯音乐)", artist="DJ金木"))
same_song = checker.check_record(
LyricRecord("__query__", placeholder, title="Amnesia(House)", artist="DJ金木")
)
different_title = checker.check_record(
LyricRecord("__query__", placeholder, title="Different Song", artist="DJ金木")
)
assert same_song.decision == DuplicateDecision.DUPLICATE
assert same_song.reason == "无有效歌词,使用文件内容兜底指纹命中"
assert different_title.decision == DuplicateDecision.DUPLICATE
def test_no_effective_lyrics_metadata_fallback_ignores_placeholder_noise() -> None:
source = """
作词:DJ金木
作曲:DJ金木
编曲:DJ金木
混音:DJ金木
【未经著作权人许可 不得翻唱 翻录或使用】
"""
noisy = """
[00:01.00]歌词来自QQ音乐
[00:02.00]作词:测试
[00:03.00]作词:DJ金木!
[00:04.00]作曲:DJ金木...
[00:05.00]未经著作权人许可 不得翻唱
"""
checker = DuplicateChecker()
checker.add_record(LyricRecord("song-1", source, title="Amnesia(House)", artist="DJ金木"))
result = checker.check_record(LyricRecord("__query__", noisy, title="Amnesia(House)", artist="DJ金木"))
assert result.decision == DuplicateDecision.DUPLICATE
assert result.reason == "无有效歌词,文件内容兜底特征高度相似"
def test_unrelated_lyrics_with_shared_watermark_are_new() -> None:
checker = DuplicateChecker()
checker.add_record(
LyricRecord(
"song-1",
"""
歌词来自QQ音乐
北方的雪落在窗前
我等一封迟来的信
""",
)
)
result = checker.check(
"""
歌词来自QQ音乐
南方的雨穿过街心
你把故事说给云听
"""
)
assert result.decision == DuplicateDecision.NEW
assert result.candidates == ()
def test_mixed_chinese_english_tokenization_recalls_candidate() -> None:
checker = DuplicateChecker()
checker.add_record(
LyricRecord(
"song-1",
"""
say hello 在风里
hold me close tonight
我们穿过蓝色街道
never let me go
""",
)
)
result = checker.check(
"""
say hello 在风里
hold me close tonight
我们穿过蓝色街道
never let me go
"""
)
assert result.decision == DuplicateDecision.DUPLICATE
def test_checker_can_persist_index(tmp_path) -> None:
index_path = tmp_path / "lyrics.pkl"
checker = DuplicateChecker()
checker.add_record(LyricRecord("song-1", BASE_LYRIC))
checker.save(index_path)
loaded = DuplicateChecker.load(index_path)
result = loaded.check(BASE_LYRIC)
assert loaded.record_count == 1
assert result.decision == DuplicateDecision.DUPLICATE
def test_record_from_lrc_file(tmp_path) -> None:
lyric_file = tmp_path / "周杰伦 - 测试歌.lrc"
lyric_file.write_text("[00:01.00]我愛你\n", encoding="utf-8")
record = record_from_file(lyric_file, base_dir=tmp_path)
assert record.title == "测试歌"
assert record.artist == "周杰伦"
assert record.lyrics == "[00:01.00]我愛你\n"
def test_record_from_song_artist_lyrics_filename(tmp_path) -> None:
lyric_file = tmp_path / "Amnesia(House)-DJ金木-歌词.txt"
lyric_file.write_text("作词:DJ金木\n", encoding="utf-8")
record = record_from_file(lyric_file, base_dir=tmp_path)
assert record.title == "Amnesia(House)"
assert record.artist == "DJ金木"
def test_evaluate_csv_reports_binary_metrics(tmp_path) -> None:
library = tmp_path / "library"
incoming = tmp_path / "incoming"
library.mkdir()
incoming.mkdir()
(library / "歌手A - 夜里.lrc").write_text(BASE_LYRIC, encoding="utf-8")
(incoming / "dup.lrc").write_text(BASE_LYRIC.replace("我爱你", "我愛你"), encoding="utf-8")
(incoming / "new.txt").write_text("南方的雨穿过街心\n你把故事说给云听\n", encoding="utf-8")
checker = DuplicateChecker()
checker.add_record(record_from_file(library / "歌手A - 夜里.lrc", base_dir=library))
index_path = tmp_path / "lyrics.pkl"
checker.save(index_path)
eval_csv = tmp_path / "eval.csv"
eval_csv.write_text(
"id,file,expected\n"
"case-1,incoming/dup.lrc,应去重\n"
"case-2,incoming/new.txt,不应去重\n",
encoding="utf-8",
)
out_path = tmp_path / "eval_out.csv"
evaluate_csv(
index_path,
eval_csv,
out_path,
base_dir=tmp_path,
positive_decisions={"duplicate"},
max_candidates=5,
)
rows = list(csv.DictReader(out_path.open(encoding="utf-8")))
assert [row["correct"] for row in rows] == ["True", "True"]
assert rows[0]["reason"] == "规范化后的原文歌词哈希完全一致"
assert (tmp_path / "eval_out.csv.summary.json").exists()
def test_generated_eval_set_marks_fragments_as_negative(tmp_path) -> None:
library = tmp_path / "library"
incoming = tmp_path / "generated" / "incoming"
eval_csv = tmp_path / "generated" / "eval.csv"
library.mkdir()
(library / "song.txt").write_text(BASE_LYRIC, encoding="utf-8")
generate_eval_set(library_dir=library, output_dir=incoming, csv_path=eval_csv, size=20, positive_ratio=0.5)
rows = list(csv.DictReader(eval_csv.open(encoding="utf-8")))
positive_types = {row["sample_type"] for row in rows if row["expected"] == "应去重"}
fragment_rows = [row for row in rows if row["sample_type"] == "single_song_fragment"]
assert "trimmed_version" not in positive_types
assert "single_song_fragment" not in positive_types
assert fragment_rows
assert all(row["expected"] == "不应去重" for row in fragment_rows)
def test_foreign_original_with_added_chinese_translation_is_duplicate() -> None:
checker = DuplicateChecker()
checker.add_record(
LyricRecord(
"song-1",
"""
I miss you tonight
Under the moonlight
Never let me go
""",
)
)
result = checker.check(
"""
I miss you tonight
今晚我想你
Under the moonlight
月光之下
Never let me go
永远不要让我离开
"""
)
assert result.decision == DuplicateDecision.DUPLICATE
assert result.reason == "规范化后的原文歌词哈希完全一致,翻译行未参与自动判重"
def test_same_timestamp_translation_split_is_high_confidence() -> None:
normalized = normalize_lyrics(
"""
[00:01.00]I miss you tonight
[00:01.00]今晚我想你
[00:02.00]Under the moonlight
[00:02.00]月光之下
"""
)
assert normalized.primary_lines == ("i miss you tonight", "under the moonlight")
assert normalized.translation_lines == ("今晚我想你", "月光之下")
assert normalized.split_confidence == "high"
def test_translation_only_overlap_is_review_not_duplicate() -> None:
checker = DuplicateChecker()
checker.add_record(
LyricRecord(
"song-1",
"""
I miss you tonight
今晚我想你
Under the moonlight
月光之下
Never let me go
永远不要让我离开
""",
)
)
result = checker.check(
"""
Te extrano esta noche
今晚我想你
Bajo la luna
月光之下
No me dejes ir
永远不要让我离开
"""
)
assert result.decision == DuplicateDecision.REVIEW
assert result.reason == "仅翻译行相似,原文字面重合不足,不自动判重"
assert result.candidates[0].translation_jaccard >= 0.45
def test_block_translation_split_is_review_when_primary_matches() -> None:
checker = DuplicateChecker()
checker.add_record(
LyricRecord(
"song-1",
"""
I miss you tonight
Under the moonlight
Never let me go
""",
)
)
result = checker.check(
"""
I miss you tonight
Under the moonlight
Never let me go
今晚我想你
月光之下
永远不要让我离开
"""
)
assert result.decision == DuplicateDecision.REVIEW
assert result.reason == "原文哈希一致,但疑似整段翻译结构拆分置信度较低,需要人工复核"