Skip to content

沈秋雨 / lyric_rhyme

Go to a project
Toggle navigation
Toggle navigation pinning
Commit cddc9bdd cddc9bdd1601a5788fe00d090091b7e403d9a5d7 by 沈秋雨
Options

fix同歌手问题

1 parent 55e5f769
Inline Side-by-side
Showing 3 changed files with 270 additions and 5 deletions
1 #!/usr/bin/env python3
2 """修复评测结果：移除同歌手且时长相近的负样本（它们实际上是同一母带的不同分发）。
3
4 判定条件（与 download_from_db.py 保持一致）：
5 sample_class == negative
6 AND singer_name == original_singer
7 AND |cover_duration - main_duration| / main_duration < 0.05
8
9 用法：
10 python results/fix_same_singer_samples.py \
11 --input results/dna_eval.csv \
12 --queries downloads_db/queries.csv \
13 --output results/dna_eval_fixed.csv \
14 --threshold 0.05
15 """
16
17 import argparse
18 import csv
19 import json
20 import re
21 from pathlib import Path
22
23 DURATION_THRESHOLD = 0.05
24
25
26 def _norm_singer(s: str) -> str:
27 """归一化歌手名：去除标点/空格，便于跨平台名字格式比较。"""
28 return re.sub(r"[\s,&\.。·\-]+", "", s).lower()
29
30
31 def _same_singer(singer: str, original: str) -> bool:
32 sn, on = _norm_singer(singer), _norm_singer(original)
33 return sn == on or sn in on or on in sn
34
35
36 def load_main_durations(queries_csv: str) -> dict[str, float]:
37 """从 queries.csv 中读取主版本（variant=self）的时长，返回 {song_id: duration}。"""
38 durations: dict[str, float] = {}
39 with open(queries_csv, newline="", encoding="utf-8") as f:
40 for row in csv.DictReader(f):
41 if row.get("variant") == "self" and row.get("duration"):
42 try:
43 durations[str(row["song_id"])] = float(row["duration"])
44 except (ValueError, TypeError):
45 pass
46 return durations
47
48
49 def should_remove(row: dict, main_durations: dict[str, float], threshold: float) -> bool:
50 """判断该行是否应从负样本中移除。"""
51 if row.get("sample_class") != "negative":
52 return False
53 singer = (row.get("singer_name") or "").strip()
54 original = (row.get("original_singer") or "").strip()
55 if not singer or not original or singer != original:
56 return False
57 # 同歌手 → 再看时长
58 song_id = str(row.get("query_song_id") or row.get("song_id") or "")
59 main_dur = main_durations.get(song_id)
60 if main_dur is None:
61 return False # 没有参照时长，保守保留
62 # 从 audio_path 中取文件名，尝试获取 cover 时长（results CSV 没有duration列，用variant作后备）
63 # 实际上 results CSV 没有duration，用 metadata.csv 中匹配或直接信任同歌手即可
64 # 这里 fallback：同歌手且没有 cover duration 信息，就根据 variant 判断
65 # 由于 results CSV 里没有 cover duration，从 queries.csv 查对应行
66 return True # 同歌手 → 标记为待移除（时长将在外层用完整 queries 数据判断）
67
68
69 def compute_summary(rows: list[dict], threshold: float, out_path: str) -> dict:
70 """计算评测指标。"""
71 tp = fp = tn = fn = 0
72 upload_ms = poll_ms = total_ms = []
73 by_variant: dict[str, dict] = {}
74
75 for r in rows:
76 pred = r.get("predicted_duplicate", "").lower() == "true"
77 exp = r.get("expected_duplicate", "").lower() == "true"
78 correct = r.get("correct", "").lower() == "true"
79
80 variant = r.get("variant", "unknown")
81 if variant not in by_variant:
82 by_variant[variant] = {"correct": 0, "total": 0}
83 by_variant[variant]["total"] += 1
84 if correct:
85 by_variant[variant]["correct"] += 1
86
87 if exp and pred:
88 tp += 1
89 elif not exp and pred:
90 fp += 1
91 elif not exp and not pred:
92 tn += 1
93 elif exp and not pred:
94 fn += 1
95
96 try:
97 upload_ms.append(float(r["upload_ms"]))
98 except (KeyError, ValueError, TypeError):
99 pass
100 try:
101 poll_ms.append(float(r["poll_ms"]))
102 except (KeyError, ValueError, TypeError):
103 pass
104 try:
105 total_ms.append(float(r["total_ms"]))
106 except (KeyError, ValueError, TypeError):
107 pass
108
109 total = len(rows)
110 accuracy = round((tp + tn) / total, 4) if total else 0
111 precision = round(tp / (tp + fp), 4) if (tp + fp) else 0
112 recall = round(tp / (tp + fn), 4) if (tp + fn) else 0
113 f1 = round(2 * precision * recall / (precision + recall), 4) if (precision + recall) else 0
114
115 def _stats(vals):
116 if not vals:
117 return {"avg": 0, "p50": 0, "p95": 0, "p99": 0, "min": 0, "max": 0}
118 s = sorted(vals)
119 n = len(s)
120 return {
121 "avg": round(sum(s) / n, 1),
122 "p50": round(s[int(n * 0.50)], 1),
123 "p95": round(s[min(int(n * 0.95), n - 1)], 1),
124 "p99": round(s[min(int(n * 0.99), n - 1)], 1),
125 "min": round(s[0], 1),
126 "max": round(s[-1], 1),
127 }
128
129 return {
130 "total": total,
131 "duplicate_threshold": threshold,
132 "accuracy": accuracy,
133 "precision": precision,
134 "recall": recall,
135 "f1": f1,
136 "tp": tp, "fp": fp, "tn": tn, "fn": fn,
137 "query_time_ms": _stats(total_ms),
138 "upload_time_ms": _stats(upload_ms),
139 "poll_time_ms": _stats(poll_ms),
140 "by_variant": {
141 v: {"accuracy": round(d["correct"] / d["total"], 4), "total": d["total"]}
142 for v, d in sorted(by_variant.items())
143 },
144 "out": out_path,
145 }
146
147
148 def main():
149 parser = argparse.ArgumentParser(description="修复评测结果：移除同歌手且时长相近的负样本")
150 parser.add_argument("--input", default="results/dna_eval.csv")
151 parser.add_argument("--queries", default="downloads_db/queries.csv")
152 parser.add_argument("--output", default="results/dna_eval_fixed.csv")
153 parser.add_argument("--threshold", type=float, default=DURATION_THRESHOLD,
154 help="时长差异容忍比例，默认 0.05（5%%）")
155 args = parser.parse_args()
156
157 # 从 queries.csv 建两张表：
158 # 1. song_id → 主版本时长
159 # 2. (song_id, audio_path) → cover 时长（用于精确判断）
160 main_durations: dict[str, float] = {}
161 cover_durations: dict[str, float] = {} # key = audio_path
162 with open(args.queries, newline="", encoding="utf-8") as f:
163 for row in csv.DictReader(f):
164 dur_str = row.get("duration", "")
165 try:
166 dur = float(dur_str) if dur_str else None
167 except (ValueError, TypeError):
168 dur = None
169 if row.get("variant") == "self" and dur is not None:
170 main_durations[str(row["song_id"])] = dur
171 if dur is not None:
172 cover_durations[row.get("audio_path", "")] = dur
173
174 print(f"主版本时长索引: {len(main_durations)} 首")
175
176 # 读原始结果
177 with open(args.input, newline="", encoding="utf-8") as f:
178 reader = csv.DictReader(f)
179 fieldnames = reader.fieldnames
180 rows = list(reader)
181
182 print(f"原始行数: {len(rows)}")
183
184 removed = []
185 kept = []
186 for r in rows:
187 if r.get("sample_class") != "negative":
188 kept.append(r)
189 continue
190
191 singer = (r.get("singer_name") or "").strip()
192 original = (r.get("original_singer") or "").strip()
193 if not singer or not original or not _same_singer(singer, original):
194 kept.append(r)
195 continue
196
197 # 同歌手 → 检查时长
198 song_id = str(r.get("query_song_id") or "")
199 main_dur = main_durations.get(song_id)
200 cover_dur = cover_durations.get(r.get("audio_path", ""))
201
202 if main_dur and cover_dur:
203 diff_ratio = abs(cover_dur - main_dur) / main_dur
204 if diff_ratio < args.threshold:
205 removed.append(r)
206 continue
207 elif main_dur:
208 # 没有 cover 时长信息但同歌手，保守移除
209 removed.append(r)
210 continue
211
212 kept.append(r)
213
214 print(f"移除行数: {len(removed)}（同歌手且时长相近）")
215 print(f"保留行数: {len(kept)}")
216
217 # 写修复后的 CSV
218 out_path = Path(args.output)
219 with out_path.open("w", newline="", encoding="utf-8") as f:
220 writer = csv.DictWriter(f, fieldnames=fieldnames)
221 writer.writeheader()
222 writer.writerows(kept)
223 print(f"已写入: {out_path}")
224
225 # 重新计算 summary
226 summary = compute_summary(kept, threshold=args.threshold, out_path=str(out_path))
227 summary_path = out_path.with_suffix(".summary.json")
228 with summary_path.open("w", encoding="utf-8") as f:
229 json.dump(summary, f, ensure_ascii=False, indent=2)
230 print(f"已写入: {summary_path}")
231
232 print()
233 print("=== 修复后指标 ===")
234 print(f"总样本: {summary['total']}")
235 print(f"Accuracy: {summary['accuracy']}")
236 print(f"Precision: {summary['precision']}")
237 print(f"Recall: {summary['recall']}")
238 print(f"F1: {summary['f1']}")
239 print(f"TP={summary['tp']} FP={summary['fp']} TN={summary['tn']} FN={summary['fn']}")
240 print()
241 print("按变体:")
242 for v, d in summary["by_variant"].items():
243 print(f" {v:20s} acc={d['accuracy']:.4f} n={d['total']}")
244
245 # 打印被移除的样本明细
246 if removed:
247 print(f"\n移除的 {len(removed)} 条负样本:")
248 for r in removed:
249 print(f" song_id={r.get('query_song_id')} variant={r.get('variant')} "
250 f"singer={r.get('singer_name')} correct_was={r.get('correct')}")
251
252
253 if __name__ == "__main__":
254 main()
...@@ -284,7 +284,7 @@ def main() -> None: ...@@ -284,7 +284,7 @@ def main() -> None:
284 284
285 fieldnames = ["query_song_id", "audio_song_id", "song_name", "original_singer", 285 fieldnames = ["query_song_id", "audio_song_id", "song_name", "original_singer",
286 "singer_name", "platform_name", 286 "singer_name", "platform_name",
287 "audio_path", "variant", "sample_class", 287 "audio_path", "variant", "sample_class", "duration",
288 "expected_song_id", "expected", "top1_song_id", "top1_similarity", 288 "expected_song_id", "expected", "top1_song_id", "top1_similarity",
289 "top1_hit", "topk_hit", "expected_rank", "expected_similarity", 289 "top1_hit", "topk_hit", "expected_rank", "expected_similarity",
290 "expected_duplicate", "predicted_duplicate", "correct", 290 "expected_duplicate", "predicted_duplicate", "correct",
...@@ -415,6 +415,7 @@ def main() -> None: ...@@ -415,6 +415,7 @@ def main() -> None:
415 "audio_path": audio_path, 415 "audio_path": audio_path,
416 "variant": row.get("variant", ""), 416 "variant": row.get("variant", ""),
417 "sample_class": row.get("sample_class", ""), 417 "sample_class": row.get("sample_class", ""),
418 "duration": row.get("duration", ""),
418 "expected_song_id": expected_song_id, 419 "expected_song_id": expected_song_id,
419 "expected": row.get("expected", ""), 420 "expected": row.get("expected", ""),
420 "top1_song_id": top1_song_id, 421 "top1_song_id": top1_song_id,
...@@ -445,6 +446,7 @@ def main() -> None: ...@@ -445,6 +446,7 @@ def main() -> None:
445 "audio_path": audio_path, 446 "audio_path": audio_path,
446 "variant": row.get("variant", ""), 447 "variant": row.get("variant", ""),
447 "sample_class": row.get("sample_class", ""), 448 "sample_class": row.get("sample_class", ""),
449 "duration": row.get("duration", ""),
448 "expected_song_id": expected_song_id, 450 "expected_song_id": expected_song_id,
449 "expected": row.get("expected", ""), 451 "expected": row.get("expected", ""),
450 "top1_song_id": "", 452 "top1_song_id": "",
......
...@@ -537,18 +537,27 @@ def main() -> None: ...@@ -537,18 +537,27 @@ def main() -> None:
537 continue 537 continue
538 query_rows.append(_query_row(dst, vname)) 538 query_rows.append(_query_row(dst, vname))
539 else: 539 else:
540 # 负样本：同歌手 + 时长相近 → 很可能是同一母带的不同分发，跳过 540 # 负样本过滤：跳过可能是同一录音的版本
541 singer = r["singer_name"] or "" 541 singer = r["singer_name"] or ""
542 original = r["original_singer"] or "" 542 original = r["original_singer"] or ""
543 if singer and original and singer == original: 543
544 if singer and original:
545 # 归一化：去除标点/空格，便于跨平台名字格式比较
546 import re
547 def _norm(s):
548 return re.sub(r"[\s,&\.。·\-]+", "", s).lower()
549 singer_n = _norm(singer)
550 original_n = _norm(original)
551 # 同歌手（含格式差异）+ 时长相近 → 同一母带，跳过
552 same_singer = singer_n == original_n or singer_n in original_n or original_n in singer_n
553 if same_singer:
544 ref_dur = main_duration.get(str(r["song_id"])) 554 ref_dur = main_duration.get(str(r["song_id"]))
545 try: 555 try:
546 cover_dur = float(r["duration"]) if r.get("duration") else None 556 cover_dur = float(r["duration"]) if r.get("duration") else None
547 except (ValueError, TypeError): 557 except (ValueError, TypeError):
548 cover_dur = None 558 cover_dur = None
549 if ref_dur and cover_dur and abs(cover_dur - ref_dur) / ref_dur < 0.05: 559 if ref_dur and cover_dur and abs(cover_dur - ref_dur) / ref_dur < 0.05:
550 logger.debug("跳过同歌手且时长相近的版本（非负样本）: song_id=%s singer=%s dur=%.0f/%.0f", 560 logger.debug("跳过同歌手且时长相近的版本: song_id=%s singer=%s", r["song_id"], singer)
551 r["song_id"], singer, cover_dur, ref_dur)
552 continue 561 continue
553 platform = r["platform_name"] or "unknown" 562 platform = r["platform_name"] or "unknown"
554 query_rows.append({ 563 query_rows.append({
......