evaluate.py
3.93 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
import numpy as np
from src.engines.chromaprint_matcher import ChromaprintMatcher
from src.engines.ecapa_embedder import ECAPAEmbedder
from src.engines.hybrid_engine import HybridEngine
def load_items(meta_path: Path):
with open(meta_path) as f:
return json.load(f)
def main():
parser = argparse.ArgumentParser(description="Evaluate ACR recognition quality")
parser.add_argument("--data", default="data/synthetic")
parser.add_argument("--model", required=True)
parser.add_argument("--index-prefix", default="data/index/reference")
parser.add_argument("--split", default="test")
parser.add_argument("--top-k", type=int, default=5)
parser.add_argument("--device", default="cpu")
parser.add_argument("--output-json", default=None)
parser.add_argument("--fast-eval", action="store_true")
parser.add_argument("--chroma-weight", type=float, default=0.25)
parser.add_argument("--ecapa-weight", type=float, default=0.5)
parser.add_argument("--melody-weight", type=float, default=0.25)
args = parser.parse_args()
data_dir = Path(args.data)
matcher = ChromaprintMatcher()
matcher.load(str(Path(args.index_prefix).parent / "chromaprint.pkl"))
embedder = ECAPAEmbedder(model_path=args.model, device=args.device)
ref_embs = np.load(f"{args.index_prefix}_embs.npy")
ref_ids = np.load(f"{args.index_prefix}_ids.npy", allow_pickle=True).tolist()
engine = HybridEngine(
matcher,
embedder,
ref_embs,
ref_ids,
chroma_weight=args.chroma_weight,
ecapa_weight=args.ecapa_weight,
melody_weight=args.melody_weight,
disable_melody=args.fast_eval,
)
for split in ["train.json", "val.json", "test.json"]:
p = data_dir / split
if p.exists():
engine.load_metadata(str(p))
items = load_items(data_dir / f"{args.split}.json")
queries = [x for x in items if str(x.get("audio_path", "")).startswith("segments/")]
if not queries:
raise SystemExit("No segment queries found for evaluation")
top1 = 0
topk = 0
by_type = {}
failures = []
for item in queries:
result = engine.recognize(str(data_dir / item["audio_path"]), top_n=args.top_k)
preds = [c["song_id"] for c in result["candidates"]]
truth = item["song_id"]
qtype = item.get("type", "unknown")
stats = by_type.setdefault(qtype, {"n": 0, "top1": 0, "topk": 0})
stats["n"] += 1
if preds and preds[0] == truth:
top1 += 1
stats["top1"] += 1
if truth in preds:
topk += 1
stats["topk"] += 1
else:
failures.append({
"truth": truth,
"query": item["audio_path"],
"type": qtype,
"preds": preds,
})
total = len(queries)
confusion_focus = {k:v for k,v in by_type.items() if k in {"confused", "humming_like"}}
report = {
"split": args.split,
"num_queries": total,
"top1": round(top1 / total, 4),
"topk": round(topk / total, 4),
"by_type": {
k: {
"n": v["n"],
"top1": round(v["top1"] / v["n"], 4) if v["n"] else 0.0,
"topk": round(v["topk"] / v["n"], 4) if v["n"] else 0.0,
}
for k, v in by_type.items()
},
"hard_case_summary": {
k: {"n": v["n"], "top1": round(v["top1"]/v["n"],4) if v["n"] else 0.0, "topk": round(v["topk"]/v["n"],4) if v["n"] else 0.0}
for k,v in confusion_focus.items()
},
"sample_failures": failures[:10],
}
output = json.dumps(report, ensure_ascii=False, indent=2)
print(output)
if args.output_json:
out = Path(args.output_json)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(output)
if __name__ == "__main__":
main()