优化性能

@@ -26,4 +26,6 @@ venv/
 
 test_api
 
-composition_dedup/composition_eval
\ No newline at end of file
+composition_dedup/composition_eval
+
+composition_testset
\ No newline at end of file

@@ -22,7 +22,6 @@ from pathlib import Path
 import librosa
 import numpy as np
 from scipy.ndimage import (
-    binary_erosion,
     generate_binary_structure,
     iterate_structure,
     maximum_filter,
@@ -60,10 +59,10 @@ MIN_HASH_TIME_DELTA = 0
 MAX_HASH_TIME_DELTA = 200
 PEAK_SORT = True
 FINGERPRINT_REDUCTION = 20
-MAX_DURATION_SEC = float(os.environ.get("COMPOSITION_DEJAVU_MAX_DURATION", "120"))  # 0=不限制
+QUERY_MAX_DURATION_SEC = float(os.environ.get("COMPOSITION_DEJAVU_QUERY_MAX_DURATION", "120"))  # 0=不限制
 
 
-def _normalize_audio(audio_path: str, max_duration: float = MAX_DURATION_SEC) -> tuple[np.ndarray, int]:
+def _normalize_audio(audio_path: str, max_duration: float = 0) -> tuple[np.ndarray, int]:
     """将音频标准化为单声道 WAV 并加载为 numpy 数组。
 
     使用 ffmpeg 先做重采样，再用 librosa 读取。
@@ -133,12 +132,7 @@ def _get_2d_peaks(arr2D: np.ndarray, amp_min: float = DEFAULT_AMP_MIN):
     neighborhood = iterate_structure(struct, PEAK_NEIGHBORHOOD_SIZE)
 
     # 找局部极大值
-    local_max = maximum_filter(arr2D, footprint=neighborhood) == arr2D
-    background = arr2D == 0
-    eroded_background = binary_erosion(background, structure=neighborhood, border_value=1)
-
-    # 布尔掩码
-    detected_peaks = local_max ^ eroded_background
+    detected_peaks = maximum_filter(arr2D, footprint=neighborhood) == arr2D
 
     # 提取峰值
     amps = arr2D[detected_peaks]
@@ -181,6 +175,43 @@ def _generate_hashes(peaks: list[tuple[int, int]], fan_value: int = DEFAULT_FAN_
                     yield (h.hexdigest()[:FINGERPRINT_REDUCTION].encode(), t1)
 
 
+def load_audio(audio_path: str, max_duration: float = 0) -> tuple[np.ndarray, int]:
+    """加载并标准化音频为 44100Hz 单声道（供多路径共用，避免重复解码）。
+
+    Args:
+        audio_path: 音频文件路径。
+        max_duration: 最大截取时长（秒），0 表示不限制。
+
+    Returns:
+        (samples, sr) 元组。
+    """
+    return _normalize_audio(audio_path, max_duration)
+
+
+def fingerprint_from_samples(
+    samples: np.ndarray, sr: int, *, compute_sha1: bool = True
+) -> tuple[str, list[tuple[bytes, int]]]:
+    """对已加载的音频样本生成 Dejavu 风格指纹（不做 I/O）。
+
+    Args:
+        samples: 单声道音频样本（应为 DEFAULT_FS=44100Hz）。
+        sr: 采样率。
+        compute_sha1: 是否计算 file_sha1。service 内部调用时传 False 可跳过
+            对 samples.tobytes() 的 21MB 哈希运算（返回值在那些路径中未被使用）。
+
+    Returns:
+        (file_sha1, fingerprints) 元组，
+        其中 fingerprints 是 [(hash_bytes, offset), ...] 列表。
+        compute_sha1=False 时 file_sha1 返回空字符串。
+    """
+    file_sha1 = hashlib.sha1(samples.tobytes()).hexdigest()[:16] if compute_sha1 else ""
+    arr2D = _specgram(samples, sr, DEFAULT_WINDOW_SIZE, DEFAULT_OVERLAP_RATIO)
+    freq_idx, time_idx = _get_2d_peaks(arr2D)
+    peaks = list(zip(freq_idx, time_idx))
+    fingerprints = list(_generate_hashes(peaks))
+    return file_sha1, fingerprints
+
+
 def fingerprint_audio(audio_path: str) -> tuple[str, list[tuple[bytes, int]]]:
     """对音频文件生成 Dejavu 风格指纹。
 
@@ -198,21 +229,7 @@ def fingerprint_audio(audio_path: str) -> tuple[str, list[tuple[bytes, int]]]:
     if not os.path.isfile(audio_path):
         raise FileNotFoundError(f"音频文件不存在: {audio_path}")
 
-    # 1. 标准化并加载音频（可选限制长度）
     samples, fs = _normalize_audio(audio_path)
-
-    # 2. 计算文件 SHA1（用于标识）
-    file_sha1 = hashlib.sha1(samples.tobytes()).hexdigest()[:16]
-
-    # 3. 计算频谱图
-    arr2D = _specgram(samples, fs, DEFAULT_WINDOW_SIZE, DEFAULT_OVERLAP_RATIO)
-
-    # 4. 检测 2D 峰值
-    freq_idx, time_idx = _get_2d_peaks(arr2D)
-    peaks = list(zip(freq_idx, time_idx))
-
-    # 5. 生成指纹哈希
-    fingerprints = list(_generate_hashes(peaks))
-
+    file_sha1, fingerprints = fingerprint_from_samples(samples, fs)
     logger.info("指纹生成完成: audio=%s, 指纹数=%d", audio_path, len(fingerprints))
     return file_sha1, fingerprints

 """Chromagram 特征提取。
 
 流程：
-1. 音频标准化：ffmpeg 转 22050Hz / Mono / WAV
-2. librosa 加载音频
-3. librosa.feature.chroma_cens() 提取 12×T Chromagram（CENS，对速度/音色鲁棒）
-4. 主音对齐：将能量最大的音级滚至第 0 行，实现转调不变性
-5. scipy.signal.resample(chroma, 128, axis=1) 时间归一化到 12×128
-6. .flatten() 展开为 1536 维向量
+1. 音频解码：ffmpeg pipe 输出 22050Hz / Mono / f32le，直接读入内存，无临时文件
+2. librosa.feature.chroma_cens() 提取 12×T Chromagram（CENS，对速度/音色鲁棒）
+3. 主音对齐：将能量最大的音级滚至第 0 行，实现转调不变性
+4. scipy.signal.resample(chroma, 128, axis=1) 时间归一化到 12×128
+5. .flatten() 展开为 1536 维向量
 """
 
 import logging
 import os
 import subprocess
-import tempfile
 
 import librosa
 import numpy as np
@@ -26,83 +24,99 @@ TARGET_FRAMES = 128
 VECTOR_DIM = 12 * TARGET_FRAMES  # 1536
 
 
-def _normalize_audio_ffmpeg(audio_path: str, output_path: str) -> None:
-    """使用 ffmpeg 将音频标准化为 22050Hz / Mono / WAV。"""
+def _load_audio_via_pipe(audio_path: str) -> np.ndarray:
+    """使用 ffmpeg pipe 将音频解码为 22050Hz mono float32，不落临时文件到磁盘。"""
     cmd = [
-        "ffmpeg",
-        "-y",
+        "ffmpeg", "-y",
         "-i", audio_path,
         "-ar", str(TARGET_SR),
         "-ac", "1",
-        "-f", "wav",
-        output_path,
+        "-f", "f32le",
+        "pipe:1",
     ]
-    result = subprocess.run(
-        cmd,
-        capture_output=True,
-        text=True,
-    )
+    result = subprocess.run(cmd, capture_output=True)
     if result.returncode != 0:
-        raise RuntimeError(f"ffmpeg 转换失败: {result.stderr}")
+        raise RuntimeError(f"ffmpeg 解码失败: {result.stderr.decode(errors='replace')}")
+    return np.frombuffer(result.stdout, dtype=np.float32)
 
 
-def extract_chroma_feature(audio_path: str) -> np.ndarray:
-    """从音频文件提取 1536 维 Chromagram 特征向量。
+def extract_chroma_feature_from_samples(
+    samples: np.ndarray,
+    sr: int,
+    hop_length: int = 512,
+    win_len_smooth: int = 41,
+) -> np.ndarray:
+    """从已加载的音频样本提取 1536 维 Chromagram 特征向量。
+
+    若 sr 不等于 TARGET_SR，先用 librosa.resample 在内存中降采样，
+    避免重新走 ffmpeg 流程。
 
     Args:
-        audio_path: 音频文件路径。
+        samples: 单声道音频样本（任意采样率）。
+        sr: samples 对应的采样率。
+        hop_length: CQT hop 大小，增大可成比例降低计算量，不影响最终 128 帧精度。
+        win_len_smooth: CENS 平滑窗口帧数，应随 hop_length 等比缩小以保持相同的时间覆盖。
 
     Returns:
         shape 为 (1536,) 的 numpy 数组。
-
-    Raises:
-        FileNotFoundError: 音频文件不存在。
-        RuntimeError: ffmpeg 转换失败。
     """
-    if not os.path.isfile(audio_path):
-        raise FileNotFoundError(f"音频文件不存在: {audio_path}")
+    y = samples if sr == TARGET_SR else librosa.resample(samples, orig_sr=sr, target_sr=TARGET_SR)
 
-    # 1. 音频标准化：ffmpeg 转 WAV
-    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
-        tmp_wav = tmp.name
+    # 提取 CENS Chromagram (12×T)
+    chroma = librosa.feature.chroma_cens(y=y, sr=TARGET_SR, hop_length=hop_length, win_len_smooth=win_len_smooth)
 
-    try:
-        _normalize_audio_ffmpeg(audio_path, tmp_wav)
+    # 主音对齐
+    tonic = int(np.argmax(chroma.sum(axis=1)))
+    if tonic != 0:
+        chroma = np.roll(chroma, -tonic, axis=0)
 
-        # 2. librosa 加载音频
-        y, _sr = librosa.load(tmp_wav, sr=TARGET_SR, mono=True)
+    # 时间归一化到 12×128
+    if chroma.shape[1] != TARGET_FRAMES:
+        chroma = resample(chroma, TARGET_FRAMES, axis=1)
+
+    feature = chroma.flatten().astype(np.float32)
+    assert feature.shape == (VECTOR_DIM,), (
+        f"特征维度错误: 期望 {VECTOR_DIM}, 实际 {feature.shape}"
+    )
+    return feature
 
-        # 3. 提取 CENS Chromagram (12×T)，对速度变化和音色具有更强鲁棒性
-        chroma = librosa.feature.chroma_cens(y=y, sr=TARGET_SR)
 
-        # 4. 主音对齐：将全局能量最大的音级循环滚至第 0 行，实现转调不变性
-        tonic = int(np.argmax(chroma.sum(axis=1)))
-        if tonic != 0:
-            chroma = np.roll(chroma, -tonic, axis=0)
+def extract_chroma_matrix_from_samples(
+    samples: np.ndarray,
+    sr: int,
+    hop_length: int = 512,
+    win_len_smooth: int = 41,
+) -> np.ndarray:
+    """从已加载的音频样本提取 12×128 Chromagram 矩阵（供 DTW 精排使用）。"""
+    return extract_chroma_feature_from_samples(samples, sr, hop_length=hop_length, win_len_smooth=win_len_smooth).reshape(12, TARGET_FRAMES)
 
-        # 5. 时间归一化到 12×128
-        if chroma.shape[1] != TARGET_FRAMES:
-            chroma = resample(chroma, TARGET_FRAMES, axis=1)
 
-        # 6. 展开为 1536 维向量
-        feature = chroma.flatten().astype(np.float32)
+def extract_chroma_feature(audio_path: str, hop_length: int = 512, win_len_smooth: int = 41) -> np.ndarray:
+    """从音频文件提取 1536 维 Chromagram 特征向量。
 
-        assert feature.shape == (VECTOR_DIM,), (
-            f"特征维度错误: 期望 {VECTOR_DIM}, 实际 {feature.shape}"
-        )
+    Args:
+        audio_path: 音频文件路径。
+        hop_length: CQT hop 大小。
+        win_len_smooth: CENS 平滑窗口帧数。
+
+    Returns:
+        shape 为 (1536,) 的 numpy 数组。
+
+    Raises:
+        FileNotFoundError: 音频文件不存在。
+        RuntimeError: ffmpeg 解码失败。
+    """
+    if not os.path.isfile(audio_path):
+        raise FileNotFoundError(f"音频文件不存在: {audio_path}")
 
-        return feature
-    finally:
-        # 清理临时文件
-        if os.path.exists(tmp_wav):
-            os.remove(tmp_wav)
+    y = _load_audio_via_pipe(audio_path)
+    return extract_chroma_feature_from_samples(y, TARGET_SR, hop_length=hop_length, win_len_smooth=win_len_smooth)
 
 
-def extract_chroma_matrix(audio_path: str) -> np.ndarray:
+def extract_chroma_matrix(audio_path: str, hop_length: int = 512, win_len_smooth: int = 41) -> np.ndarray:
     """从音频文件提取 12×128 Chromagram 矩阵（未展平，供 DTW 精排使用）。
 
     Returns:
         shape 为 (12, 128) 的 numpy 数组，已做主音对齐。
     """
-    feature = extract_chroma_feature(audio_path)
-    return feature.reshape(12, TARGET_FRAMES)
+    return extract_chroma_feature(audio_path, hop_length=hop_length, win_len_smooth=win_len_smooth).reshape(12, TARGET_FRAMES)

@@ -12,6 +12,7 @@ tqdm>=4.66
 
 # Audio composition feature extraction
 librosa>=0.10.0
+numba>=0.59.0
 scipy>=1.11
 numpy>=1.24
 
@@ -21,3 +22,6 @@ pgvector>=0.2.0
 # HTTP API server
 fastapi>=0.110.0
 uvicorn[standard]>=0.29.0
+
+# Environment variable loading
+python-dotenv>=1.0

@@ -8,8 +8,8 @@ expected_song_id 的 top-k/top1 命中只作为诊断字段。
 用法:
 python scripts/evaluate_composition.py \
     --dsn "postgresql:///lyric_dedup" \
-    --queries composition_dedup/composition_testset4/queries.csv \
-    --out composition_dedup/composition_eval/composition_eval_result_v3.csv
+    --queries composition_testset/test_samples.csv \
+    --out composition_dedup/composition_eval/nohop_result.csv
 """
 
 import argparse
@@ -17,10 +17,14 @@ import csv
 import json
 import logging
 import sys
+import time
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
+from dotenv import load_dotenv
+load_dotenv(Path(__file__).resolve().parent.parent / ".env")
+
 from composition_dedup.service import CompositionConfig, CompositionDedupService
 
 logger = logging.getLogger(__name__)
@@ -92,8 +96,12 @@ def main() -> None:
         invalid_negative_pair = (not expected_dup) and audio_song_id == expected_song_id
 
         try:
-            candidates = service.query(audio_path, top_k=args.top_k)
+            timings: dict = {}
+            _t0 = time.perf_counter()
+            candidates = service.query(audio_path, top_k=args.top_k, timings=timings)
+            query_time_ms = round((time.perf_counter() - _t0) * 1000, 1)
         except Exception as e:
+            query_time_ms = round((time.perf_counter() - _t0) * 1000, 1)
             logger.error("[%d/%d] 查询失败: %s, %s", i, len(rows), audio_path, e)
             result_rows.append({
                 "query_song_id": query_song_id,
@@ -106,6 +114,7 @@ def main() -> None:
                 "top1_song_id": "",
                 "top1_similarity": "",
                 "top1_source": "",
+                "dejavu_aligned_count": "",
                 "top1_hit": False,
                 "topk_hit": False,
                 "expected_rank": "",
@@ -115,6 +124,14 @@ def main() -> None:
                 "expected_duplicate": expected_dup,
                 "predicted_duplicate": False,
                 "correct": not expected_dup,  # 查询失败视为 not_duplicate
+                "query_time_ms": query_time_ms,
+                "chroma_extract_ms": timings.get("chroma_extract_ms", ""),
+                "db_cosine_ms": timings.get("db_cosine_ms", ""),
+                "db_fetch_ms": timings.get("db_fetch_ms", ""),
+                "dtw_ms": timings.get("dtw_ms", ""),
+                "dejavu_decode_ms": timings.get("dejavu_decode_ms", ""),
+                "dejavu_fingerprint_ms": timings.get("dejavu_fingerprint_ms", ""),
+                "dejavu_db_ms": timings.get("dejavu_db_ms", ""),
                 "error": str(e),
             })
             continue
@@ -123,6 +140,7 @@ def main() -> None:
         top1_song_id = str(top1.song_id) if top1 else ""
         top1_sim = round(top1.similarity, 4) if top1 else ""
         top1_source = top1.source if top1 else ""
+        dejavu_aligned_count = top1.dejavu_aligned_count if top1 else ""
 
         # 诊断召回：expected_song_id 是否进入 top1/top-k。
         top1_hit = bool(expected_song_id) and top1_song_id == expected_song_id
@@ -157,6 +175,7 @@ def main() -> None:
             "top1_song_id": top1_song_id,
             "top1_similarity": top1_sim,
             "top1_source": top1_source,
+            "dejavu_aligned_count": dejavu_aligned_count if dejavu_aligned_count is not None else "",
             "top1_hit": top1_hit,
             "topk_hit": topk_hit,
             "expected_rank": expected_rank,
@@ -166,11 +185,19 @@ def main() -> None:
             "expected_duplicate": expected_dup,
             "predicted_duplicate": predicted_dup,
             "correct": correct,
+            "query_time_ms": query_time_ms,
+            "chroma_extract_ms": timings.get("chroma_extract_ms", ""),
+            "db_cosine_ms": timings.get("db_cosine_ms", ""),
+            "db_fetch_ms": timings.get("db_fetch_ms", ""),
+            "dtw_ms": timings.get("dtw_ms", ""),
+            "dejavu_decode_ms": timings.get("dejavu_decode_ms", ""),
+            "dejavu_fingerprint_ms": timings.get("dejavu_fingerprint_ms", ""),
+            "dejavu_db_ms": timings.get("dejavu_db_ms", ""),
             "error": "",
         })
 
         logger.info(
-            "[%d/%d] variant=%s source=%s expected=%s predicted_duplicate=%s threshold=%.4f expected_song_id=%s top1=%s sim=%s top1_hit=%s topk_hit=%s expected_rank=%s expected_sim=%s correct=%s",
+            "[%d/%d] variant=%s source=%s expected=%s predicted_duplicate=%s threshold=%.4f expected_song_id=%s top1=%s sim=%s top1_hit=%s topk_hit=%s expected_rank=%s expected_sim=%s correct=%s time_ms=%s",
             i,
             len(rows),
             row.get("variant", ""),
@@ -186,6 +213,7 @@ def main() -> None:
             expected_rank if expected_rank != "" else "-",
             expected_similarity if expected_similarity != "" else "-",
             correct,
+            query_time_ms,
         )
 
         if i % 10 == 0 or i == len(rows):
@@ -194,9 +222,14 @@ def main() -> None:
     # 写逐条结果
     fieldnames = ["query_song_id", "audio_song_id", "audio_path", "variant", "sample_class",
                   "expected_song_id", "expected", "top1_song_id", "top1_similarity", "top1_source",
+                  "dejavu_aligned_count",
                   "top1_hit", "topk_hit", "expected_rank", "expected_similarity",
                   "invalid_negative_pair", "invalid_boolean_sample",
-                  "expected_duplicate", "predicted_duplicate", "correct", "error"]
+                  "expected_duplicate", "predicted_duplicate", "correct",
+                  "query_time_ms",
+                  "chroma_extract_ms", "db_cosine_ms", "db_fetch_ms", "dtw_ms",
+                  "dejavu_decode_ms", "dejavu_fingerprint_ms", "dejavu_db_ms",
+                  "error"]
     with out_path.open("w", newline="", encoding="utf-8") as f:
         writer = csv.DictWriter(f, fieldnames=fieldnames)
         writer.writeheader()

@@ -6,11 +6,11 @@
 
 用法:
 python scripts/generate_composition_testset.py \
-    --audio-dir /Volumes/移动硬盘/lyric_audio_type11 \
-    --negative-audio-dir /Volumes/移动硬盘/composition_test \
-    --out-dir composition_dedup/composition_testset \
-    --num-songs 80 \
-    --num-negative-songs 40 \
+    --audio-dir /Volumes/移动硬盘/composition_test \
+    --negative-audio-dir /Volumes/移动硬盘/composition_drop \
+    --out-dir composition_testset \
+    --num-songs 100 \
+    --num-negative-songs 100 \
     --negative-variants \
     --seed 123
 

@@ -16,6 +16,9 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
+from dotenv import load_dotenv
+load_dotenv(Path(__file__).resolve().parent.parent / ".env")
+
 from tqdm import tqdm
 
 from composition_dedup.service import CompositionConfig, CompositionDedupService