Skip to content

沈秋雨 / lyric_rhyme

Go to a project
Toggle navigation
Toggle navigation pinning
Commit d742438b d742438be22d9c6759c2f9903ea070028b5e41ab by 沈秋雨
Options

同步最新评测流程

1 parent 5b7fb8f7
Inline Side-by-side
Showing 5 changed files with 255 additions and 40 deletions
1 #!/usr/bin/env python3
2 """创建阿里云 ICE DNA 库（Model=Audio）并输出 DBId。
3
4 用法:
5 python scripts/aliyun_dna/create_dna_library.py --name "测试库-20260627"
6 python scripts/aliyun_dna/create_dna_library.py --name "prod-audio-lib" --desc "生产音频指纹库"
7
8 创建成功后将 DBId 写入 .env（ALIYUN_DNA_DB_ID）或手动复制使用。
9 """
10
11 import argparse
12 import logging
13 import os
14 import sys
15 from pathlib import Path
16
17 from dotenv import load_dotenv
18
19 load_dotenv(Path(__file__).resolve().parents[2] / ".env")
20
21 from alibabacloud_ice20201109.client import Client as ICEClient
22 from alibabacloud_ice20201109 import models as ice_models
23 from alibabacloud_tea_openapi.models import Config as OpenApiConfig
24
25 logger = logging.getLogger(__name__)
26
27
28 def get_ice_client() -> ICEClient:
29 config = OpenApiConfig(
30 access_key_id=os.environ["ALIYUN_ICE_ACCESS_KEY_ID"],
31 access_key_secret=os.environ["ALIYUN_ICE_ACCESS_KEY_SECRET"],
32 endpoint=f"ice.{os.environ.get('ALIYUN_ICE_REGION', 'cn-hangzhou')}.aliyuncs.com",
33 region_id=os.environ.get("ALIYUN_ICE_REGION", "cn-hangzhou"),
34 )
35 return ICEClient(config)
36
37
38 def main() -> None:
39 logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
40
41 ap = argparse.ArgumentParser(description="创建阿里云 ICE DNA 库")
42 ap.add_argument("--name", required=True, help="DNA 库名称")
43 ap.add_argument("--desc", default="", help="DNA 库描述")
44 ap.add_argument(
45 "--write-env", action="store_true",
46 help="创建成功后自动将 DBId 写入项目根目录的 .env（覆盖 ALIYUN_DNA_DB_ID）",
47 )
48 args = ap.parse_args()
49
50 client = get_ice_client()
51 request = ice_models.CreateDNADBRequest(
52 name=args.name,
53 model="Audio",
54 description=args.desc or f"音频 DNA 库：{args.name}",
55 )
56
57 try:
58 resp = client.create_dnadb(request)
59 db_info = resp.body.dbinfo
60 db_id = db_info.dbid
61 status = db_info.status
62 except Exception as e:
63 logger.error("创建 DNA 库失败: %s", e)
64 sys.exit(1)
65
66 logger.info("创建成功！")
67 logger.info(" 名称: %s", db_info.name)
68 logger.info(" DBId: %s", db_id)
69 logger.info(" 状态: %s（offline → active 通常需要几分钟）", status)
70
71 if args.write_env:
72 env_path = Path(__file__).resolve().parents[2] / ".env"
73 text = env_path.read_text(encoding="utf-8")
74 if "ALIYUN_DNA_DB_ID=" in text:
75 import re
76 text = re.sub(r"ALIYUN_DNA_DB_ID=.*", f"ALIYUN_DNA_DB_ID={db_id}", text)
77 else:
78 text += f"\nALIYUN_DNA_DB_ID={db_id}\n"
79 env_path.write_text(text, encoding="utf-8")
80 logger.info("已将 ALIYUN_DNA_DB_ID=%s 写入 %s", db_id, env_path)
81 else:
82 print(f"\nDBId: {db_id}")
83 print(f"可用 --write-env 自动写入 .env，或手动设置：")
84 print(f" ALIYUN_DNA_DB_ID={db_id}")
85
86
87 if __name__ == "__main__":
88 main()
1 调用CreateDNADB创建DNA库。
2
3 ## 接口说明
4
5 单个用户默认可以创建指纹库个数为 5 个，如需提高，请[提交工单](https://smartservice.console.aliyun.com/service/create-ticket?spm=a2c4g.11186623.0.0.645019b6Btnu4q)申请。无用指纹库可以调用 DeleteDNADB 接口删除。
6
7 ## 调试
8
9 [您可以在OpenAPI Explorer中直接运行该接口，免去您计算签名的困扰。运行成功后，OpenAPI Explorer可以自动生成SDK代码示例。](https://api.aliyun.com/api/ICE/2020-11-09/CreateDNADB)
10
11 [![](https://img.alicdn.com/tfs/TB16JcyXHr1gK0jSZR0XXbP8XXa-24-26.png) 调试](https://api.aliyun.com/api/ICE/2020-11-09/CreateDNADB)
12
13 ## **授权信息**
14
15 下表是API对应的授权信息，可以在RAM权限策略语句的`Action`元素中使用，用来给RAM用户或RAM角色授予调用此API的权限。具体说明如下：
16
17 - 操作：是指具体的权限点。
18
19 - 访问级别：是指每个操作的访问级别，取值为写入（Write）、读取（Read）或列出（List）。
20
21 - 资源类型：是指操作中支持授权的资源类型。具体说明如下：
22
23 - 对于必选的资源类型，用前面加 \* 表示。
24
25 - 对于不支持资源级授权的操作，用`全部资源`表示。
26
27 - 条件关键字：是指云产品自身定义的条件关键字。
28
29 - 关联操作：是指成功执行操作所需要的其他权限。操作者必须同时具备关联操作的权限，操作才能成功。
30
31
32 | **操作** | **访问级别** | **资源类型** | **条件关键字** | **关联操作** |
33 | --- | --- | --- | --- | --- |
34 | ice:CreateDNADB | | \\*全部资源 `*` | 无 | 无 |
35
36 ## 请求参数
37
38 | **名称** | **类型** | **必填** | **描述** | **示例值** |
39 | --- | --- | --- | --- | --- |
40 | Name | string | 是 | DNA 库名称。 | example name |
41 | Model | string | 否 | DNA 库模型。包含： - **Video**：视频 - **Audio**：音频 - **Image**：图片（废弃） - **Text**：文本【仅上海区域支持】（废弃） | Video |
42 | Description | string | 否 | DNA 库描述。 | 这是一个视频DNA库。 |
43
44 ## **返回参数**
45
46 | **名称** | **类型** | **描述** | **示例值** |
47 | --- | --- | --- | --- |
48 | | object | Schema of Response | |
49 | RequestId | string | 请求 ID。 | 25818875-5F78-4A13-BEF6-D7393642CA58 |
50 | DBInfo | object | DNA 库详细信息。 | |
51 | Description | string | DNA 库描述。 | 这是一个视频DNA库。 |
52 | Name | string | DNA 库名称。 | example name |
53 | Status | string | DNA 库的状态。新建成功后默认为 offline 状态，待后台处理完成后会自动转换成 active 状态。 | offline |
54 | DBId | string | DNA 库 ID。建议您保存此 ID 便于后续调用其他相关接口时使用。 | 88c6ca184c0e47098a5b665e2a12\\*\\*\\*\\* |
55 | Model | string | DNA 库模型。 | Video |
56
57 ## 示例
58
59 正常返回示例
60
61 `JSON`格式
62
63 ```
64 {
65 "RequestId": "25818875-5F78-4A13-BEF6-D7393642CA58",
66 "DBInfo": {
67 "Description": "这是一个视频DNA库。",
68 "Name": "example name",
69 "Status": "offline",
70 "DBId": "88c6ca184c0e47098a5b665e2a12****",
71 "Model": "Video"
72 }
73 }
74 ```
75
76 ## 错误码
77
78 访问[错误中心](https://api.aliyun.com/document/ICE/2020-11-09/errorCode)查看更多错误码。
79
80 ## **变更历史**
81
82 更多信息，参考[变更详情](https://api.aliyun.com/document/ICE/2020-11-09/CreateDNADB#workbench-doc-change-demo)。
...\ No newline at end of file ...\ No newline at end of file
...@@ -63,28 +63,27 @@ DEFAULT_SAMPLE_RATE = 16000 # 默认降采样到 16kHz ...@@ -63,28 +63,27 @@ DEFAULT_SAMPLE_RATE = 16000 # 默认降采样到 16kHz
63 63
64 64
65 def _resample_audio(audio_path: str, target_sr: int = DEFAULT_SAMPLE_RATE) -> str | None: 65 def _resample_audio(audio_path: str, target_sr: int = DEFAULT_SAMPLE_RATE) -> str | None:
66 """降采样音频到目标采样率（单声道），返回临时文件路径。""" 66 """用 ffmpeg 将音频转换为目标采样率的单声道 WAV，返回临时文件路径。"""
67 try: 67 import subprocess
68 import librosa
69 import soundfile as sf
70
71 y, sr = librosa.load(audio_path, sr=target_sr, mono=True)
72 tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) 68 tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
73 sf.write(tmp.name, y, target_sr, subtype="PCM_16")
74 tmp.close() 69 tmp.close()
75 70 cmd = [
71 "ffmpeg", "-y", "-i", audio_path,
72 "-ar", str(target_sr), "-ac", "1",
73 "-sample_fmt", "s16",
74 tmp.name,
75 ]
76 result = subprocess.run(cmd, capture_output=True)
77 if result.returncode != 0:
78 Path(tmp.name).unlink(missing_ok=True)
79 logger.warning(" ffmpeg 转换失败: %s", result.stderr.decode(errors="replace").strip()[-200:])
80 return None
76 orig_size = Path(audio_path).stat().st_size 81 orig_size = Path(audio_path).stat().st_size
77 new_size = Path(tmp.name).stat().st_size 82 new_size = Path(tmp.name).stat().st_size
78 ratio = (1 - new_size / orig_size) * 100 if orig_size > 0 else 0 83 logger.info(" 转码: %s -> %dHz WAV, %s -> %s",
79 logger.info(" 降采样: %dHz -> %dHz, 大小 %s -> %s (减少 %.0f%%)", 84 Path(audio_path).suffix, target_sr,
80 sr, target_sr, _human_size(orig_size), _human_size(new_size), ratio) 85 _human_size(orig_size), _human_size(new_size))
81 return tmp.name 86 return tmp.name
82 except ImportError:
83 logger.warning(" 缺少 librosa/soundfile，跳过降采样")
84 return None
85 except Exception as e:
86 logger.warning(" 降采样失败: %s, 使用原始文件", e)
87 return None
88 87
89 88
90 def _human_size(n: int) -> str: 89 def _human_size(n: int) -> str:
...@@ -236,8 +235,14 @@ def main() -> None: ...@@ -236,8 +235,14 @@ def main() -> None:
236 help="每处理 N 条就追加保存一次结果（默认 10，0 表示只在结束时保存）") 235 help="每处理 N 条就追加保存一次结果（默认 10，0 表示只在结束时保存）")
237 parser.add_argument("--concurrency", type=int, default=1, 236 parser.add_argument("--concurrency", type=int, default=1,
238 help="并发查询数（默认 1，建议不超过 8 以免触发限流）") 237 help="并发查询数（默认 1，建议不超过 8 以免触发限流）")
238 parser.add_argument("--db-id", default="",
239 help="覆盖 .env 中的 ALIYUN_DNA_DB_ID，用于指定测试库")
239 args = parser.parse_args() 240 args = parser.parse_args()
240 241
242 # --db-id 优先于环境变量
243 if args.db_id:
244 os.environ["ALIYUN_DNA_DB_ID"] = args.db_id
245
241 # 验证配置 246 # 验证配置
242 required_env = ["ALIYUN_ICE_ACCESS_KEY_ID", "ALIYUN_ICE_ACCESS_KEY_SECRET", 247 required_env = ["ALIYUN_ICE_ACCESS_KEY_ID", "ALIYUN_ICE_ACCESS_KEY_SECRET",
243 "ALIYUN_OSS_BUCKET", "ALIYUN_OSS_ENDPOINT", "ALIYUN_DNA_DB_ID"] 248 "ALIYUN_OSS_BUCKET", "ALIYUN_OSS_ENDPOINT", "ALIYUN_DNA_DB_ID"]
...@@ -246,6 +251,8 @@ def main() -> None: ...@@ -246,6 +251,8 @@ def main() -> None:
246 logger.error("缺少环境变量: %s", key) 251 logger.error("缺少环境变量: %s", key)
247 sys.exit(1) 252 sys.exit(1)
248 253
254 logger.info("使用 DNA 库: %s", os.environ["ALIYUN_DNA_DB_ID"])
255
249 with open(args.queries, newline="", encoding="utf-8") as f: 256 with open(args.queries, newline="", encoding="utf-8") as f:
250 rows = list(csv.DictReader(f)) 257 rows = list(csv.DictReader(f))
251 258
...@@ -266,7 +273,9 @@ def main() -> None: ...@@ -266,7 +273,9 @@ def main() -> None:
266 out_path = Path(args.out) 273 out_path = Path(args.out)
267 out_path.parent.mkdir(parents=True, exist_ok=True) 274 out_path.parent.mkdir(parents=True, exist_ok=True)
268 275
269 fieldnames = ["query_song_id", "audio_song_id", "audio_path", "variant", "sample_class", 276 fieldnames = ["query_song_id", "audio_song_id", "song_name", "original_singer",
277 "singer_name", "platform_name",
278 "audio_path", "variant", "sample_class",
270 "expected_song_id", "expected", "top1_song_id", "top1_similarity", 279 "expected_song_id", "expected", "top1_song_id", "top1_similarity",
271 "top1_hit", "topk_hit", "expected_rank", "expected_similarity", 280 "top1_hit", "topk_hit", "expected_rank", "expected_similarity",
272 "expected_duplicate", "predicted_duplicate", "correct", 281 "expected_duplicate", "predicted_duplicate", "correct",
...@@ -308,6 +317,11 @@ def main() -> None: ...@@ -308,6 +317,11 @@ def main() -> None:
308 audio_song_id = _song_id_from_audio_path(audio_path) 317 audio_song_id = _song_id_from_audio_path(audio_path)
309 expected_song_id = str(row.get("expected_song_id", "")) 318 expected_song_id = str(row.get("expected_song_id", ""))
310 expected_dup = row.get("expected", "").strip().lower() == "duplicate" 319 expected_dup = row.get("expected", "").strip().lower() == "duplicate"
320 # 从 queries.csv 透传的元数据
321 song_name = row.get("song_name", "")
322 original_singer = row.get("original_singer", "")
323 singer_name = row.get("singer_name", "")
324 platform_name = row.get("platform_name", "")
311 325
312 upload_path = audio_path 326 upload_path = audio_path
313 tmp_resampled = None 327 tmp_resampled = None
...@@ -385,6 +399,10 @@ def main() -> None: ...@@ -385,6 +399,10 @@ def main() -> None:
385 return { 399 return {
386 "query_song_id": query_song_id, 400 "query_song_id": query_song_id,
387 "audio_song_id": audio_song_id, 401 "audio_song_id": audio_song_id,
402 "song_name": song_name,
403 "original_singer": original_singer,
404 "singer_name": singer_name,
405 "platform_name": platform_name,
388 "audio_path": audio_path, 406 "audio_path": audio_path,
389 "variant": row.get("variant", ""), 407 "variant": row.get("variant", ""),
390 "sample_class": row.get("sample_class", ""), 408 "sample_class": row.get("sample_class", ""),
...@@ -411,6 +429,10 @@ def main() -> None: ...@@ -411,6 +429,10 @@ def main() -> None:
411 return { 429 return {
412 "query_song_id": query_song_id, 430 "query_song_id": query_song_id,
413 "audio_song_id": audio_song_id, 431 "audio_song_id": audio_song_id,
432 "song_name": song_name,
433 "original_singer": original_singer,
434 "singer_name": singer_name,
435 "platform_name": platform_name,
414 "audio_path": audio_path, 436 "audio_path": audio_path,
415 "variant": row.get("variant", ""), 437 "variant": row.get("variant", ""),
416 "sample_class": row.get("sample_class", ""), 438 "sample_class": row.get("sample_class", ""),
......
...@@ -60,29 +60,27 @@ DEFAULT_SAMPLE_RATE = 16000 # 默认降采样到 16kHz ...@@ -60,29 +60,27 @@ DEFAULT_SAMPLE_RATE = 16000 # 默认降采样到 16kHz
60 60
61 61
62 def _resample_audio(audio_path: str, target_sr: int = DEFAULT_SAMPLE_RATE) -> str | None: 62 def _resample_audio(audio_path: str, target_sr: int = DEFAULT_SAMPLE_RATE) -> str | None:
63 """降采样音频到目标采样率（单声道），返回临时文件路径。""" 63 """用 ffmpeg 将音频转换为目标采样率的单声道 WAV，返回临时文件路径。"""
64 try: 64 import subprocess
65 import librosa
66 import soundfile as sf
67
68 y, sr = librosa.load(audio_path, sr=target_sr, mono=True)
69 tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) 65 tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
70 sf.write(tmp.name, y, target_sr, subtype="PCM_16")
71 tmp.close() 66 tmp.close()
72 67 cmd = [
68 "ffmpeg", "-y", "-i", audio_path,
69 "-ar", str(target_sr), "-ac", "1",
70 "-sample_fmt", "s16",
71 tmp.name,
72 ]
73 result = subprocess.run(cmd, capture_output=True)
74 if result.returncode != 0:
75 Path(tmp.name).unlink(missing_ok=True)
76 logger.warning(" ffmpeg 转换失败: %s", result.stderr.decode(errors="replace").strip()[-200:])
77 return None
73 orig_size = Path(audio_path).stat().st_size 78 orig_size = Path(audio_path).stat().st_size
74 new_size = Path(tmp.name).stat().st_size 79 new_size = Path(tmp.name).stat().st_size
75 ratio = (1 - new_size / orig_size) * 100 if orig_size > 0 else 0 80 logger.info(" 转码: %s -> %dHz WAV, %s -> %s",
76 logger.info(" 降采样: %dHz -> %dHz, 大小 %s -> %s (减少 %.0f%%)", 81 Path(audio_path).suffix, target_sr,
77 sr, target_sr, 82 _human_size(orig_size), _human_size(new_size))
78 _human_size(orig_size), _human_size(new_size), ratio)
79 return tmp.name 83 return tmp.name
80 except ImportError:
81 logger.warning(" 缺少 librosa/soundfile，跳过降采样")
82 return None
83 except Exception as e:
84 logger.warning(" 降采样失败: %s, 使用原始文件", e)
85 return None
86 84
87 85
88 def _human_size(n: int) -> str: 86 def _human_size(n: int) -> str:
...@@ -278,6 +276,8 @@ def main() -> None: ...@@ -278,6 +276,8 @@ def main() -> None:
278 parser.add_argument("--dry-run", action="store_true", help="只列出待上传文件，不实际操作") 276 parser.add_argument("--dry-run", action="store_true", help="只列出待上传文件，不实际操作")
279 parser.add_argument("--save-type", default="save", choices=["save", "forcesave", "onlysave"], 277 parser.add_argument("--save-type", default="save", choices=["save", "forcesave", "onlysave"],
280 help="DNA 存储类型（默认 save=去重入库）") 278 help="DNA 存储类型（默认 save=去重入库）")
279 parser.add_argument("--db-id", default="",
280 help="覆盖 .env 中的 ALIYUN_DNA_DB_ID，用于指定新建的测试库")
281 parser.add_argument("--sample-rate", type=int, default=DEFAULT_SAMPLE_RATE, 281 parser.add_argument("--sample-rate", type=int, default=DEFAULT_SAMPLE_RATE,
282 help=f"降采样目标采样率（默认 {DEFAULT_SAMPLE_RATE}Hz）") 282 help=f"降采样目标采样率（默认 {DEFAULT_SAMPLE_RATE}Hz）")
283 parser.add_argument("--no-resample", action="store_true", 283 parser.add_argument("--no-resample", action="store_true",
...@@ -294,6 +294,10 @@ def main() -> None: ...@@ -294,6 +294,10 @@ def main() -> None:
294 help="下载音频 URL 的超时秒数（默认 60）") 294 help="下载音频 URL 的超时秒数（默认 60）")
295 args = parser.parse_args() 295 args = parser.parse_args()
296 296
297 # --db-id 优先于环境变量
298 if args.db_id:
299 os.environ["ALIYUN_DNA_DB_ID"] = args.db_id
300
297 # 验证配置 301 # 验证配置
298 required_env = ["ALIYUN_ICE_ACCESS_KEY_ID", "ALIYUN_ICE_ACCESS_KEY_SECRET", 302 required_env = ["ALIYUN_ICE_ACCESS_KEY_ID", "ALIYUN_ICE_ACCESS_KEY_SECRET",
299 "ALIYUN_OSS_BUCKET", "ALIYUN_OSS_ENDPOINT", "ALIYUN_DNA_DB_ID"] 303 "ALIYUN_OSS_BUCKET", "ALIYUN_OSS_ENDPOINT", "ALIYUN_DNA_DB_ID"]
...@@ -302,6 +306,8 @@ def main() -> None: ...@@ -302,6 +306,8 @@ def main() -> None:
302 logger.error("缺少环境变量: %s", key) 306 logger.error("缺少环境变量: %s", key)
303 sys.exit(1) 307 sys.exit(1)
304 308
309 logger.info("使用 DNA 库: %s", os.environ["ALIYUN_DNA_DB_ID"])
310
305 use_db_mode = bool(args.db_table) 311 use_db_mode = bool(args.db_table)
306 312
307 if use_db_mode: 313 if use_db_mode:
......
...@@ -453,18 +453,35 @@ def main() -> None: ...@@ -453,18 +453,35 @@ def main() -> None:
453 "genre": r["genre"], 453 "genre": r["genre"],
454 "language_tag": r["language_tag"], 454 "language_tag": r["language_tag"],
455 }) 455 })
456 else: 456 # 正样本：主版本自身查询（验证入库是否成功）
457 platform = r["platform_name"] or "unknown"
458 query_rows.append({ 457 query_rows.append({
459 "song_id": r["song_id"], 458 "song_id": r["song_id"],
460 "audio_path": r["local_path"], 459 "audio_path": r["local_path"],
461 "variant": f"cover_{platform}", 460 "variant": "self",
462 "sample_class": "positive", 461 "sample_class": "positive",
463 "expected_song_id": r["song_id"], 462 "expected_song_id": r["song_id"],
464 "expected": "duplicate", 463 "expected": "duplicate",
465 "song_name": r["song_name"], 464 "song_name": r["song_name"],
466 "original_singer": r["original_singer"], 465 "original_singer": r["original_singer"],
467 "singer_name": r["singer_name"], 466 "singer_name": r["original_singer"],
467 "platform_name": r["platform_name"],
468 "duration": r["duration"],
469 "is_main_version": r["is_main_version"],
470 })
471 else:
472 # 负样本：翻唱/其他版本，不应匹配到 DNA 库中的主版本
473 platform = r["platform_name"] or "unknown"
474 singer = r["singer_name"] or "unknown"
475 query_rows.append({
476 "song_id": r["song_id"],
477 "audio_path": r["local_path"],
478 "variant": f"cover_{platform}",
479 "sample_class": "negative",
480 "expected_song_id": "",
481 "expected": "not_duplicate",
482 "song_name": r["song_name"],
483 "original_singer": r["original_singer"],
484 "singer_name": singer,
468 "platform_name": r["platform_name"], 485 "platform_name": r["platform_name"],
469 "duration": r["duration"], 486 "duration": r["duration"],
470 "is_main_version": r["is_main_version"], 487 "is_main_version": r["is_main_version"],
......