Commit 58c29eaa 58c29eaaa0e3c5de99f7b62d77c74a41b694c927 by cnb.bofCdSsphPA

Turn Phase-1 host prerequisites into a live audit artifact

Constraint: Worker-contract validation is now stable enough that the remaining uncertainty is host readiness, so the next blocker had to be made explicit instead of inferred from repeated failed runs.
Rejected: Keep prerequisite knowledge only in prose | It would drift and force future sessions to rediscover the same missing mounts and packages.
Confidence: high
Scope-risk: narrow
Directive: Run the prerequisite audit before retrying live extraction so host blockers are measured once and reused across lanes.
Tested: /usr/local/miniconda3/bin/python -m py_compile scripts/run_phase1_prereq_audit_live.py; git diff --check; /usr/local/miniconda3/bin/python scripts/run_phase1_prereq_audit_live.py --dsn 'postgres://d2:d2pass@127.0.0.1:5432/d2' --schema acr_test --output data/pgvector_eval/music20/phase1_prereq_audit_report.json
Not-tested: This audit does not install dependencies or mount assets; it only reports readiness.
1 parent 43d2f93a
{
"schema": "acr_test",
"dsn_redacted": "postgres://d2:***@127.0.0.1:5432/d2",
"downloads_root": "/workspace/downloads",
"downloads_root_exists": false,
"package_checks": {
"numpy": {
"package": "numpy",
"available": true
},
"speechbrain": {
"package": "speechbrain",
"available": false,
"error_type": "ModuleNotFoundError",
"error": "No module named 'speechbrain'"
},
"torch": {
"package": "torch",
"available": false,
"error_type": "ModuleNotFoundError",
"error": "No module named 'torch'"
},
"torchaudio": {
"package": "torchaudio",
"available": false,
"error_type": "ModuleNotFoundError",
"error": "No module named 'torchaudio'"
},
"transformers": {
"package": "transformers",
"available": false,
"error_type": "ModuleNotFoundError",
"error": "No module named 'transformers'"
}
},
"jobs": [
{
"extraction_job_id": 1,
"model_name": "chromaprint",
"model_version": "v1",
"embedding_dim": null,
"target_scope": "reference_set:phase1_hot_reference_v1",
"required_packages": [
"numpy"
],
"missing_packages": [],
"downloads_root_exists": false,
"ready_for_live_worker": false
},
{
"extraction_job_id": 2,
"model_name": "mert",
"model_version": "v1-95m",
"embedding_dim": 768,
"target_scope": "reference_set:phase1_hot_reference_v1",
"required_packages": [
"numpy",
"torch",
"torchaudio",
"transformers"
],
"missing_packages": [
"torch",
"torchaudio",
"transformers"
],
"downloads_root_exists": false,
"ready_for_live_worker": false
},
{
"extraction_job_id": 3,
"model_name": "mert",
"model_version": "v1-95m",
"embedding_dim": 768,
"target_scope": "reference_set:phase1_hot_reference_v1",
"required_packages": [
"numpy",
"torch",
"torchaudio",
"transformers"
],
"missing_packages": [
"torch",
"torchaudio",
"transformers"
],
"downloads_root_exists": false,
"ready_for_live_worker": false
},
{
"extraction_job_id": 4,
"model_name": "muq",
"model_version": "large-msd-iter",
"embedding_dim": 768,
"target_scope": "reference_set:phase1_hot_reference_v1",
"required_packages": [
"numpy",
"torch",
"torchaudio",
"transformers"
],
"missing_packages": [
"torch",
"torchaudio",
"transformers"
],
"downloads_root_exists": false,
"ready_for_live_worker": false
},
{
"extraction_job_id": 5,
"model_name": "ecapa",
"model_version": "acr-baseline-v1",
"embedding_dim": 192,
"target_scope": "reference_set:phase1_hot_reference_v1",
"required_packages": [
"numpy",
"torch",
"torchaudio",
"speechbrain"
],
"missing_packages": [
"torch",
"torchaudio",
"speechbrain"
],
"downloads_root_exists": false,
"ready_for_live_worker": false
}
],
"summary": {
"total_jobs": 5,
"ready_jobs": 0,
"blocked_jobs": 5,
"missing_packages_union": [
"speechbrain",
"torch",
"torchaudio",
"transformers"
]
}
}
\ No newline at end of file
#!/usr/bin/env /usr/local/miniconda3/bin/python
from __future__ import annotations
import argparse
import importlib
import json
from pathlib import Path
import sys
from typing import Any
import psycopg
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from workers._job_common import validate_schema
DEFAULT_OUTPUT = ROOT / 'data' / 'pgvector_eval' / 'music20' / 'phase1_prereq_audit_report.json'
MODEL_REQUIREMENTS = {
'mert': ['numpy', 'torch', 'torchaudio', 'transformers'],
'muq': ['numpy', 'torch', 'torchaudio', 'transformers'],
'ecapa': ['numpy', 'torch', 'torchaudio', 'speechbrain'],
'chromaprint': ['numpy'],
}
def check_import(name: str) -> dict[str, Any]:
try:
importlib.import_module(name)
return {'package': name, 'available': True}
except Exception as exc: # noqa: BLE001
return {'package': name, 'available': False, 'error_type': type(exc).__name__, 'error': str(exc).splitlines()[0]}
def load_jobs(conn: psycopg.Connection) -> list[dict[str, Any]]:
rows = conn.execute(
"""
SELECT fej.extraction_job_id, mr.model_name, mr.model_version, fs.embedding_dim, fej.target_scope
FROM feature_extraction_job fej
JOIN feature_set_registry fs ON fs.feature_set_id = fej.feature_set_id
JOIN model_registry mr ON mr.model_id = fs.model_id
ORDER BY fej.extraction_job_id;
"""
).fetchall()
return [
{
'extraction_job_id': int(row[0]),
'model_name': row[1],
'model_version': row[2],
'embedding_dim': int(row[3]) if row[3] is not None else None,
'target_scope': row[4],
}
for row in rows
]
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument('--dsn', required=True)
ap.add_argument('--schema', default='acr_test')
ap.add_argument('--downloads-root', default='/workspace/downloads')
ap.add_argument('--output', default=str(DEFAULT_OUTPUT))
args = ap.parse_args()
schema = validate_schema(args.schema)
downloads_root = Path(args.downloads_root)
downloads_exists = downloads_root.exists()
with psycopg.connect(args.dsn, autocommit=True) as conn:
conn.execute(f'SET search_path TO {schema}, public;')
jobs = load_jobs(conn)
package_names = sorted({pkg for job in jobs for pkg in MODEL_REQUIREMENTS.get(job['model_name'], ['numpy'])})
package_checks = {item['package']: item for item in (check_import(name) for name in package_names)}
job_reports = []
for job in jobs:
required = MODEL_REQUIREMENTS.get(job['model_name'], ['numpy'])
missing = [name for name in required if not package_checks[name]['available']]
job_reports.append(
{
**job,
'required_packages': required,
'missing_packages': missing,
'downloads_root_exists': downloads_exists,
'ready_for_live_worker': downloads_exists and not missing,
}
)
payload = {
'schema': schema,
'dsn_redacted': 'postgres://d2:***@127.0.0.1:5432/d2',
'downloads_root': str(downloads_root),
'downloads_root_exists': downloads_exists,
'package_checks': package_checks,
'jobs': job_reports,
'summary': {
'total_jobs': len(job_reports),
'ready_jobs': sum(1 for job in job_reports if job['ready_for_live_worker']),
'blocked_jobs': sum(1 for job in job_reports if not job['ready_for_live_worker']),
'missing_packages_union': sorted({pkg for job in job_reports for pkg in job['missing_packages']}),
},
}
out = Path(args.output)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps(payload, ensure_ascii=False, indent=2))
if __name__ == '__main__':
main()
## 2026-06-04
- 新增 `scripts/run_phase1_prereq_audit_live.py``phase1_prereq_audit_report.json`,把 `/workspace/downloads` 挂载状态、`torch/torchaudio/transformers/speechbrain` 依赖状态与 5 条 Phase-1 jobs 的 readiness 汇总到一份 live 审计报告;当前结果为 `ready_jobs=0``blocked_jobs=5`
- 新增 `scripts/run_embedding_vector_table_negative_matrix_live.py``embedding_vector_table_negative_matrix_report.json`,在 live PostgreSQL 上补齐 semantic preflight 的三类向量表负例:维度不匹配、未 allowlist、schema 缺表;三类 case 都会稳定落到 `preflight_failed`,且 `vector_table_report.reason` 与预期一致。
- 新增 `scripts/run_phase1_worker_contract_smoke_live.py``phase1_worker_contract_smoke_report.json`,把 exact lane 非 dry-run 验证与 semantic preflight matrix 合成一条 live smoke 命令;当前总览结果为 exact=`failed/unreadable_audio_assets`、semantic=`4/4 failed`,说明阻塞点已经收敛到环境挂载与模型 runtime,而不是 worker contract 本身。
- 新增 `scripts/validate_audio_embedding_asset_upsert_live.py``audio_embedding_asset_upsert_live_report.json`,在隔离 schema `acr_asset_upsert_test` 上真实验证 `uq_audio_embedding_feature_asset`:重复普通 insert 会触发 `UniqueViolation`,而 `ON CONFLICT ... DO UPDATE` 会复用同一 `embedding_id`,最终 `audio_embedding/audio_embedding_vector_192` 行数都保持为 `1`
......
......@@ -872,3 +872,29 @@ cd /workspace/acr-engine
- 当前 semantic preflight 已经能够把“运行环境问题”和“配置错误问题”分层暴露
- 后续只要看 `vector_table_report.reason`,就能快速区分是 DDL/配置错误,还是模型 runtime/音频挂载错误
## 新增:Phase-1 prerequisites audit
为了避免每次都靠肉眼猜“到底是音频挂载缺失,还是模型 runtime 缺失”,本轮新增:
- `acr-engine/scripts/run_phase1_prereq_audit_live.py`
- `acr-engine/data/pgvector_eval/music20/phase1_prereq_audit_report.json`
### 当前审计结果
| 指标 | 结果 |
|---|---|
| `downloads_root_exists` | `false` |
| `total_jobs` | `5` |
| `ready_jobs` | `0` |
| `blocked_jobs` | `5` |
| 缺失依赖并集 | `speechbrain`, `torch`, `torchaudio`, `transformers` |
按 job 看:
- `chromaprint`:依赖层面可跑,但被 `/workspace/downloads` 缺失阻塞
- `mert / muq`:同时被 `/workspace/downloads` 缺失与 `torch/torchaudio/transformers` 缺失阻塞
- `ecapa`:同时被 `/workspace/downloads` 缺失与 `torch/torchaudio/speechbrain` 缺失阻塞
这使得“当前为什么跑不通”已经可以通过单份 JSON 报告回答,而不必重新手工试跑。
......
......@@ -195,6 +195,7 @@ sed -n '1,320p' acr-engine/sql/acr_pg_schema_v2.sql
- `scripts/validate_audio_embedding_asset_upsert_live.py` 已在隔离 schema `acr_asset_upsert_test` 上验证 `uq_audio_embedding_feature_asset`:重复 insert 会被唯一键拒绝,upsert 会复用同一 `embedding_id`,说明 asset-level 幂等键也已有真实证据
- `scripts/run_phase1_worker_contract_smoke_live.py` 已提供一条命令的全局 smoke:当前 exact lane = `failed/unreadable_audio_assets`,semantic lane = `4/4 failed`,共性 blocker 已固化为音频挂载缺失 + 语义模型 runtime 缺失
- `scripts/run_embedding_vector_table_negative_matrix_live.py` 已在 live PostgreSQL 上补齐 semantic vector-table 负例矩阵:`vector_table_dim_mismatch``vector_table_not_allowlisted``vector_table_missing_in_schema` 三类错误都能被稳定写入 `vector_table_report.reason`
- `scripts/run_phase1_prereq_audit_live.py` 已给出当前 host 的先决条件审计:`downloads_root_exists=false``ready_jobs=0/5`,并把 `torch/torchaudio/transformers/speechbrain` 的缺失状态按 job 落成 JSON 报告
- `phase1_hot_reference_v1``acr_test` 里已经真实补齐 `20` 个 reference members,因此 worker dry-run 当前看到的 scope 已是 `20 recordings / 20 assets / 20 windows`
- worker contract 现在已有基础前置状态保护;重复执行同一 chromaprint dry-run job 会被 `expected_status=pending` 明确拒绝,证据见 `phase1_worker_double_claim_guard_report.json`
- exact lane 的 `run_chromaprint_job.py` 已具备非 dry-run 写入路径;当前在 `acr_test` 的 live 结果是因为 `/workspace/downloads/...` 缺失而明确 `failed`,不是继续假装 `completed`
......