export_manifest_to_pgvector_json.py
2.84 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
#!/usr/bin/env python3
"""Export project manifests into a pgvector-friendly JSON payload.
This does not require PostgreSQL at runtime. It prepares normalized rows so a
future loader can bulk ingest them into Postgres/pgvector safely.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
def load_json(path: Path):
return json.loads(path.read_text())
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--data", required=True, help="manifest directory")
parser.add_argument("--output", required=True)
parser.add_argument("--split", default="train")
parser.add_argument("--source-dataset", default="unknown")
args = parser.parse_args()
data_dir = Path(args.data)
catalog = load_json(data_dir / "catalog.json")
split_rows = load_json(data_dir / f"{args.split}.json")
songs = {}
references = []
segments = []
for row in catalog:
song_id = row["song_id"]
songs.setdefault(song_id, {
"song_id": song_id,
"title": song_id,
"artist": None,
"version_id": None,
"source_dataset": row.get("source_dataset", args.source_dataset),
"license": None,
})
if row.get("type") == "reference":
references.append({
"song_id": song_id,
"audio_uri": row["audio_path"],
"duration_sec": row["duration"],
"sample_rate": 16000,
})
for row in split_rows:
if row.get("type") == "reference":
continue
song_id = row["song_id"]
songs.setdefault(song_id, {
"song_id": song_id,
"title": song_id,
"artist": None,
"version_id": None,
"source_dataset": row.get("source_dataset", args.source_dataset),
"license": None,
})
segments.append({
"song_id": song_id,
"audio_uri": row["audio_path"],
"offset_sec": row.get("offset", 0.0),
"duration_sec": row["duration"],
"split": args.split,
"type": row.get("type", "unknown"),
"segment_type": row.get("segment_type"),
"source_dataset": row.get("source_dataset", args.source_dataset),
})
payload = {
"songs": list(songs.values()),
"references": references,
"segments": segments,
}
out = Path(args.output)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
print(json.dumps({
"status": "ok",
"output": str(out.resolve()),
"songs": len(payload["songs"]),
"references": len(payload["references"]),
"segments": len(payload["segments"]),
}, indent=2, ensure_ascii=False))
if __name__ == "__main__":
main()