external_adapters.py 5.44 KB
"""Dataset adapter skeletons for external/open music corpora."""

from __future__ import annotations

from dataclasses import dataclass, asdict
from pathlib import Path
from typing import Dict, List
import argparse
import json


@dataclass
class DatasetRecord:
    name: str
    source_url: str
    license: str
    commercial_use: str
    notes: str


class BaseAdapter:
    name = "base"

    def describe(self) -> Dict:
        raise NotImplementedError

    def init_layout(self, root: Path) -> Dict:
        root.mkdir(parents=True, exist_ok=True)
        for sub in ["raw", "processed", "manifests", "licenses"]:
            (root / sub).mkdir(exist_ok=True)
        manifest = {
            "dataset": self.name,
            "root": str(root),
            "status": "initialized",
            "next_steps": [
                "download raw audio according to upstream license terms",
                "convert to catalog/query manifests",
                "record license evidence before training",
            ],
        }
        with open(root / "manifests" / "bootstrap.json", "w") as f:
            json.dump(manifest, f, indent=2, ensure_ascii=False)
        return manifest


class FMAAdapter(BaseAdapter):
    name = "fma"

    def describe(self) -> Dict:
        return {
            "name": "FMA",
            "source_url": "https://github.com/mdeff/fma",
            "recommended_subset": "fma_small",
            "catalog_strategy": "full tracks as references; random 5-15s crops as queries",
            "license_policy": "review per subset/track before commercial training",
        }


class MTGJamendoAdapter(BaseAdapter):
    name = "mtg_jamendo"

    def describe(self) -> Dict:
        return {
            "name": "MTG-Jamendo",
            "source_url": "https://github.com/MTG/mtg-jamendo-dataset",
            "recommended_subset": "small curated slice",
            "catalog_strategy": "download upstream audio subset then build catalog/query manifests",
            "license_policy": "verify CC terms for intended commercial use",
        }


class CCMusicAdapter(BaseAdapter):
    name = "ccmusic"

    def describe(self) -> Dict:
        return {
            "name": "CCMusic",
            "source_url": "https://ccmusic-database.github.io/en/database/ccm.html",
            "recommended_subset": "whitelisted approved subset only",
            "catalog_strategy": "use approved corpora only; normalize to project manifests",
            "license_policy": "application/permission review required before use",
        }


class ModelScopeMusicAdapter(BaseAdapter):
    name = "modelscope_music"

    def describe(self) -> Dict:
        return {
            "name": "ModelScope music datasets",
            "source_url": "https://modelscope.cn/search?page=1&search=music&type=dataset",
            "recommended_subset": "manual whitelist only",
            "catalog_strategy": "treat as discovery surface; add per-dataset adapter after legal review",
            "license_policy": "deny until whitelisted",
        }


ADAPTERS = {
    "fma": FMAAdapter(),
    "mtg_jamendo": MTGJamendoAdapter(),
    "ccmusic": CCMusicAdapter(),
    "modelscope_music": ModelScopeMusicAdapter(),
}

REGISTRY: List[DatasetRecord] = [
    DatasetRecord(
        name="FMA",
        source_url="https://github.com/mdeff/fma",
        license="Track-dependent / metadata CC BY 4.0; verify per subset",
        commercial_use="review_required",
        notes="Good first realistic MIR baseline",
    ),
    DatasetRecord(
        name="MTG-Jamendo",
        source_url="https://github.com/MTG/mtg-jamendo-dataset",
        license="Creative Commons source tracks; verify exact subset terms",
        commercial_use="review_required",
        notes="Good retrieval/tagging corpus with scripts",
    ),
    DatasetRecord(
        name="CCMusic",
        source_url="https://ccmusic-database.github.io/en/database/ccm.html",
        license="varies / application may be required",
        commercial_use="review_required",
        notes="Useful Chinese MIR source, needs permission review",
    ),
    DatasetRecord(
        name="ModelScope-music",
        source_url="https://modelscope.cn/search?page=1&search=music&type=dataset",
        license="varies by dataset",
        commercial_use="deny_until_whitelisted",
        notes="Discovery surface only until per-dataset review is complete",
    ),
]


def write_registry(output_path: str):
    out = Path(output_path)
    out.parent.mkdir(parents=True, exist_ok=True)
    with open(out, "w") as f:
        json.dump([asdict(x) for x in REGISTRY], f, indent=2, ensure_ascii=False)
    return out


def main():
    parser = argparse.ArgumentParser()
    sub = parser.add_subparsers(dest="cmd", required=True)

    p = sub.add_parser("registry")
    p.add_argument("--output", default="data/dataset_registry.json")

    p = sub.add_parser("init")
    p.add_argument("dataset", choices=sorted(ADAPTERS))
    p.add_argument("--root", default="data/external")

    p = sub.add_parser("describe")
    p.add_argument("dataset", choices=sorted(ADAPTERS))

    args = parser.parse_args()
    if args.cmd == "registry":
        path = write_registry(args.output)
        print(path)
    elif args.cmd == "init":
        root = Path(args.root) / args.dataset
        print(json.dumps(ADAPTERS[args.dataset].init_layout(root), indent=2, ensure_ascii=False))
    elif args.cmd == "describe":
        print(json.dumps(ADAPTERS[args.dataset].describe(), indent=2, ensure_ascii=False))


if __name__ == "__main__":
    main()