refactor(etl): 优化音频转存与歌词上传流程，支持音频 MD5 计算和歌词换行处理

- 移除 audit_hk_songs_duplicates.py 中删除零时长歌曲的逻辑 - 新增 utils.py 工具模块，提供音频 MD5 计算、歌词处理、OSS 上传等函数 - etl_to_crawler/oss.py 添加 transfer_url_with_md5 函数，用于下载音频、计算 MD5 并上传 OSS - etl_to_crawler/lyric.py 增加 ensure_newlines 函数，确保歌词内容行间带换行符 - etl_to_crawler/runner.py 改进音频转存调用，返回音频 MD5 值 - 通过 safe_upload_lyric 函数上传去时间戳的纯文本歌词，失败时使用原始歌词 URL - etl_to_crawler/writer.py 在歌曲数据写入中新增 audio_md5 字段支持 - 重构 yinyan_song_records 逻辑，新增初始化函数 initialize_yinyan_song_records - run_etl.py 增加 --init-yinyan-records 命令行参数支持仅初始化状态表 - 优化 ETL 运行流程，从预标记状态表中拉取待处理歌曲逐批处理，提高效率 - 新增相关单元测试以覆盖音频 MD5 计算和任务处理逻辑修改

refactor(etl): 优化音频转存与歌词上传流程，支持音频 MD5 计算和歌词换行处理
- 移除 audit_hk_songs_duplicates.py 中删除零时长歌曲的逻辑 - 新增 utils.py 工具模块，提供音频 MD5 计算、歌词处理、OSS 上传等函数 - etl_to_crawler/oss.py 添加 transfer_url_with_md5 函数，用于下载音频、计算 MD5 并上传 OSS - etl_to_crawler/lyric.py 增加 ensure_newlines 函数，确保歌词内容行间带换行符 - etl_to_crawler/runner.py 改进音频转存调用，返回音频 MD5 值 - 通过 safe_upload_lyric 函数上传去时间戳的纯文本歌词，失败时使用原始歌词 URL - etl_to_crawler/writer.py 在歌曲数据写入中新增 audio_md5 字段支持 - 重构 yinyan_song_records 逻辑，新增初始化函数 initialize_yinyan_song_records - run_etl.py 增加 --init-yinyan-records 命令行参数支持仅初始化状态表 - 优化 ETL 运行流程，从预标记状态表中拉取待处理歌曲逐批处理，提高效率 - 新增相关单元测试以覆盖音频 MD5 计算和任务处理逻辑修改
沈秋雨
Commit dc82bc10 ... dc82bc1084c4892bfcaccd24155b4c04b2acc8e5 authored 2026-07-09 15:57:53 +0800 by 沈秋雨
Showing 11 changed files with 668 additions and 164 deletions
audit_hk_songs_duplicates.py
etl_to_crawler/lyric.py
etl_to_crawler/oss.py
etl_to_crawler/reader.py
etl_to_crawler/runner.py
etl_to_crawler/utils.py
etl_to_crawler/writer.py
run_etl.py
tests/test_oss.py
tests/test_runner.py
tests/test_writer.py
--- a/audit_hk_songs_duplicates.py
View file @dc82bc1
+++ b/audit_hk_songs_duplicates.py
View file @dc82bc1
@@ -558,52 +558,9 @@ def main() -> int:
        help="补处理暂存表中 merge+skipped 记录：existing_into_new 入库+软删旧记录，new_into_existing 补作者合并",
    )
    parser.add_argument("--skip-oss", action="store_true", help="--fix-merge 时跳过歌词 OSS 上传，保留原始文本")
-    parser.add_argument("--dry-run", action="store_true", help="仅打印计划，不写库（配合 --fix-merge / --delete-zero-duration 使用）")
+    parser.add_argument("--dry-run", action="store_true", help="仅打印计划，不写库（配合 --fix-merge 使用）")
-    parser.add_argument(
-        "--delete-zero-duration",
-        action="store_true",
-        help="软删除 song_time=0 的记录（置 deleted='1'）",
-    )
    args = parser.parse_args()
-    # --delete-zero-duration 模式
-    if args.delete_zero_duration:
-        conn = pymysql.connect(**TARGET_DB_CONFIG)
-        try:
-            table = quote_identifier(DEFAULT_TARGET_TABLE)
-            with conn.cursor() as cursor:
-                cursor.execute(
-                    f"SELECT id, name, lyricist, composer, song_time "
-                    f"FROM {table} WHERE deleted = '0' AND song_time = 0"
-                )
-                rows = list(cursor.fetchall())
-            print(f"song_time=0 记录数: {len(rows)}")
-            if args.dry_run:
-                for row in rows[:20]:
-                    print(row)
-                if len(rows) > 20:
-                    print(f"... {len(rows) - 20} more rows")
-                return 0
-            if not rows:
-                print("无需处理")
-                return 0
-            ids = [row["id"] for row in rows]
-            placeholders = ",".join(["%s"] * len(ids))
-            with conn.cursor() as cursor:
-                cursor.execute(
-                    f"UPDATE {table} "
-                    f"SET deleted = '1', modify_time = NOW(), "
-                    f"    off_shelf_remark = 'song_time=0 soft-deleted by audit script' "
-                    f"WHERE deleted = '0' AND id IN ({placeholders})",
-                    ids,
-                )
-                affected = cursor.rowcount
-            conn.commit()
-            print(f"soft_deleted={affected}")
-        finally:
-            conn.close()
-        return 0
    # --fix-merge 模式：连双库，执行合并方向后处理
    if args.fix_merge:
        target_conn = pymysql.connect(**TARGET_DB_CONFIG)
--- a/etl_to_crawler/lyric.py
View file @dc82bc1
+++ b/etl_to_crawler/lyric.py
View file @dc82bc1
@@ -4,6 +4,16 @@ _TIMESTAMP_RE = re.compile(r'\[\d{2}:\d{2}\.\d{2,3}\]')
 _META_TAG_RE = re.compile(r'\[[a-zA-Z]+:[^\]]*\]')
+def ensure_newlines(text: str | None) -> str:
+    """确保歌词每行之间有换行符，保留时间戳等原始内容"""
+    if not text:
+        return ''
+    # 将字面量 \\n / \\r 转为真正换行符
+    text = text.replace('\\n', '\n').replace('\\r', '')
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    return '\n'.join(lines)
 def strip_timestamps(text: str | None) -> str:
    if not text:
        return ''
--- a/etl_to_crawler/oss.py
View file @dc82bc1
+++ b/etl_to_crawler/oss.py
View file @dc82bc1
 import requests
 import oss2
 from urllib.parse import urlparse
+from .utils import compute_audio_md5
 ARCHIVE_DEV_HOST = 'archive-dev.oss-cn-beijing.aliyuncs.com'
@@ -21,6 +22,22 @@ def transfer_url(url: str | None, oss_key: str, bucket: oss2.Bucket, base_url: s
    return f"{base_url.rstrip('/')}/{oss_key}"
+def transfer_url_with_md5(url: str | None, oss_key: str, bucket: oss2.Bucket, base_url: str) -> tuple[str, str]:
+    """
+    将音频 URL 转存到 OSS，并基于下载到的音频字节计算 MD5。
+    已在目标 OSS 的 URL 无需重新下载，无法可靠计算 MD5，返回空 MD5。
+    """
+    if not url:
+        return '', ''
+    if ARCHIVE_DEV_HOST in url:
+        return url, ''
+    resp = requests.get(url, timeout=30)
+    resp.raise_for_status()
+    audio_md5 = compute_audio_md5(resp.content)
+    bucket.put_object(oss_key, resp.content)
+    return f"{base_url.rstrip('/')}/{oss_key}", audio_md5
 def build_oss_key(platform: str, category: str, filename: str) -> str:
    """
    构造 archive-dev 内的存储路径。
--- a/etl_to_crawler/reader.py
View file @dc82bc1
+++ b/etl_to_crawler/reader.py
View file @dc82bc1
@@ -16,6 +16,18 @@ ORDER BY id
 LIMIT %s
 """
+_HK_SONGS_BY_SOURCE_IDS_QUERY = """
+SELECT id, name, lyricist, composer, audio_url, lyrics_url,
+       cover_url, singer, issue_time, source_song_id, song_time
+FROM hk_songs_test
+WHERE deleted = '0'
+  AND source_song_id IN ({placeholders})
+  AND name IS NOT NULL AND name != ''
+  AND audio_url IS NOT NULL AND audio_url != ''
+  AND singer IS NOT NULL AND singer != ''
+ORDER BY id
+"""
 _PLATFORM_QUERY = """
 SELECT
    sar.song_id AS source_song_id,
@@ -61,6 +73,21 @@ def iter_hk_songs_batches(
                break
+def fetch_hk_songs_by_source_ids(conn: pymysql.Connection, source_song_ids: list[int]) -> dict[int, dict]:
+    if not source_song_ids:
+        return {}
+    placeholders = ','.join(['%s'] * len(source_song_ids))
+    query = _HK_SONGS_BY_SOURCE_IDS_QUERY.format(placeholders=placeholders)
+    with conn.cursor() as cur:
+        cur.execute(query, source_song_ids)
+        rows = cur.fetchall()
+    return {
+        int(row['source_song_id']): row
+        for row in rows
+        if row.get('source_song_id') is not None
+    }
 def fetch_platform_records(source_conn: pymysql.Connection, song_ids: list[int]) -> list[dict]:
    if not song_ids:
        return []
--- a/etl_to_crawler/runner.py
View file @dc82bc1
+++ b/etl_to_crawler/runner.py
View file @dc82bc1
 import uuid
 import json
 import logging
-from pathlib import Path
 from tqdm import tqdm
 from .config import PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE, BATCH_SIZE, OSS_CONFIG
 from .connections import get_hk_songs_conn, get_source_conn, get_spider_conn, get_pg_conn, get_oss_bucket
-from .reader import iter_hk_songs_batches, fetch_platform_records, select_primary_record
+from .reader import (
+    iter_hk_songs_batches,
+    fetch_hk_songs_by_source_ids,
+    fetch_platform_records,
+    select_primary_record,
+)
 from .spider import (
    fetch_qq_songs, fetch_qq_singers,
    fetch_kugou_songs, fetch_kugou_singers,
    fetch_netease_songs, fetch_netease_singers,
 )
 from .writer import (
+    fetch_pending_yinyan_song_records,
+    insert_yinyan_song_records,
    upsert_yinyan_song_records,
    upsert_qq_singers, upsert_qq_albums, upsert_qq_songs,
    upsert_qq_singer_songs, upsert_qq_singer_albums,
@@ -21,8 +27,9 @@ from .writer import (
    upsert_netease_singers, upsert_netease_albums, upsert_netease_songs,
    upsert_netease_singer_songs, upsert_netease_singer_albums,
 )
-from .oss import transfer_url, build_oss_key
+from .oss import transfer_url, transfer_url_with_md5, build_oss_key
-from .lyric import strip_timestamps
+from .utils import upload_plain_lyric_to_bucket
+from .lyric import ensure_newlines
 logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
 log = logging.getLogger(__name__)
@@ -37,6 +44,22 @@ def _safe_transfer(url, oss_key, bucket, base_url):
        return url  # 失败时保留原 URL，不阻断流程
+def _safe_transfer_audio(url, oss_key, bucket, base_url) -> tuple[str, str]:
+    try:
+        return transfer_url_with_md5(url, oss_key, bucket, base_url)
+    except Exception as e:
+        log.warning("Audio transfer failed for %s: %s", url, e)
+        return url or '', ''
+def _safe_upload_lyric(platform: str, unique_id: str, lyric: str | None, fallback_url: str | None, bucket, base_url: str) -> str:
+    try:
+        return upload_plain_lyric_to_bucket(platform, unique_id, lyric or '', bucket, base_url) or (fallback_url or '')
+    except Exception as e:
+        log.warning("Lyric upload failed for %s/%s: %s", platform, unique_id, e)
+        return fallback_url or ''
 def _json_default(value):
    return str(value)
@@ -57,7 +80,7 @@ def _process_qq(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url):
    singer_list = singers_map.get(song_id_int, [])
    # OSS 转移
-    audio_url = _safe_transfer(
+    audio_url, audio_md5 = _safe_transfer_audio(
        hk_row['audio_url'],
        build_oss_key('qq', 'audio', mid + '.mp3'),
        bucket, base_url
@@ -127,6 +150,8 @@ def _process_qq(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url):
    # 写入 song
    platform_song_id = int(pr['platform_mid']) if pr.get('platform_mid') else song_id_int
    song_uuid = str(uuid.uuid4())
+    raw_lyric = sp.get('lyric') or ''
+    lyric_url = _safe_upload_lyric('qq', mid, raw_lyric, hk_row.get('lyrics_url'), bucket, base_url)
    upsert_qq_songs(pg_cur, [{
        'song_uuid': song_uuid,
        'platform_song_id': platform_song_id,
@@ -136,11 +161,12 @@ def _process_qq(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url):
        'title': sp.get('title', hk_row['name']),
        'name': hk_row['name'],
        'duration': sp.get('duration') or hk_row.get('song_time') or 0,
-        'lyric': strip_timestamps(sp.get('lyric')),
+        'lyric': ensure_newlines(raw_lyric),
        'composer_name': sp.get('composer_name') or hk_row.get('composer'),
        'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
        'url': audio_url,
-        'lyric_url': hk_row.get('lyrics_url'),
+        'audio_md5': audio_md5,
+        'lyric_url': lyric_url,
        'platform_index_url': sp.get('platform_index_url') or f'https://y.qq.com/n/ryqq/songDetail/{mid}',
        'published_at': sp.get('published_at') or hk_row.get('issue_time'),
        'singers_json': singers_json,
@@ -170,7 +196,7 @@ def _process_kugou(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url
    singers_map = fetch_kugou_singers(spider_conn, [song_id])
    singer_list = singers_map.get(song_id, [])
-    audio_url = _safe_transfer(
+    audio_url, audio_md5 = _safe_transfer_audio(
        hk_row['audio_url'],
        build_oss_key('kugou', 'audio', str(song_id) + '.mp3'),
        bucket, base_url
@@ -234,6 +260,8 @@ def _process_kugou(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url
        }])
    song_uuid = str(uuid.uuid4())
+    raw_lyric = sp.get('lyric') or ''
+    lyric_url = _safe_upload_lyric('kugou', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url)
    upsert_kugou_songs(pg_cur, [{
        'song_uuid': song_uuid,
        'platform_song_id': song_id,
@@ -244,11 +272,12 @@ def _process_kugou(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_url
        'title': sp.get('title', hk_row['name']),
        'name': hk_row['name'],
        'duration': sp.get('duration') or hk_row.get('song_time') or 0,
-        'lyric': strip_timestamps(sp.get('lyric')),
+        'lyric': ensure_newlines(raw_lyric),
        'composer_name': sp.get('composer_name') or hk_row.get('composer'),
        'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
        'url': audio_url,
-        'lyric_url': hk_row.get('lyrics_url'),
+        'audio_md5': audio_md5,
+        'lyric_url': lyric_url,
        'platform_index_url': sp.get('platform_index_url') or f'http://www.kugou.com/song/#hash={sp.get("hid") or pr.get("platform_mid", "")}',
        'published_at': sp.get('published_at') or hk_row.get('issue_time'),
        'singers_json': singers_json,
@@ -277,7 +306,7 @@ def _process_netease(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_u
    singers_map = fetch_netease_singers(spider_conn, [song_id])
    singer_list = singers_map.get(song_id, [])
-    audio_url = _safe_transfer(
+    audio_url, audio_md5 = _safe_transfer_audio(
        hk_row['audio_url'],
        build_oss_key('netease', 'audio', str(song_id) + '.mp3'),
        bucket, base_url
@@ -354,6 +383,8 @@ def _process_netease(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_u
        }])
    song_uuid = str(uuid.uuid4())
+    raw_lyric = sp.get('lyric') or ''
+    lyric_url = _safe_upload_lyric('netease', str(song_id), raw_lyric, hk_row.get('lyrics_url'), bucket, base_url)
    upsert_netease_songs(pg_cur, [{
        'song_uuid': song_uuid,
        'platform_song_id': song_id,
@@ -363,11 +394,12 @@ def _process_netease(hk_row: dict, pr: dict, spider_conn, pg_cur, bucket, base_u
        'title': sp.get('title', hk_row['name']),
        'name': hk_row['name'],
        'duration': sp.get('duration') or hk_row.get('song_time') or 0,
-        'lyric': strip_timestamps(sp.get('lyric')),
+        'lyric': ensure_newlines(raw_lyric),
        'composer_name': sp.get('composer_name') or hk_row.get('composer'),
        'lyricist_name': sp.get('lyricist_name') or hk_row.get('lyricist'),
        'url': audio_url,
-        'lyric_url': hk_row.get('lyrics_url'),
+        'audio_md5': audio_md5,
+        'lyric_url': lyric_url,
        'platform_index_url': sp.get('platform_index_url') or f'https://music.163.com/#/song?id={song_id}',
        'published_at': sp.get('published_at') or hk_row.get('issue_time'),
        'singers_json': singers_json,
@@ -393,36 +425,52 @@ _PROCESSORS = {
 }
-DEFAULT_STATE_FILE = Path('output/etl_to_crawler_state.json')
+def initialize_yinyan_song_records(platforms: list[str], max_batches: int | None = None) -> None:
+    hk_conn = get_hk_songs_conn()
+    src_conn = get_source_conn()
+    pg_conn = get_pg_conn()
-def _load_state(path: Path) -> dict:
+    total = 0
-    if not path.exists():
+    try:
-        return {}
+        for i, batch in enumerate(tqdm(iter_hk_songs_batches(hk_conn, BATCH_SIZE), desc='init-yinyan')):
-    with path.open('r', encoding='utf-8') as f:
+            if max_batches is not None and i >= max_batches:
-        return json.load(f)
+                break
+            song_ids = [int(r['source_song_id']) for r in batch if r.get('source_song_id')]
+            platform_records = fetch_platform_records(src_conn, song_ids)
+            pr_by_song: dict[int, list] = {}
+            for pr in platform_records:
+                if pr['platform'] in platforms:
+                    pr_by_song.setdefault(int(pr['source_song_id']), []).append(pr)
-def _save_state(path: Path, state: dict) -> None:
+            init_rows = []
-    path.parent.mkdir(parents=True, exist_ok=True)
+            for hk_row in batch:
-    with path.open('w', encoding='utf-8') as f:
+                src_id = int(hk_row['source_song_id']) if hk_row.get('source_song_id') else None
-        json.dump(state, f, ensure_ascii=False, indent=2, sort_keys=True)
+                if not src_id or src_id not in pr_by_song:
-        f.write('\n')
+                    continue
+                primary_record = select_primary_record(pr_by_song[src_id])
+                if primary_record:
+                    init_rows.append({
+                        'song_id': src_id,
+                        'record_id': int(primary_record['record_id']),
+                    })
+            if init_rows:
+                with pg_conn.cursor() as pg_cur:
+                    insert_yinyan_song_records(pg_cur, init_rows)
+                pg_conn.commit()
+                total += len(init_rows)
+    finally:
+        hk_conn.close()
+        src_conn.close()
+        pg_conn.close()
-def _state_last_id(state: dict, key: str) -> int:
+    log.info("Initialized yinyan_song_records candidates=%d", total)
-    value = state.get(key, {}).get('last_hk_songs_id', 0)
-    return int(value or 0)
 def run(
    platforms: list[str],
    max_batches: int | None = None,
-    resume: bool = False,
-    state_file: str | Path = DEFAULT_STATE_FILE,
-    state_key: str = 'all',
-    start_after_id: int | None = None,
-    reset_state: bool = False,
 ) -> None:
    hk_conn = get_hk_songs_conn()
    src_conn = get_source_conn()
@@ -430,25 +478,21 @@ def run(
    pg_conn = get_pg_conn()
    bucket = get_oss_bucket()
    base_url = OSS_CONFIG['base_url']
-    state_path = Path(state_file)
-    state = _load_state(state_path) if resume and not reset_state else {}
-    resume_start_id = 0
-    if resume:
-        resume_start_id = _state_last_id(state, state_key)
-    if start_after_id is not None:
-        resume_start_id = start_after_id
    total_ok = total_err = 0
    imported: list[dict] = []
    try:
-        for i, batch in enumerate(tqdm(
+        batch_index = 0
-            iter_hk_songs_batches(hk_conn, BATCH_SIZE, start_after_id=resume_start_id),
+        pbar = tqdm(desc='batches')
-            desc='batches',
+        while max_batches is None or batch_index < max_batches:
-        )):
+            with pg_conn.cursor() as pg_cur:
-            if max_batches is not None and i >= max_batches:
+                pending_records = fetch_pending_yinyan_song_records(pg_cur, BATCH_SIZE)
+            if not pending_records:
                break
-            song_ids = [int(r['source_song_id']) for r in batch if r.get('source_song_id')]
+            song_ids = [int(r['song_id']) for r in pending_records]
+            pending_by_song = {int(r['song_id']): int(r['record_id']) for r in pending_records}
+            hk_by_song = fetch_hk_songs_by_source_ids(hk_conn, song_ids)
            platform_records = fetch_platform_records(src_conn, song_ids)
            # index platform records by source_song_id
@@ -458,12 +502,15 @@ def run(
                    pr_by_song.setdefault(int(pr['source_song_id']), []).append(pr)
            with pg_conn.cursor() as pg_cur:
-                for hk_row in batch:
+                pushed_count = 0
-                    src_id = int(hk_row['source_song_id']) if hk_row.get('source_song_id') else None
+                for src_id in song_ids:
+                    hk_row = hk_by_song.get(src_id)
+                    if not hk_row:
+                        continue
                    if not src_id or src_id not in pr_by_song:
                        continue
-                    primary_record = select_primary_record(pr_by_song[src_id])
                    wrote_yinyan_record = False
+                    pending_record_id = pending_by_song[src_id]
                    for pr in pr_by_song[src_id]:
                        processor = _PROCESSORS.get(pr['platform'])
                        if not processor:
@@ -471,9 +518,19 @@ def run(
                        pg_cur.execute('SAVEPOINT sp_song')
                        try:
                            result = processor(hk_row, pr, spider_conn, pg_cur, bucket, base_url)
-                            if result and primary_record and not wrote_yinyan_record:
+                            if (
-                                upsert_yinyan_song_records(pg_cur, [(src_id, int(primary_record['record_id']))])
+                                result
+                                and not wrote_yinyan_record
+                                and int(pr['record_id']) == pending_record_id
+                            ):
+                                upsert_yinyan_song_records(pg_cur, [{
+                                    'song_id': src_id,
+                                    'record_id': pending_record_id,
+                                    'platform': result['platform'],
+                                    'platform_song_id': int(result['platform_song_id']),
+                                }])
                                wrote_yinyan_record = True
+                                pushed_count += 1
                            pg_cur.execute('RELEASE SAVEPOINT sp_song')
                            total_ok += 1
                            if result:
@@ -485,9 +542,12 @@ def run(
                                      hk_row.get('name'), pr['platform'], e)
                            total_err += 1
                pg_conn.commit()
-                if resume:
+                if pushed_count == 0:
-                    state.setdefault(state_key, {})['last_hk_songs_id'] = int(batch[-1]['id'])
+                    log.error("No yinyan_song_records rows were marked pushed in this batch; stopping to avoid retry loop")
-                    _save_state(state_path, state)
+                    break
+            batch_index += 1
+            pbar.update(1)
+        pbar.close()
    finally:
        hk_conn.close()
--- a/etl_to_crawler/utils.py 0 → 100644
View file @dc82bc1
+++ b/etl_to_crawler/utils.py 0 → 100644
View file @dc82bc1
+"""工具函数模块"""
+import asyncio
+import hashlib
+import logging
+import re
+from datetime import datetime
+from urllib.parse import urlparse
+from zoneinfo import ZoneInfo
+try:
+    import httpx
+except ModuleNotFoundError:
+    httpx = None
+try:
+    from app.core.config import settings
+    from app.core.oss_client import oss_client
+except ModuleNotFoundError:
+    settings = None
+    oss_client = None
+CHINA_TZ = ZoneInfo("Asia/Shanghai")
+logger = logging.getLogger(__name__)
+# OSS 上传默认超时（秒），优先使用全局配置
+_OSS_UPLOAD_TIMEOUT = settings.OSS_UPLOAD_TIMEOUT_SECONDS if settings else 30
+# 音频下载默认超时（秒），优先使用全局配置
+_AUDIO_DOWNLOAD_TIMEOUT = settings.AUDIO_DOWNLOAD_TIMEOUT_SECONDS if settings else 30
+# 音频下载最大限制（字节），优先使用全局配置
+_MAX_AUDIO_SIZE = settings.MAX_AUDIO_DOWNLOAD_SIZE if settings else 50 * 1024 * 1024
+# 歌词中常见的词曲作者匹配规则
+_LYRICIST_PATTERNS = [
+    r"作\s*词\s*[:：]\s*([^\r\n/／]+)",
+    r"词\s*[:：]\s*([^\r\n/／]+)",
+    r"Lyrics?\s*[:：]\s*([^\r\n/／]+)",
+]
+_COMPOSER_PATTERNS = [
+    r"作\s*曲\s*[:：]\s*([^\r\n/／]+)",
+    r"曲\s*[:：]\s*([^\r\n/／]+)",
+    r"Composer\s*[:：]\s*([^\r\n/／]+)",
+]
+def _search_with_patterns(patterns: list, text: str) -> str:
+    """根据多个正则匹配文本，返回首个命中分组"""
+    if not text:
+        return ""
+    for pattern in patterns:
+        match = re.search(pattern, text)
+        if match:
+            return match.group(1).strip()
+    return ""
+def extract_lyricist_composer(lyric: str) -> tuple[str, str]:
+    """从歌词文本中提取词作者和曲作者
+    Returns:
+        (lyricist_name, composer_name)
+    """
+    lyricist = _search_with_patterns(_LYRICIST_PATTERNS, lyric)
+    composer = _search_with_patterns(_COMPOSER_PATTERNS, lyric)
+    return lyricist, composer
+def now_cn() -> datetime:
+    """返回中国时区的当前时间（无时区信息），保留微秒"""
+    return datetime.now(CHINA_TZ).replace(tzinfo=None)
+def extract_plain_lyric(lyric: str) -> str:
+    """去除 LRC 歌词中的时间戳和标签，保留纯文本
+    Args:
+        lyric: 原始歌词内容（可能含 [mm:ss.xx] 时间戳和 [ti:xxx] 等标签）
+    Returns:
+        纯文本歌词，行之间用换行符连接
+    """
+    if not lyric:
+        return ""
+    lines = []
+    for line in lyric.splitlines():
+        # 去除 LRC 时间戳 [mm:ss.xx] 或 [mm:ss.xxx]
+        cleaned = re.sub(r"\[\d{2}:\d{2}(?:\.\d{2,3})?\]", "", line)
+        # 去除标签 [xx:yy]
+        cleaned = re.sub(r"\[[a-zA-Z]+:[^\]]+\]", "", cleaned)
+        cleaned = cleaned.strip()
+        if cleaned:
+            lines.append(cleaned)
+    return "\n".join(lines)
+def upload_plain_lyric_to_bucket(platform: str, unique_id: str, lyric: str, bucket, base_url: str) -> str:
+    """将歌词去时间戳后上传到当前 ETL 使用的 OSS bucket"""
+    plain_lyric = extract_plain_lyric(lyric)
+    if not plain_lyric:
+        return ""
+    oss_key = f"crawler/{platform}/lyric/{unique_id}.txt"
+    bucket.put_object(oss_key, plain_lyric.encode("utf-8"))
+    return f"{base_url.rstrip('/')}/{oss_key}"
+async def upload_lyric_to_oss(platform: str, unique_id: str, lyric: str) -> str:
+    """将歌词去除时间戳后上传为纯文本文件到 OSS
+    Args:
+        platform: 平台标识，如 qqmusic/kugou/kuwo/netease/migu
+        unique_id: 平台唯一标识，如 song_mid/hash/rid 等
+        lyric: 原始歌词内容
+    Returns:
+        OSS 文件访问 URL，上传失败或歌词为空返回空字符串
+    """
+    if not lyric:
+        return ""
+    if oss_client is None:
+        logger.error("OSS client is not configured")
+        return ""
+    plain_lyric = extract_plain_lyric(lyric)
+    if not plain_lyric:
+        return ""
+    oss_key = f"lyrics/{platform}/{unique_id}.txt"
+    try:
+        file_url = await asyncio.wait_for(
+            asyncio.to_thread(
+                oss_client.upload_bytes,
+                plain_lyric.encode("utf-8"),
+                oss_key,
+                "public-read",
+                "text/plain; charset=utf-8",
+            ),
+            timeout=_OSS_UPLOAD_TIMEOUT,
+        )
+        return file_url or ""
+    except asyncio.TimeoutError:
+        logger.exception(f"上传歌词到 OSS 超时 {_OSS_UPLOAD_TIMEOUT}s {platform}/{unique_id}")
+        return ""
+    except Exception as e:
+        logger.exception(f"上传歌词到 OSS 失败 {platform}/{unique_id}: {e}")
+        return ""
+async def download_and_upload_audio(
+    audio_url: str,
+    oss_key: str,
+    *,
+    headers: dict | None = None,
+    allow_external_oss_url: bool = False,
+) -> tuple[str, str]:
+    """流式下载音频并上传到 OSS
+    内置大小限制、下载超时、上传超时保护，防止大文件或慢网络拖垮 worker。
+    Args:
+        audio_url: 音频下载地址
+        oss_key: OSS 对象键，如 songs/qqmusic/xxxx.mp3
+        headers: 可选的下载请求头
+        allow_external_oss_url: 若 audio_url 已是当前 OSS 域名下的地址，是否直接透传
+    Returns:
+        (OSS 文件 URL, 音频 MD5)，失败返回 ("", "")
+    """
+    if not audio_url:
+        return "", ""
+    if allow_external_oss_url and settings.OSS_FILE_BASE_NAME and audio_url.startswith(settings.OSS_FILE_BASE_NAME):
+        return audio_url, ""
+    audio_bytes = bytearray()
+    try:
+        async with httpx.AsyncClient(timeout=_AUDIO_DOWNLOAD_TIMEOUT) as client:
+            async with client.stream("GET", audio_url, headers=headers or {}) as response:
+                response.raise_for_status()
+                content_length = response.headers.get("Content-Length")
+                if content_length and int(content_length) > _MAX_AUDIO_SIZE:
+                    logger.warning(
+                        f"音频文件过大，跳过: {oss_key}, "
+                        f"size={int(content_length) / 1024 / 1024:.2f}MB"
+                    )
+                    return "", ""
+                async for chunk in response.aiter_bytes(chunk_size=64 * 1024):
+                    audio_bytes.extend(chunk)
+                    if len(audio_bytes) > _MAX_AUDIO_SIZE:
+                        logger.warning(
+                            f"音频下载超过大小限制，跳过: {oss_key}, "
+                            f"size>{_MAX_AUDIO_SIZE / 1024 / 1024:.2f}MB"
+                        )
+                        return "", ""
+    except Exception as e:
+        logger.error(f"下载音频失败 {oss_key}: {e}")
+        return "", ""
+    audio_bytes = bytes(audio_bytes)
+    audio_md5 = compute_audio_md5(audio_bytes)
+    try:
+        file_url = await asyncio.wait_for(
+            asyncio.to_thread(oss_client.upload_bytes, audio_bytes, oss_key),
+            timeout=_OSS_UPLOAD_TIMEOUT,
+        )
+    except asyncio.TimeoutError:
+        logger.error(f"上传音频到 OSS 超时 {_OSS_UPLOAD_TIMEOUT}s: {oss_key}")
+        return "", ""
+    except Exception as e:
+        logger.error(f"上传音频到 OSS 失败 {oss_key}: {e}")
+        return "", ""
+    return file_url or "", audio_md5
+_CONTENT_TYPE_EXT_MAP = {
+    "image/jpeg": "jpg",
+    "image/jpg": "jpg",
+    "image/png": "png",
+    "image/webp": "webp",
+    "image/gif": "gif",
+}
+def _guess_image_ext(content_type: str, url: str) -> str:
+    """根据 Content-Type 或 URL 路径推断图片扩展名，默认 jpg"""
+    if content_type:
+        mime = content_type.split(";")[0].strip().lower()
+        ext = _CONTENT_TYPE_EXT_MAP.get(mime)
+        if ext:
+            return ext
+    path = urlparse(url).path
+    suffix = path.rsplit(".", 1)[-1].lower()
+    if suffix in {"jpg", "jpeg", "png", "webp", "gif"}:
+        return "jpg" if suffix == "jpeg" else suffix
+    return "jpg"
+async def upload_cover_to_oss(platform: str, unique_id: str, image_url: str) -> str:
+    """下载封面图片并上传到 OSS
+    Args:
+        platform: 平台标识，如 qqmusic/kugou/kuwo/netease/migu
+        unique_id: 平台唯一标识，如 song_mid/hash/rid 等
+        image_url: 封面图片原始 URL
+    Returns:
+        OSS 文件访问 URL，上传失败或 URL 为空返回空字符串
+    """
+    if not image_url:
+        return ""
+    try:
+        async with httpx.AsyncClient(timeout=10, follow_redirects=True) as client:
+            resp = await client.get(image_url, headers={
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                "Referer": "https://www.kugou.com/",
+            })
+            resp.raise_for_status()
+            image_bytes = resp.content
+            content_type = resp.headers.get("content-type", "")
+    except Exception as e:
+        logger.exception(f"下载封面图片失败 {platform}/{unique_id} {image_url}: {e}")
+        return ""
+    ext = _guess_image_ext(content_type, image_url)
+    oss_key = f"covers/{platform}/{unique_id}.{ext}"
+    mime = _CONTENT_TYPE_EXT_MAP.get(content_type.split(";")[0].strip().lower(), f"image/{ext}")
+    try:
+        file_url = await asyncio.wait_for(
+            asyncio.to_thread(
+                oss_client.upload_bytes,
+                image_bytes,
+                oss_key,
+                "public-read",
+                mime,
+            ),
+            timeout=_OSS_UPLOAD_TIMEOUT,
+        )
+        return file_url or ""
+    except asyncio.TimeoutError:
+        logger.exception(f"上传封面到 OSS 超时 {_OSS_UPLOAD_TIMEOUT}s {platform}/{unique_id}")
+        return ""
+    except Exception as e:
+        logger.exception(f"上传封面到 OSS 失败 {platform}/{unique_id}: {e}")
+        return ""
+def compute_audio_md5(audio_bytes: bytes) -> str:
+    """计算音频字节流的 MD5 值（32 位小写十六进制字符串）
+    Args:
+        audio_bytes: 音频文件字节流
+    Returns:
+        MD5 字符串，输入为空时返回空字符串
+    """
+    if not audio_bytes:
+        return ""
+    return hashlib.md5(audio_bytes).hexdigest()
--- a/etl_to_crawler/writer.py
View file @dc82bc1
+++ b/etl_to_crawler/writer.py
View file @dc82bc1
 import uuid
-def upsert_yinyan_song_records(cur, pairs: list[tuple[int, int]]) -> None:
+def insert_yinyan_song_records(cur, records: list[dict]) -> None:
-    """pairs: [(source_song_id, hk_music_record_id), ...]"""
+    """Initialize yinyan_song_records rows before crawler import."""
-    if not pairs:
+    if not records:
        return
    cur.executemany(
-        "DELETE FROM yinyan_song_records WHERE song_id = %s",
+        """
-        [(song_id,) for song_id, _ in pairs],
+        INSERT INTO yinyan_song_records (song_id, record_id, is_yinyan_push)
+        VALUES (%s, %s, FALSE)
+        ON CONFLICT (song_id, record_id) DO NOTHING
+        """,
+        [(r['song_id'], r['record_id']) for r in records],
    )
+def fetch_pending_yinyan_song_records(cur, limit: int) -> list[dict]:
+    cur.execute(
+        """
+        SELECT song_id, record_id
+        FROM yinyan_song_records
+        WHERE is_yinyan_push = FALSE
+        ORDER BY song_id
+        LIMIT %s
+        """,
+        (limit,),
+    )
+    rows = cur.fetchall()
+    return [{'song_id': row[0], 'record_id': row[1]} for row in rows]
+def upsert_yinyan_song_records(cur, records: list[dict]) -> None:
+    """Mark pre-initialized yinyan song-record rows as pushed to crawler."""
+    if not records:
+        return
    cur.executemany(
        """
-        INSERT INTO yinyan_song_records (song_id, record_id)
+        UPDATE yinyan_song_records
-        VALUES (%s, %s)
+        SET platform = %s,
+            platform_song_id = %s,
+            is_yinyan_push = TRUE
+        WHERE song_id = %s AND record_id = %s
        """,
-        pairs,
+        [
+            (r['platform'], r['platform_song_id'], r['song_id'], r['record_id'])
+            for r in records
+        ],
    )
@@ -67,8 +98,8 @@ def upsert_qq_songs(cur, songs: list[dict]) -> None:
            (id, platform_song_id, mid, album_id, cover, title, name, duration,
             lyric, composer_name, lyricist_name, url, lyric_url,
             platform_index_url, published_at, singers, status, created_at, updated_at,
-             provider_name, crawler_source_data)
+             audio_md5, provider_name, crawler_source_data)
-        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s::json)
+        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s::json)
        ON CONFLICT (platform_song_id) DO NOTHING
    """
    rows = [(
@@ -77,6 +108,7 @@ def upsert_qq_songs(cur, songs: list[dict]) -> None:
        s.get('lyric'), s.get('composer_name'), s.get('lyricist_name'),
        s.get('url', ''), s.get('lyric_url'),
        s.get('platform_index_url'), s.get('published_at'), s.get('singers_json', '[]'),
+        s.get('audio_md5'),
        s.get('provider_name'), s.get('crawler_source_data'),
    ) for s in songs]
    cur.executemany(sql, rows)
@@ -157,8 +189,8 @@ def upsert_kugou_songs(cur, songs: list[dict]) -> None:
            (id, platform_song_id, hash, album_audio_id, album_id, cover, title, name, duration,
             lyric, composer_name, lyricist_name, url, lyric_url,
             platform_index_url, published_at, singers, status, created_at, updated_at,
-             provider_name, crawler_source_data)
+             audio_md5, provider_name, crawler_source_data)
-        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s::json)
+        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s::json)
        ON CONFLICT (platform_song_id) DO NOTHING
    """
    rows = [(
@@ -168,6 +200,7 @@ def upsert_kugou_songs(cur, songs: list[dict]) -> None:
        s.get('lyric'), s.get('composer_name'), s.get('lyricist_name'),
        s.get('url', ''), s.get('lyric_url'),
        s.get('platform_index_url'), s.get('published_at'), s.get('singers_json', '[]'),
+        s.get('audio_md5'),
        s.get('provider_name'), s.get('crawler_source_data'),
    ) for s in songs]
    cur.executemany(sql, rows)
@@ -244,8 +277,8 @@ def upsert_netease_songs(cur, songs: list[dict]) -> None:
            (id, platform_song_id, album_id, cover, title, name, duration,
             lyric, composer_name, lyricist_name, url, lyric_url,
             platform_index_url, published_at, album, singers, status, created_at, updated_at,
-             provider_name, crawler_source_data)
+             audio_md5, provider_name, crawler_source_data)
-        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s::json)
+        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::json, %s::jsonb, 0, NOW(), NOW(), %s, %s, %s::json)
        ON CONFLICT (platform_song_id) DO NOTHING
    """
    rows = [(
@@ -254,6 +287,7 @@ def upsert_netease_songs(cur, songs: list[dict]) -> None:
        s.get('lyric'), s.get('composer_name'), s.get('lyricist_name'),
        s.get('url', ''), s.get('lyric_url'),
        s.get('platform_index_url'), s.get('published_at'), s.get('album_json'), s.get('singers_json', '[]'),
+        s.get('audio_md5'),
        s.get('provider_name'), s.get('crawler_source_data'),
    ) for s in songs]
    cur.executemany(sql, rows)
--- a/run_etl.py
View file @dc82bc1
+++ b/run_etl.py
View file @dc82bc1
 #!/usr/bin/env python3
 import argparse
 from etl_to_crawler.config import PLATFORM_QQ, PLATFORM_KUGOU, PLATFORM_NETEASE, PLATFORMS
-from etl_to_crawler.runner import run
+from etl_to_crawler.runner import initialize_yinyan_song_records, run
 PLATFORM_MAP = {
    'qq': PLATFORM_QQ,
@@ -16,14 +16,8 @@ if __name__ == '__main__':
                        help='要导入的平台（默认 all）')
    parser.add_argument('--max-batches', type=int, default=None,
                        help='最多处理多少批次（冒烟测试用）')
-    parser.add_argument('--resume', action='store_true',
+    parser.add_argument('--init-yinyan-records', action='store_true',
-                        help='启用断点续传，按 hk_songs_test.id 从上次成功提交的批次后继续')
+                        help='只初始化 yinyan_song_records 待导入状态表，不执行 crawler 导入')
-    parser.add_argument('--state-file', default='output/etl_to_crawler_state.json',
-                        help='断点状态文件路径')
-    parser.add_argument('--start-after-id', type=int, default=None,
-                        help='手动指定从哪个 hk_songs_test.id 之后开始读取')
-    parser.add_argument('--reset-state', action='store_true',
-                        help='忽略已有断点状态，从头或 --start-after-id 指定位置重新开始')
    args = parser.parse_args()
    if args.platform == 'all':
@@ -32,12 +26,7 @@ if __name__ == '__main__':
        platforms = [PLATFORM_MAP[args.platform]]
    print(f"Starting ETL for platforms: {platforms}")
-    run(
+    if args.init_yinyan_records:
-        platforms,
+        initialize_yinyan_song_records(platforms, max_batches=args.max_batches)
-        max_batches=args.max_batches,
+    else:
-        resume=args.resume,
+        run(platforms, max_batches=args.max_batches)
-        state_file=args.state_file,
-        state_key=args.platform,
-        start_after_id=args.start_after_id,
-        reset_state=args.reset_state,
-    )
--- a/tests/test_oss.py
View file @dc82bc1
+++ b/tests/test_oss.py
View file @dc82bc1
 from unittest.mock import MagicMock, patch
-from etl_to_crawler.oss import transfer_url
+from etl_to_crawler.oss import transfer_url, transfer_url_with_md5
 ARCHIVE_URL = "https://archive-dev.oss-cn-beijing.aliyuncs.com/some/path.mp3"
 OTHER_URL = "https://hikoon-data-platform.oss-cn-beijing.aliyuncs.com/qq-audio/abc.mp3"
@@ -31,3 +31,15 @@ def test_external_url_downloads_and_uploads():
        result = transfer_url(OTHER_URL, "crawler/qq/audio/abc.mp3", bucket, BASE_URL)
    bucket.put_object.assert_called_once_with("crawler/qq/audio/abc.mp3", fake_content)
    assert result == f"{BASE_URL}/crawler/qq/audio/abc.mp3"
+def test_transfer_url_with_md5_hashes_downloaded_content_before_upload():
+    bucket = MagicMock()
+    fake_content = b"audio_bytes"
+    with patch('etl_to_crawler.oss.requests.get') as mock_get:
+        mock_get.return_value.content = fake_content
+        mock_get.return_value.raise_for_status = MagicMock()
+        result, audio_md5 = transfer_url_with_md5(OTHER_URL, "crawler/qq/audio/abc.mp3", bucket, BASE_URL)
+    bucket.put_object.assert_called_once_with("crawler/qq/audio/abc.mp3", fake_content)
+    assert result == f"{BASE_URL}/crawler/qq/audio/abc.mp3"
+    assert audio_md5 == "04d43544b267629d9089eaed3b847a99"
--- a/tests/test_runner.py
View file @dc82bc1
+++ b/tests/test_runner.py
View file @dc82bc1
@@ -48,12 +48,15 @@ def test_run_imports_all_platform_records_but_writes_one_primary_yinyan_relation
    monkeypatch.setattr(runner, 'get_spider_conn', lambda: _Connection())
    monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn)
    monkeypatch.setattr(runner, 'get_oss_bucket', lambda: object())
-    monkeypatch.setattr(runner, 'iter_hk_songs_batches', lambda conn, batch_size, start_after_id=0: [[{
+    monkeypatch.setattr(runner, 'fetch_pending_yinyan_song_records', lambda cur, batch_size: [
+        {'song_id': 10, 'record_id': 200},
+    ] if pg_conn.commits == 0 else [])
+    monkeypatch.setattr(runner, 'fetch_hk_songs_by_source_ids', lambda conn, song_ids: {10: {
        'source_song_id': 10,
        'name': '歌',
        'audio_url': 'https://example.com/a.mp3',
        'singer': '歌手',
-    }]])
+    }})
    monkeypatch.setattr(runner, 'fetch_platform_records', lambda conn, song_ids: [
        {
            'source_song_id': 10,
@@ -85,37 +88,56 @@ def test_run_imports_all_platform_records_but_writes_one_primary_yinyan_relation
    processors['1'].assert_called_once()
    processors['2'].assert_called_once()
-    yinyan_writer.assert_called_once_with(pg_conn.cur, [(10, 200)])
+    yinyan_writer.assert_called_once_with(pg_conn.cur, [{
+        'song_id': 10,
+        'record_id': 200,
+        'platform': 'kugou',
+        'platform_song_id': 200,
+    }])
    assert pg_conn.commits == 1
-def test_run_resume_starts_after_saved_id_and_updates_state_after_commit(monkeypatch, tmp_path):
+def test_initialize_yinyan_song_records_inserts_primary_records(monkeypatch):
    pg_conn = _PgConnection()
-    state_file = tmp_path / 'etl_state.json'
+    inserted = []
-    state_file.write_text('{"all": {"last_hk_songs_id": 40}}', encoding='utf-8')
-    seen_start_ids = []
    monkeypatch.setattr(runner, 'get_hk_songs_conn', lambda: _Connection())
    monkeypatch.setattr(runner, 'get_source_conn', lambda: _Connection())
-    monkeypatch.setattr(runner, 'get_spider_conn', lambda: _Connection())
    monkeypatch.setattr(runner, 'get_pg_conn', lambda: pg_conn)
-    monkeypatch.setattr(runner, 'get_oss_bucket', lambda: object())
-    def fake_batches(conn, batch_size, start_after_id=0):
+    monkeypatch.setattr(runner, 'iter_hk_songs_batches', lambda conn, batch_size: [[
-        seen_start_ids.append(start_after_id)
-        return iter([[
        {'id': 50, 'source_song_id': 10, 'name': '歌', 'audio_url': 'https://example.com/a.mp3', 'singer': '歌手'},
-            {'id': 60, 'source_song_id': 20, 'name': '歌2', 'audio_url': 'https://example.com/b.mp3', 'singer': '歌手2'},
    ]])
+    monkeypatch.setattr(runner, 'fetch_platform_records', lambda conn, song_ids: [
+        {
+            'source_song_id': 10,
+            'record_id': 100,
+            'platform': '1',
+            'platform_unique_key': 'qq-mid',
+            'platform_mid': '100',
+            'album_audio_id': None,
+            'is_main_version': 0,
+            'is_high': 0,
+            'pub_time': '2020-01-01',
+        },
+        {
+            'source_song_id': 10,
+            'record_id': 200,
+            'platform': '2',
+            'platform_unique_key': '200',
+            'platform_mid': 'kg-hash',
+            'album_audio_id': None,
+            'is_main_version': 1,
+            'is_high': 0,
+            'pub_time': '2021-01-01',
+        },
+    ])
+    monkeypatch.setattr(runner, 'insert_yinyan_song_records', lambda cur, rows: inserted.extend(rows))
-    monkeypatch.setattr(runner, 'iter_hk_songs_batches', fake_batches)
+    runner.initialize_yinyan_song_records(['1', '2'])
-    monkeypatch.setattr(runner, 'fetch_platform_records', lambda conn, song_ids: [])
-    runner.run(['1', '2', '4'], resume=True, state_file=state_file, state_key='all')
-    assert seen_start_ids == [40]
+    assert inserted == [{'song_id': 10, 'record_id': 200}]
    assert pg_conn.commits == 1
-    assert '"last_hk_songs_id": 60' in state_file.read_text(encoding='utf-8')
 def test_process_netease_builds_album_json_for_song_insert(monkeypatch):
@@ -147,6 +169,7 @@ def test_process_netease_builds_album_json_for_song_insert(monkeypatch):
    })
    monkeypatch.setattr(runner, 'fetch_netease_singers', lambda conn, song_ids: {})
    monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url)
+    monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5'))
    monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None)
    monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None)
    monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs))
@@ -174,3 +197,60 @@ def test_process_netease_builds_album_json_for_song_insert(monkeypatch):
    assert inserted_songs[0]['album_json']
    assert '"id": 20' in inserted_songs[0]['album_json']
    assert '"title": "专辑"' in inserted_songs[0]['album_json']
+def test_process_netease_keeps_timestamped_lyric_and_uploads_plain_lyric(monkeypatch):
+    pg_cur = MagicMock()
+    pg_cur.fetchone.return_value = ('song-uuid',)
+    inserted_songs = []
+    uploaded = {}
+    class Bucket:
+        def put_object(self, key, body):
+            uploaded['key'] = key
+            uploaded['body'] = body
+    monkeypatch.setattr(runner, 'fetch_netease_songs', lambda conn, song_ids: {
+        300: {
+            'id': 300,
+            'album_id': None,
+            'cover': 'https://example.com/cover.jpg',
+            'title': '录音标题',
+            'duration': 180,
+            'lyric': '[ti:歌名]\n[00:01.00]第一句\n[00:02.00]第二句',
+            'composer_name': '曲作者',
+            'lyricist_name': '词作者',
+            'platform_index_url': None,
+            'published_at': '2020-01-02',
+        },
+    })
+    monkeypatch.setattr(runner, 'fetch_netease_singers', lambda conn, song_ids: {})
+    monkeypatch.setattr(runner, '_safe_transfer', lambda url, oss_key, bucket, base_url: url)
+    monkeypatch.setattr(runner, '_safe_transfer_audio', lambda url, oss_key, bucket, base_url: (url, 'audio-md5'))
+    monkeypatch.setattr(runner, 'upsert_netease_singers', lambda cur, singers: None)
+    monkeypatch.setattr(runner, 'upsert_netease_albums', lambda cur, albums: None)
+    monkeypatch.setattr(runner, 'upsert_netease_songs', lambda cur, songs: inserted_songs.extend(songs))
+    monkeypatch.setattr(runner, 'upsert_netease_singer_songs', lambda cur, pairs: None)
+    runner._process_netease(
+        {
+            'name': '词曲名',
+            'audio_url': 'https://example.com/audio.mp3',
+            'lyrics_url': 'https://example.com/original.lrc',
+            'cover_url': '',
+            'composer': '词曲曲作者',
+            'lyricist': '词曲词作者',
+            'issue_time': '2019-01-01',
+            'song_time': 120,
+        },
+        {'platform_unique_key': '300'},
+        spider_conn=object(),
+        pg_cur=pg_cur,
+        bucket=Bucket(),
+        base_url='https://bucket.example.com',
+    )
+    assert inserted_songs[0]['lyric'] == '[ti:歌名]\n[00:01.00]第一句\n[00:02.00]第二句'
+    assert inserted_songs[0]['audio_md5'] == 'audio-md5'
+    assert inserted_songs[0]['lyric_url'] == 'https://bucket.example.com/crawler/netease/lyric/300.txt'
+    assert uploaded['body'].decode('utf-8') == '第一句\n第二句'
--- a/tests/test_writer.py
View file @dc82bc1
+++ b/tests/test_writer.py
View file @dc82bc1
 from unittest.mock import MagicMock, call
 from etl_to_crawler.writer import (
+    insert_yinyan_song_records,
    upsert_kugou_albums,
    upsert_kugou_singers,
    upsert_kugou_songs,
@@ -41,18 +42,35 @@ def test_upsert_qq_singer_songs():
    assert 'crawler_qqmusic_singer_songs' in sql
-def test_upsert_yinyan_song_records_replaces_existing_song_relation():
+def test_upsert_yinyan_song_records_marks_existing_relation_as_pushed():
    cur = MagicMock()
-    upsert_yinyan_song_records(cur, [(10, 100), (11, 101)])
+    upsert_yinyan_song_records(cur, [
+        {'song_id': 10, 'record_id': 100, 'platform': 'qq', 'platform_song_id': 1000},
+        {'song_id': 11, 'record_id': 101, 'platform': 'kugou', 'platform_song_id': 1001},
+    ])
-    assert cur.executemany.call_count == 2
+    sql, rows = cur.executemany.call_args[0]
-    delete_sql, delete_rows = cur.executemany.call_args_list[0][0]
+    assert 'UPDATE yinyan_song_records' in sql
-    insert_sql, insert_rows = cur.executemany.call_args_list[1][0]
+    assert 'platform = %s' in sql
-    assert 'DELETE FROM yinyan_song_records' in delete_sql
+    assert 'platform_song_id = %s' in sql
-    assert 'WHERE song_id = %s' in delete_sql
+    assert 'is_yinyan_push = TRUE' in sql
-    assert delete_rows == [(10,), (11,)]
+    assert 'WHERE song_id = %s AND record_id = %s' in sql
-    assert 'INSERT INTO yinyan_song_records' in insert_sql
+    assert rows == [('qq', 1000, 10, 100), ('kugou', 1001, 11, 101)]
-    assert insert_rows == [(10, 100), (11, 101)]
+def test_insert_yinyan_song_records_initializes_unpushed_rows():
+    cur = MagicMock()
+    insert_yinyan_song_records(cur, [
+        {'song_id': 10, 'record_id': 100},
+        {'song_id': 11, 'record_id': 101},
+    ])
+    sql, rows = cur.executemany.call_args[0]
+    assert 'INSERT INTO yinyan_song_records' in sql
+    assert 'is_yinyan_push' in sql
+    assert 'FALSE' in sql
+    assert 'ON CONFLICT (song_id, record_id) DO NOTHING' in sql
+    assert rows == [(10, 100), (11, 101)]
 def test_upsert_netease_songs_writes_album_json_column():
@@ -104,12 +122,14 @@ def test_upsert_kugou_songs_writes_provider_and_source_data():
        'singers_json': '[]',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 200}',
+        'audio_md5': 'md5-200',
    }])
    sql, rows = cur.executemany.call_args[0]
+    assert 'audio_md5' in sql
    assert 'provider_name, crawler_source_data' in sql
    assert '%s::json' in sql
-    assert rows[0][-2:] == ('yinyan', '{"id": 200}')
+    assert rows[0][-3:] == ('md5-200', 'yinyan', '{"id": 200}')
 def test_upsert_kugou_singers_writes_provider_and_source_data():
@@ -211,10 +231,12 @@ def test_upsert_qq_entities_write_provider_and_source_data():
        'singers_json': '[]',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 200}',
+        'audio_md5': 'md5-200',
    }])
    sql, rows = cur.executemany.call_args[0]
+    assert 'audio_md5' in sql
    assert 'provider_name, crawler_source_data' in sql
-    assert rows[0][-2:] == ('yinyan', '{"id": 200}')
+    assert rows[0][-3:] == ('md5-200', 'yinyan', '{"id": 200}')
 def test_upsert_netease_entities_write_provider_and_source_data():
@@ -273,7 +295,9 @@ def test_upsert_netease_entities_write_provider_and_source_data():
        'singers_json': '[]',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 300}',
+        'audio_md5': 'md5-300',
    }])
    sql, rows = cur.executemany.call_args[0]
+    assert 'audio_md5' in sql
    assert 'provider_name, crawler_source_data' in sql
-    assert rows[0][-2:] == ('yinyan', '{"id": 300}')
+    assert rows[0][-3:] == ('md5-300', 'yinyan', '{"id": 300}')