test_writer.py 8.99 KB
from unittest.mock import MagicMock, call
from etl_to_crawler.writer import (
    upsert_kugou_albums,
    upsert_kugou_singers,
    upsert_kugou_songs,
    upsert_netease_albums,
    upsert_netease_singers,
    upsert_netease_songs,
    upsert_qq_albums,
    upsert_qq_singers,
    upsert_qq_songs,
    upsert_qq_singer_songs,
    upsert_yinyan_song_records,
)


def test_upsert_qq_singers_executes_insert():
    cur = MagicMock()
    singers = [{
        'id': 1, 'mid': 'abc', 'name': '歌手', 'avatar': 'https://archive-dev.oss-cn-beijing.aliyuncs.com/a.jpg',
        'sex': 'M', 'area': '华语', 'index': 'G', 'intro': None, 'home_url': None
    }]
    upsert_qq_singers(cur, singers)
    assert cur.executemany.called
    args = cur.executemany.call_args
    assert 'crawler_qqmusic_singers' in args[0][0]
    assert 'ON CONFLICT' in args[0][0]


def test_upsert_qq_singers_empty_does_nothing():
    cur = MagicMock()
    upsert_qq_singers(cur, [])
    cur.executemany.assert_not_called()


def test_upsert_qq_singer_songs():
    cur = MagicMock()
    upsert_qq_singer_songs(cur, [(1, 'uuid-abc'), (2, 'uuid-def')])
    assert cur.executemany.called
    sql = cur.executemany.call_args[0][0]
    assert 'crawler_qqmusic_singer_songs' in sql


def test_upsert_yinyan_song_records_replaces_existing_song_relation():
    cur = MagicMock()
    upsert_yinyan_song_records(cur, [(10, 100), (11, 101)])

    assert cur.executemany.call_count == 2
    delete_sql, delete_rows = cur.executemany.call_args_list[0][0]
    insert_sql, insert_rows = cur.executemany.call_args_list[1][0]
    assert 'DELETE FROM yinyan_song_records' in delete_sql
    assert 'WHERE song_id = %s' in delete_sql
    assert delete_rows == [(10,), (11,)]
    assert 'INSERT INTO yinyan_song_records' in insert_sql
    assert insert_rows == [(10, 100), (11, 101)]


def test_upsert_netease_songs_writes_album_json_column():
    cur = MagicMock()
    upsert_netease_songs(cur, [{
        'song_uuid': 'song-uuid',
        'platform_song_id': 300,
        'album_id': 20,
        'album_json': '{"id": 20, "title": "专辑"}',
        'cover': 'https://example.com/cover.jpg',
        'title': '标题',
        'name': '词曲名',
        'duration': 180,
        'lyric': '歌词',
        'composer_name': '曲作者',
        'lyricist_name': '词作者',
        'url': 'https://example.com/audio.mp3',
        'lyric_url': 'https://example.com/lyric.lrc',
        'platform_index_url': 'https://music.163.com/#/song?id=300',
        'published_at': '2020-01-01',
        'singers_json': '[]',
    }])

    sql, rows = cur.executemany.call_args[0]
    assert 'album, singers' in sql
    assert '%s::json, %s::jsonb' in sql
    assert rows[0][14] == '{"id": 20, "title": "专辑"}'


def test_upsert_kugou_songs_writes_provider_and_source_data():
    cur = MagicMock()
    upsert_kugou_songs(cur, [{
        'song_uuid': 'song-uuid',
        'platform_song_id': 200,
        'hash': 'hash',
        'album_audio_id': 300,
        'album_id': 20,
        'cover': 'https://example.com/cover.jpg',
        'title': '标题',
        'name': '词曲名',
        'duration': 180,
        'lyric': '歌词',
        'composer_name': '曲作者',
        'lyricist_name': '词作者',
        'url': 'https://example.com/audio.mp3',
        'lyric_url': 'https://example.com/lyric.lrc',
        'platform_index_url': 'https://www.kugou.com/song/#hash=hash',
        'published_at': '2020-01-01',
        'singers_json': '[]',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 200}',
    }])

    sql, rows = cur.executemany.call_args[0]
    assert 'provider_name, crawler_source_data' in sql
    assert '%s::json' in sql
    assert rows[0][-2:] == ('yinyan', '{"id": 200}')


def test_upsert_kugou_singers_writes_provider_and_source_data():
    cur = MagicMock()
    upsert_kugou_singers(cur, [{
        'id': 1,
        'name': '歌手',
        'avatar': 'https://example.com/avatar.jpg',
        'sex': 'M',
        'area': '华语',
        'index': 'G',
        'intro': '简介',
        'home_url': 'https://example.com/singer',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 1}',
    }])

    sql, rows = cur.executemany.call_args[0]
    assert 'provider_name, crawler_source_data' in sql
    assert rows[0][-2:] == ('yinyan', '{"id": 1}')


def test_upsert_kugou_albums_writes_provider_and_source_data():
    cur = MagicMock()
    upsert_kugou_albums(cur, [{
        'id': 20,
        'cover': 'https://example.com/album.jpg',
        'title': '专辑',
        'intro': '简介',
        'type': '专辑类型',
        'company_id': 7,
        'company': '唱片公司',
        'is_owner': 1,
        'published_at': '2020-01-01',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 20}',
    }])

    sql, rows = cur.executemany.call_args[0]
    assert 'provider_name, crawler_source_data' in sql
    assert rows[0][-2:] == ('yinyan', '{"id": 20}')


def test_upsert_qq_entities_write_provider_and_source_data():
    cur = MagicMock()
    upsert_qq_singers(cur, [{
        'id': 1,
        'mid': 'singer-mid',
        'name': '歌手',
        'avatar': 'https://example.com/avatar.jpg',
        'sex': 'M',
        'area': '华语',
        'index': 'G',
        'intro': '简介',
        'home_url': 'https://example.com/singer',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 1}',
    }])
    sql, rows = cur.executemany.call_args[0]
    assert 'provider_name, crawler_source_data' in sql
    assert rows[0][-2:] == ('yinyan', '{"id": 1}')

    cur = MagicMock()
    upsert_qq_albums(cur, [{
        'id': 20,
        'mid': 'album-mid',
        'cover': 'https://example.com/album.jpg',
        'title': '专辑',
        'intro': '简介',
        'type': '专辑类型',
        'company_id': 7,
        'company': '唱片公司',
        'is_owner': 1,
        'published_at': '2020-01-01',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 20}',
    }])
    sql, rows = cur.executemany.call_args[0]
    assert 'provider_name, crawler_source_data' in sql
    assert rows[0][-2:] == ('yinyan', '{"id": 20}')

    cur = MagicMock()
    upsert_qq_songs(cur, [{
        'song_uuid': 'song-uuid',
        'platform_song_id': 200,
        'mid': 'song-mid',
        'album_id': 20,
        'cover': 'https://example.com/cover.jpg',
        'title': '标题',
        'name': '词曲名',
        'duration': 180,
        'lyric': '歌词',
        'composer_name': '曲作者',
        'lyricist_name': '词作者',
        'url': 'https://example.com/audio.mp3',
        'lyric_url': 'https://example.com/lyric.lrc',
        'platform_index_url': 'https://y.qq.com/n/ryqq/songDetail/song-mid',
        'published_at': '2020-01-01',
        'singers_json': '[]',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 200}',
    }])
    sql, rows = cur.executemany.call_args[0]
    assert 'provider_name, crawler_source_data' in sql
    assert rows[0][-2:] == ('yinyan', '{"id": 200}')


def test_upsert_netease_entities_write_provider_and_source_data():
    cur = MagicMock()
    upsert_netease_singers(cur, [{
        'id': 1,
        'name': '歌手',
        'avatar': 'https://example.com/avatar.jpg',
        'sex': 'M',
        'area': '华语',
        'index': 'G',
        'intro': '简介',
        'home_url': 'https://example.com/singer',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 1}',
    }])
    sql, rows = cur.executemany.call_args[0]
    assert 'provider_name, crawler_source_data' in sql
    assert rows[0][-2:] == ('yinyan', '{"id": 1}')

    cur = MagicMock()
    upsert_netease_albums(cur, [{
        'id': 20,
        'cover': 'https://example.com/album.jpg',
        'title': '专辑',
        'intro': '简介',
        'type': '专辑类型',
        'company_id': 7,
        'company': '唱片公司',
        'is_owner': 1,
        'published_at': '2020-01-01',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 20}',
    }])
    sql, rows = cur.executemany.call_args[0]
    assert 'provider_name, crawler_source_data' in sql
    assert rows[0][-2:] == ('yinyan', '{"id": 20}')

    cur = MagicMock()
    upsert_netease_songs(cur, [{
        'song_uuid': 'song-uuid',
        'platform_song_id': 300,
        'album_id': 20,
        'album_json': '{"id": 20, "title": "专辑"}',
        'cover': 'https://example.com/cover.jpg',
        'title': '标题',
        'name': '词曲名',
        'duration': 180,
        'lyric': '歌词',
        'composer_name': '曲作者',
        'lyricist_name': '词作者',
        'url': 'https://example.com/audio.mp3',
        'lyric_url': 'https://example.com/lyric.lrc',
        'platform_index_url': 'https://music.163.com/#/song?id=300',
        'published_at': '2020-01-01',
        'singers_json': '[]',
        'provider_name': 'yinyan',
        'crawler_source_data': '{"id": 300}',
    }])
    sql, rows = cur.executemany.call_args[0]
    assert 'provider_name, crawler_source_data' in sql
    assert rows[0][-2:] == ('yinyan', '{"id": 300}')