test_writer.py
4.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
from unittest.mock import MagicMock, call
from etl_to_crawler.writer import (
upsert_kugou_albums,
upsert_kugou_singers,
upsert_kugou_songs,
upsert_netease_songs,
upsert_qq_singers,
upsert_qq_singer_songs,
upsert_yinyan_song_records,
)
def test_upsert_qq_singers_executes_insert():
cur = MagicMock()
singers = [{
'id': 1, 'mid': 'abc', 'name': '歌手', 'avatar': 'https://archive-dev.oss-cn-beijing.aliyuncs.com/a.jpg',
'sex': 'M', 'area': '华语', 'index': 'G', 'intro': None, 'home_url': None
}]
upsert_qq_singers(cur, singers)
assert cur.executemany.called
args = cur.executemany.call_args
assert 'crawler_qqmusic_singers' in args[0][0]
assert 'ON CONFLICT' in args[0][0]
def test_upsert_qq_singers_empty_does_nothing():
cur = MagicMock()
upsert_qq_singers(cur, [])
cur.executemany.assert_not_called()
def test_upsert_qq_singer_songs():
cur = MagicMock()
upsert_qq_singer_songs(cur, [(1, 'uuid-abc'), (2, 'uuid-def')])
assert cur.executemany.called
sql = cur.executemany.call_args[0][0]
assert 'crawler_qqmusic_singer_songs' in sql
def test_upsert_yinyan_song_records_replaces_existing_song_relation():
cur = MagicMock()
upsert_yinyan_song_records(cur, [(10, 100), (11, 101)])
assert cur.executemany.call_count == 2
delete_sql, delete_rows = cur.executemany.call_args_list[0][0]
insert_sql, insert_rows = cur.executemany.call_args_list[1][0]
assert 'DELETE FROM yinyan_song_records' in delete_sql
assert 'WHERE song_id = %s' in delete_sql
assert delete_rows == [(10,), (11,)]
assert 'INSERT INTO yinyan_song_records' in insert_sql
assert insert_rows == [(10, 100), (11, 101)]
def test_upsert_netease_songs_writes_album_json_column():
cur = MagicMock()
upsert_netease_songs(cur, [{
'song_uuid': 'song-uuid',
'platform_song_id': 300,
'album_id': 20,
'album_json': '{"id": 20, "title": "专辑"}',
'cover': 'https://example.com/cover.jpg',
'title': '标题',
'name': '词曲名',
'duration': 180,
'lyric': '歌词',
'composer_name': '曲作者',
'lyricist_name': '词作者',
'url': 'https://example.com/audio.mp3',
'lyric_url': 'https://example.com/lyric.lrc',
'platform_index_url': 'https://music.163.com/#/song?id=300',
'published_at': '2020-01-01',
'singers_json': '[]',
}])
sql, rows = cur.executemany.call_args[0]
assert 'album, singers' in sql
assert '%s::json, %s::jsonb' in sql
assert rows[0][-2] == '{"id": 20, "title": "专辑"}'
def test_upsert_kugou_songs_writes_provider_and_source_data():
cur = MagicMock()
upsert_kugou_songs(cur, [{
'song_uuid': 'song-uuid',
'platform_song_id': 200,
'hash': 'hash',
'album_audio_id': 300,
'album_id': 20,
'cover': 'https://example.com/cover.jpg',
'title': '标题',
'name': '词曲名',
'duration': 180,
'lyric': '歌词',
'composer_name': '曲作者',
'lyricist_name': '词作者',
'url': 'https://example.com/audio.mp3',
'lyric_url': 'https://example.com/lyric.lrc',
'platform_index_url': 'https://www.kugou.com/song/#hash=hash',
'published_at': '2020-01-01',
'singers_json': '[]',
'provider_name': 'kugou',
'crawler_source_data': '{"id": 200}',
}])
sql, rows = cur.executemany.call_args[0]
assert 'provider_name, crawler_source_data' in sql
assert '%s::json' in sql
assert rows[0][-2:] == ('kugou', '{"id": 200}')
def test_upsert_kugou_singers_writes_provider_and_source_data():
cur = MagicMock()
upsert_kugou_singers(cur, [{
'id': 1,
'name': '歌手',
'avatar': 'https://example.com/avatar.jpg',
'sex': 'M',
'area': '华语',
'index': 'G',
'intro': '简介',
'home_url': 'https://example.com/singer',
'provider_name': 'kugou',
'crawler_source_data': '{"id": 1}',
}])
sql, rows = cur.executemany.call_args[0]
assert 'provider_name, crawler_source_data' in sql
assert rows[0][-2:] == ('kugou', '{"id": 1}')
def test_upsert_kugou_albums_writes_provider_and_source_data():
cur = MagicMock()
upsert_kugou_albums(cur, [{
'id': 20,
'cover': 'https://example.com/album.jpg',
'title': '专辑',
'intro': '简介',
'type': '专辑类型',
'company_id': 7,
'company': '唱片公司',
'is_owner': 1,
'published_at': '2020-01-01',
'provider_name': 'kugou',
'crawler_source_data': '{"id": 20}',
}])
sql, rows = cur.executemany.call_args[0]
assert 'provider_name, crawler_source_data' in sql
assert rows[0][-2:] == ('kugou', '{"id": 20}')