Commit 65615e7b 65615e7ba1bf368e72f2ef89e207a4fec7914fb6 by 沈秋雨

refactor(lyrics-cache): 重构歌词缓存读取写入逻辑并同步新增歌词缓存

- 抽取歌词缓存读取和写入为独立函数 _read_existing_lyrics_cache 与 _write_existing_lyrics_cache
- load_existing_l2_candidates 中改用新缓存读写函数,增加缓存加载日志和进度输出
- 处理下载异常时保证已下载缓存的歌词及时持久化,避免丢失数据
- 新增 sync_inserted_lyrics_cache 函数实现新增歌词缓存同步写入功能
- 后台写入线程写入数据库后调用歌词缓存同步,记录新增条数日志
- 修改批量写入队列数据结构,传递缓存同步相关数据
- 优化去重统计日志显示格式,新增 format_dedup_stats 函数统一格式化
- L2审核界面新增 skip 筛选选项,调整相关前端和后端代码支持该状态
- 移除 L2 审核界面中保存按钮,避免误操作手动保存
- 补充单元测试覆盖缓存同步、缓存读取异常处理和去重统计日志格式输出
1 parent 93b03ea9
...@@ -466,6 +466,7 @@ ...@@ -466,6 +466,7 @@
466 <option value="new">new</option> 466 <option value="new">new</option>
467 <option value="merge">merge</option> 467 <option value="merge">merge</option>
468 <option value="review">review</option> 468 <option value="review">review</option>
469 <option value="skip">skip</option>
469 </select> 470 </select>
470 <input id="searchInput" type="search" placeholder="搜索歌名 / ID"> 471 <input id="searchInput" type="search" placeholder="搜索歌名 / ID">
471 <label for="pageSizeSelect">每页</label> 472 <label for="pageSizeSelect">每页</label>
...@@ -858,7 +859,6 @@ ...@@ -858,7 +859,6 @@
858 ${value === 'duplicate' ? '确认重复' : value === 'not_duplicate' ? '确认不重复' : '待确认'} 859 ${value === 'duplicate' ? '确认重复' : value === 'not_duplicate' ? '确认不重复' : '待确认'}
859 </button>`).join('')} 860 </button>`).join('')}
860 <input id="reviewNote" value="${esc(selectedReview.note || '')}" placeholder="人工备注"> 861 <input id="reviewNote" value="${esc(selectedReview.note || '')}" placeholder="人工备注">
861 <button id="saveReviewBtn" class="secondary">保存</button>
862 <button id="deleteReviewBtn" class="secondary" style="color:#c0392b;border-color:#c0392b">删除</button> 862 <button id="deleteReviewBtn" class="secondary" style="color:#c0392b;border-color:#c0392b">删除</button>
863 </div> 863 </div>
864 `} 864 `}
...@@ -906,14 +906,6 @@ ...@@ -906,14 +906,6 @@
906 renderDetail(); 906 renderDetail();
907 }); 907 });
908 } 908 }
909 const saveBtn = document.getElementById('saveReviewBtn');
910 if (saveBtn) {
911 saveBtn.addEventListener('click', () => {
912 if (!candidate) return;
913 setReview(candidate, { note: document.getElementById('reviewNote')?.value || '' });
914 renderDetail();
915 });
916 }
917 const deleteBtn = document.getElementById('deleteReviewBtn'); 909 const deleteBtn = document.getElementById('deleteReviewBtn');
918 if (deleteBtn) { 910 if (deleteBtn) {
919 deleteBtn.addEventListener('click', async () => { 911 deleteBtn.addEventListener('click', async () => {
......
...@@ -174,8 +174,8 @@ def _staging_groups_response(mode: str, term: str, page: int, page_size: int, de ...@@ -174,8 +174,8 @@ def _staging_groups_response(mode: str, term: str, page: int, page_size: int, de
174 # hits 模式下只展示有命中的记录(merge/review) 174 # hits 模式下只展示有命中的记录(merge/review)
175 if mode == "hits": 175 if mode == "hits":
176 where_clauses.append("dedup_action IN ('merge', 'review')") 176 where_clauses.append("dedup_action IN ('merge', 'review')")
177 # 决策筛选:new / merge / review 177 # 决策筛选:new / merge / review / skip
178 if decision and decision in ('new', 'merge', 'review'): 178 if decision and decision in ('new', 'merge', 'review', 'skip'):
179 where_clauses.append("dedup_action = %s") 179 where_clauses.append("dedup_action = %s")
180 params.append(decision) 180 params.append(decision)
181 where = "WHERE " + " AND ".join(where_clauses) if where_clauses else "" 181 where = "WHERE " + " AND ".join(where_clauses) if where_clauses else ""
...@@ -640,7 +640,7 @@ def _groups_response(path: Path, top_k: str, mode: str, term: str, page: int, pa ...@@ -640,7 +640,7 @@ def _groups_response(path: Path, top_k: str, mode: str, term: str, page: int, pa
640 if mode == "hits": 640 if mode == "hits":
641 groups = [g for g in groups if g.get("has_hit")] 641 groups = [g for g in groups if g.get("has_hit")]
642 # 决策筛选 642 # 决策筛选
643 if decision and decision in ('new', 'merge', 'review', 'duplicate'): 643 if decision and decision in ('new', 'merge', 'review', 'duplicate', 'skip'):
644 groups = [g for g in groups if g.get("decision") == decision or (decision == 'merge' and g.get("decision") == 'merge')] 644 groups = [g for g in groups if g.get("decision") == decision or (decision == 'merge' and g.get("decision") == 'merge')]
645 filtered = [group for group in groups if _matches_term(group, term)] 645 filtered = [group for group in groups if _matches_term(group, term)]
646 start = max(0, (page - 1) * page_size) 646 start = max(0, (page - 1) * page_size)
......
...@@ -43,6 +43,8 @@ from import_hk_songs import ( ...@@ -43,6 +43,8 @@ from import_hk_songs import (
43 process_lyrics, 43 process_lyrics,
44 DedupReport, 44 DedupReport,
45 L2CandidateIndex, 45 L2CandidateIndex,
46 format_dedup_stats,
47 sync_inserted_lyrics_cache,
46 load_approved_import_ids, 48 load_approved_import_ids,
47 load_existing_l2_candidates, 49 load_existing_l2_candidates,
48 ) 50 )
...@@ -140,6 +142,26 @@ class TestPendingL1Index: ...@@ -140,6 +142,26 @@ class TestPendingL1Index:
140 assert len(l1_index) == 1 142 assert len(l1_index) == 1
141 143
142 144
145 class TestDedupStatsLog:
146 """测试去重统计日志文案"""
147
148 def test_l1_counter_is_labeled_as_match_not_skip(self):
149 stats = {
150 'l1_dup': 2,
151 'l2_dup': 2,
152 'l2_review': 23,
153 'l2_new': 1725,
154 'author_merged': 1,
155 'skipped': 0,
156 }
157
158 message = format_dedup_stats(stats)
159
160 assert 'L1命中=2' in message
161 assert 'L1跳过' not in message
162 assert '最终跳过=0' in message
163
164
143 class TestDbWriterShutdown: 165 class TestDbWriterShutdown:
144 """测试后台 DB 写入线程的优雅停止""" 166 """测试后台 DB 写入线程的优雅停止"""
145 167
...@@ -213,6 +235,72 @@ class TestExistingLyricsCache: ...@@ -213,6 +235,72 @@ class TestExistingLyricsCache:
213 assert records[0].lyrics == '缓存里的歌词\n第二行' 235 assert records[0].lyrics == '缓存里的歌词\n第二行'
214 assert stats == {'cached': 1, 'downloaded': 1, 'failed': 0} 236 assert stats == {'cached': 1, 'downloaded': 1, 'failed': 0}
215 237
238 def test_load_existing_l2_candidates_persists_successes_before_later_download_exception(self, tmp_path):
239 cache_path = tmp_path / 'lyrics_cache.jsonl'
240 rows = [
241 {
242 'id': 100,
243 'lyrics_url': 'https://oss.example.test/100.txt',
244 'name': '第一首',
245 'singer': '歌手',
246 'lyricist': '词',
247 'composer': '曲',
248 },
249 {
250 'id': 101,
251 'lyrics_url': 'https://oss.example.test/101.txt',
252 'name': '第二首',
253 'singer': '歌手',
254 'lyricist': '词',
255 'composer': '曲',
256 },
257 ]
258
259 def downloader(url, timeout=10):
260 if url.endswith('/101.txt'):
261 raise RuntimeError('network interrupted')
262 return '已下载歌词'.encode('utf-8'), 'text/plain'
263
264 with pytest.raises(RuntimeError, match='network interrupted'):
265 load_existing_l2_candidates(rows, cache_path, downloader=downloader)
266
267 cache_lines = cache_path.read_text(encoding='utf-8').splitlines()
268 assert len(cache_lines) == 1
269 cached = json.loads(cache_lines[0])
270 assert cached['id'] == '100'
271 assert cached['lyrics'] == '已下载歌词'
272
273 def test_sync_inserted_lyrics_cache_adds_committed_inserted_rows(self, tmp_path):
274 cache_path = tmp_path / 'lyrics_cache.jsonl'
275 cache_path.write_text(
276 json.dumps({
277 'id': '100',
278 'url': 'https://oss.example.test/100.txt',
279 'lyrics': '已有歌词',
280 }, ensure_ascii=False) + '\n',
281 encoding='utf-8',
282 )
283 tuple_row = [None] * 45
284 tuple_row[0] = 200
285 tuple_row[1] = '新歌'
286 tuple_row[2] = '新词'
287 tuple_row[3] = '新曲'
288 tuple_row[8] = 'https://oss.example.test/200.txt'
289 tuple_row[33] = '新歌手'
290 source_row = {'lyrics_txt_content': '新歌词正文'}
291
292 added = sync_inserted_lyrics_cache(cache_path, [(tuple(tuple_row), source_row)])
293
294 cache_items = [
295 json.loads(line)
296 for line in cache_path.read_text(encoding='utf-8').splitlines()
297 ]
298 assert added == 1
299 assert len(cache_items) == 2
300 assert cache_items[1]['id'] == '200'
301 assert cache_items[1]['url'] == 'https://oss.example.test/200.txt'
302 assert cache_items[1]['lyrics'] == '新歌词正文'
303
216 304
217 # ==================================================================== 305 # ====================================================================
218 # L1 元数据去重 check_l1 306 # L1 元数据去重 check_l1
......