refactor(lyrics-cache): 重构歌词缓存读取写入逻辑并同步新增歌词缓存
- 抽取歌词缓存读取和写入为独立函数 _read_existing_lyrics_cache 与 _write_existing_lyrics_cache - load_existing_l2_candidates 中改用新缓存读写函数,增加缓存加载日志和进度输出 - 处理下载异常时保证已下载缓存的歌词及时持久化,避免丢失数据 - 新增 sync_inserted_lyrics_cache 函数实现新增歌词缓存同步写入功能 - 后台写入线程写入数据库后调用歌词缓存同步,记录新增条数日志 - 修改批量写入队列数据结构,传递缓存同步相关数据 - 优化去重统计日志显示格式,新增 format_dedup_stats 函数统一格式化 - L2审核界面新增 skip 筛选选项,调整相关前端和后端代码支持该状态 - 移除 L2 审核界面中保存按钮,避免误操作手动保存 - 补充单元测试覆盖缓存同步、缓存读取异常处理和去重统计日志格式输出
Showing
4 changed files
with
92 additions
and
12 deletions
This diff is collapsed.
Click to expand it.
| ... | @@ -466,6 +466,7 @@ | ... | @@ -466,6 +466,7 @@ |
| 466 | <option value="new">new</option> | 466 | <option value="new">new</option> |
| 467 | <option value="merge">merge</option> | 467 | <option value="merge">merge</option> |
| 468 | <option value="review">review</option> | 468 | <option value="review">review</option> |
| 469 | <option value="skip">skip</option> | ||
| 469 | </select> | 470 | </select> |
| 470 | <input id="searchInput" type="search" placeholder="搜索歌名 / ID"> | 471 | <input id="searchInput" type="search" placeholder="搜索歌名 / ID"> |
| 471 | <label for="pageSizeSelect">每页</label> | 472 | <label for="pageSizeSelect">每页</label> |
| ... | @@ -858,7 +859,6 @@ | ... | @@ -858,7 +859,6 @@ |
| 858 | ${value === 'duplicate' ? '确认重复' : value === 'not_duplicate' ? '确认不重复' : '待确认'} | 859 | ${value === 'duplicate' ? '确认重复' : value === 'not_duplicate' ? '确认不重复' : '待确认'} |
| 859 | </button>`).join('')} | 860 | </button>`).join('')} |
| 860 | <input id="reviewNote" value="${esc(selectedReview.note || '')}" placeholder="人工备注"> | 861 | <input id="reviewNote" value="${esc(selectedReview.note || '')}" placeholder="人工备注"> |
| 861 | <button id="saveReviewBtn" class="secondary">保存</button> | ||
| 862 | <button id="deleteReviewBtn" class="secondary" style="color:#c0392b;border-color:#c0392b">删除</button> | 862 | <button id="deleteReviewBtn" class="secondary" style="color:#c0392b;border-color:#c0392b">删除</button> |
| 863 | </div> | 863 | </div> |
| 864 | `} | 864 | `} |
| ... | @@ -906,14 +906,6 @@ | ... | @@ -906,14 +906,6 @@ |
| 906 | renderDetail(); | 906 | renderDetail(); |
| 907 | }); | 907 | }); |
| 908 | } | 908 | } |
| 909 | const saveBtn = document.getElementById('saveReviewBtn'); | ||
| 910 | if (saveBtn) { | ||
| 911 | saveBtn.addEventListener('click', () => { | ||
| 912 | if (!candidate) return; | ||
| 913 | setReview(candidate, { note: document.getElementById('reviewNote')?.value || '' }); | ||
| 914 | renderDetail(); | ||
| 915 | }); | ||
| 916 | } | ||
| 917 | const deleteBtn = document.getElementById('deleteReviewBtn'); | 909 | const deleteBtn = document.getElementById('deleteReviewBtn'); |
| 918 | if (deleteBtn) { | 910 | if (deleteBtn) { |
| 919 | deleteBtn.addEventListener('click', async () => { | 911 | deleteBtn.addEventListener('click', async () => { | ... | ... |
| ... | @@ -174,8 +174,8 @@ def _staging_groups_response(mode: str, term: str, page: int, page_size: int, de | ... | @@ -174,8 +174,8 @@ def _staging_groups_response(mode: str, term: str, page: int, page_size: int, de |
| 174 | # hits 模式下只展示有命中的记录(merge/review) | 174 | # hits 模式下只展示有命中的记录(merge/review) |
| 175 | if mode == "hits": | 175 | if mode == "hits": |
| 176 | where_clauses.append("dedup_action IN ('merge', 'review')") | 176 | where_clauses.append("dedup_action IN ('merge', 'review')") |
| 177 | # 决策筛选:new / merge / review | 177 | # 决策筛选:new / merge / review / skip |
| 178 | if decision and decision in ('new', 'merge', 'review'): | 178 | if decision and decision in ('new', 'merge', 'review', 'skip'): |
| 179 | where_clauses.append("dedup_action = %s") | 179 | where_clauses.append("dedup_action = %s") |
| 180 | params.append(decision) | 180 | params.append(decision) |
| 181 | where = "WHERE " + " AND ".join(where_clauses) if where_clauses else "" | 181 | where = "WHERE " + " AND ".join(where_clauses) if where_clauses else "" |
| ... | @@ -640,7 +640,7 @@ def _groups_response(path: Path, top_k: str, mode: str, term: str, page: int, pa | ... | @@ -640,7 +640,7 @@ def _groups_response(path: Path, top_k: str, mode: str, term: str, page: int, pa |
| 640 | if mode == "hits": | 640 | if mode == "hits": |
| 641 | groups = [g for g in groups if g.get("has_hit")] | 641 | groups = [g for g in groups if g.get("has_hit")] |
| 642 | # 决策筛选 | 642 | # 决策筛选 |
| 643 | if decision and decision in ('new', 'merge', 'review', 'duplicate'): | 643 | if decision and decision in ('new', 'merge', 'review', 'duplicate', 'skip'): |
| 644 | groups = [g for g in groups if g.get("decision") == decision or (decision == 'merge' and g.get("decision") == 'merge')] | 644 | groups = [g for g in groups if g.get("decision") == decision or (decision == 'merge' and g.get("decision") == 'merge')] |
| 645 | filtered = [group for group in groups if _matches_term(group, term)] | 645 | filtered = [group for group in groups if _matches_term(group, term)] |
| 646 | start = max(0, (page - 1) * page_size) | 646 | start = max(0, (page - 1) * page_size) | ... | ... |
| ... | @@ -43,6 +43,8 @@ from import_hk_songs import ( | ... | @@ -43,6 +43,8 @@ from import_hk_songs import ( |
| 43 | process_lyrics, | 43 | process_lyrics, |
| 44 | DedupReport, | 44 | DedupReport, |
| 45 | L2CandidateIndex, | 45 | L2CandidateIndex, |
| 46 | format_dedup_stats, | ||
| 47 | sync_inserted_lyrics_cache, | ||
| 46 | load_approved_import_ids, | 48 | load_approved_import_ids, |
| 47 | load_existing_l2_candidates, | 49 | load_existing_l2_candidates, |
| 48 | ) | 50 | ) |
| ... | @@ -140,6 +142,26 @@ class TestPendingL1Index: | ... | @@ -140,6 +142,26 @@ class TestPendingL1Index: |
| 140 | assert len(l1_index) == 1 | 142 | assert len(l1_index) == 1 |
| 141 | 143 | ||
| 142 | 144 | ||
| 145 | class TestDedupStatsLog: | ||
| 146 | """测试去重统计日志文案""" | ||
| 147 | |||
| 148 | def test_l1_counter_is_labeled_as_match_not_skip(self): | ||
| 149 | stats = { | ||
| 150 | 'l1_dup': 2, | ||
| 151 | 'l2_dup': 2, | ||
| 152 | 'l2_review': 23, | ||
| 153 | 'l2_new': 1725, | ||
| 154 | 'author_merged': 1, | ||
| 155 | 'skipped': 0, | ||
| 156 | } | ||
| 157 | |||
| 158 | message = format_dedup_stats(stats) | ||
| 159 | |||
| 160 | assert 'L1命中=2' in message | ||
| 161 | assert 'L1跳过' not in message | ||
| 162 | assert '最终跳过=0' in message | ||
| 163 | |||
| 164 | |||
| 143 | class TestDbWriterShutdown: | 165 | class TestDbWriterShutdown: |
| 144 | """测试后台 DB 写入线程的优雅停止""" | 166 | """测试后台 DB 写入线程的优雅停止""" |
| 145 | 167 | ||
| ... | @@ -213,6 +235,72 @@ class TestExistingLyricsCache: | ... | @@ -213,6 +235,72 @@ class TestExistingLyricsCache: |
| 213 | assert records[0].lyrics == '缓存里的歌词\n第二行' | 235 | assert records[0].lyrics == '缓存里的歌词\n第二行' |
| 214 | assert stats == {'cached': 1, 'downloaded': 1, 'failed': 0} | 236 | assert stats == {'cached': 1, 'downloaded': 1, 'failed': 0} |
| 215 | 237 | ||
| 238 | def test_load_existing_l2_candidates_persists_successes_before_later_download_exception(self, tmp_path): | ||
| 239 | cache_path = tmp_path / 'lyrics_cache.jsonl' | ||
| 240 | rows = [ | ||
| 241 | { | ||
| 242 | 'id': 100, | ||
| 243 | 'lyrics_url': 'https://oss.example.test/100.txt', | ||
| 244 | 'name': '第一首', | ||
| 245 | 'singer': '歌手', | ||
| 246 | 'lyricist': '词', | ||
| 247 | 'composer': '曲', | ||
| 248 | }, | ||
| 249 | { | ||
| 250 | 'id': 101, | ||
| 251 | 'lyrics_url': 'https://oss.example.test/101.txt', | ||
| 252 | 'name': '第二首', | ||
| 253 | 'singer': '歌手', | ||
| 254 | 'lyricist': '词', | ||
| 255 | 'composer': '曲', | ||
| 256 | }, | ||
| 257 | ] | ||
| 258 | |||
| 259 | def downloader(url, timeout=10): | ||
| 260 | if url.endswith('/101.txt'): | ||
| 261 | raise RuntimeError('network interrupted') | ||
| 262 | return '已下载歌词'.encode('utf-8'), 'text/plain' | ||
| 263 | |||
| 264 | with pytest.raises(RuntimeError, match='network interrupted'): | ||
| 265 | load_existing_l2_candidates(rows, cache_path, downloader=downloader) | ||
| 266 | |||
| 267 | cache_lines = cache_path.read_text(encoding='utf-8').splitlines() | ||
| 268 | assert len(cache_lines) == 1 | ||
| 269 | cached = json.loads(cache_lines[0]) | ||
| 270 | assert cached['id'] == '100' | ||
| 271 | assert cached['lyrics'] == '已下载歌词' | ||
| 272 | |||
| 273 | def test_sync_inserted_lyrics_cache_adds_committed_inserted_rows(self, tmp_path): | ||
| 274 | cache_path = tmp_path / 'lyrics_cache.jsonl' | ||
| 275 | cache_path.write_text( | ||
| 276 | json.dumps({ | ||
| 277 | 'id': '100', | ||
| 278 | 'url': 'https://oss.example.test/100.txt', | ||
| 279 | 'lyrics': '已有歌词', | ||
| 280 | }, ensure_ascii=False) + '\n', | ||
| 281 | encoding='utf-8', | ||
| 282 | ) | ||
| 283 | tuple_row = [None] * 45 | ||
| 284 | tuple_row[0] = 200 | ||
| 285 | tuple_row[1] = '新歌' | ||
| 286 | tuple_row[2] = '新词' | ||
| 287 | tuple_row[3] = '新曲' | ||
| 288 | tuple_row[8] = 'https://oss.example.test/200.txt' | ||
| 289 | tuple_row[33] = '新歌手' | ||
| 290 | source_row = {'lyrics_txt_content': '新歌词正文'} | ||
| 291 | |||
| 292 | added = sync_inserted_lyrics_cache(cache_path, [(tuple(tuple_row), source_row)]) | ||
| 293 | |||
| 294 | cache_items = [ | ||
| 295 | json.loads(line) | ||
| 296 | for line in cache_path.read_text(encoding='utf-8').splitlines() | ||
| 297 | ] | ||
| 298 | assert added == 1 | ||
| 299 | assert len(cache_items) == 2 | ||
| 300 | assert cache_items[1]['id'] == '200' | ||
| 301 | assert cache_items[1]['url'] == 'https://oss.example.test/200.txt' | ||
| 302 | assert cache_items[1]['lyrics'] == '新歌词正文' | ||
| 303 | |||
| 216 | 304 | ||
| 217 | # ==================================================================== | 305 | # ==================================================================== |
| 218 | # L1 元数据去重 check_l1 | 306 | # L1 元数据去重 check_l1 | ... | ... |
-
Please register or sign in to post a comment