Skip to content

沈秋雨 / import_hk_songs

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 65615e7b 65615e7ba1bf368e72f2ef89e207a4fec7914fb6 by 沈秋雨
Options

refactor(lyrics-cache): 重构歌词缓存读取写入逻辑并同步新增歌词缓存 

- 抽取歌词缓存读取和写入为独立函数 _read_existing_lyrics_cache 与 _write_existing_lyrics_cache
- load_existing_l2_candidates 中改用新缓存读写函数，增加缓存加载日志和进度输出
- 处理下载异常时保证已下载缓存的歌词及时持久化，避免丢失数据
- 新增 sync_inserted_lyrics_cache 函数实现新增歌词缓存同步写入功能
- 后台写入线程写入数据库后调用歌词缓存同步，记录新增条数日志
- 修改批量写入队列数据结构，传递缓存同步相关数据
- 优化去重统计日志显示格式，新增 format_dedup_stats 函数统一格式化
- L2审核界面新增 skip 筛选选项，调整相关前端和后端代码支持该状态
- 移除 L2 审核界面中保存按钮，避免误操作手动保存
- 补充单元测试覆盖缓存同步、缓存读取异常处理和去重统计日志格式输出
1 parent 93b03ea9
Expand all
Inline Side-by-side
Showing 4 changed files with 92 additions and 12 deletions
......@@ -466,6 +466,7 @@
<option value="new">new</option>
<option value="merge">merge</option>
<option value="review">review</option>
<option value="skip">skip</option>
</select>
<input id="searchInput" type="search" placeholder="搜索歌名 / ID">
<label for="pageSizeSelect">每页</label>
......@@ -858,7 +859,6 @@
${value === 'duplicate' ? '确认重复' : value === 'not_duplicate' ? '确认不重复' : '待确认'}
</button>`).join('')}
<input id="reviewNote" value="${esc(selectedReview.note || '')}" placeholder="人工备注">
<button id="saveReviewBtn" class="secondary">保存</button>
<button id="deleteReviewBtn" class="secondary" style="color:#c0392b;border-color:#c0392b">删除</button>
</div>
`}
......@@ -906,14 +906,6 @@
renderDetail();
});
}
const saveBtn = document.getElementById('saveReviewBtn');
if (saveBtn) {
saveBtn.addEventListener('click', () => {
if (!candidate) return;
setReview(candidate, { note: document.getElementById('reviewNote')?.value || '' });
renderDetail();
});
}
const deleteBtn = document.getElementById('deleteReviewBtn');
if (deleteBtn) {
deleteBtn.addEventListener('click', async () => {
......
......@@ -174,8 +174,8 @@ def _staging_groups_response(mode: str, term: str, page: int, page_size: int, de
# hits 模式下只展示有命中的记录（merge/review）
if mode == "hits":
where_clauses.append("dedup_action IN ('merge', 'review')")
# 决策筛选：new / merge / review
if decision and decision in ('new', 'merge', 'review'):
# 决策筛选：new / merge / review / skip
if decision and decision in ('new', 'merge', 'review', 'skip'):
where_clauses.append("dedup_action = %s")
params.append(decision)
where = "WHERE " + " AND ".join(where_clauses) if where_clauses else ""
......@@ -640,7 +640,7 @@ def _groups_response(path: Path, top_k: str, mode: str, term: str, page: int, pa
if mode == "hits":
groups = [g for g in groups if g.get("has_hit")]
# 决策筛选
if decision and decision in ('new', 'merge', 'review', 'duplicate'):
if decision and decision in ('new', 'merge', 'review', 'duplicate', 'skip'):
groups = [g for g in groups if g.get("decision") == decision or (decision == 'merge' and g.get("decision") == 'merge')]
filtered = [group for group in groups if _matches_term(group, term)]
start = max(0, (page - 1) * page_size)
......
......@@ -43,6 +43,8 @@ from import_hk_songs import (
process_lyrics,
DedupReport,
L2CandidateIndex,
format_dedup_stats,
sync_inserted_lyrics_cache,
load_approved_import_ids,
load_existing_l2_candidates,
)
......@@ -140,6 +142,26 @@ class TestPendingL1Index:
assert len(l1_index) == 1
class TestDedupStatsLog:
"""测试去重统计日志文案"""
def test_l1_counter_is_labeled_as_match_not_skip(self):
stats = {
'l1_dup': 2,
'l2_dup': 2,
'l2_review': 23,
'l2_new': 1725,
'author_merged': 1,
'skipped': 0,
}
message = format_dedup_stats(stats)
assert 'L1命中=2' in message
assert 'L1跳过' not in message
assert '最终跳过=0' in message
class TestDbWriterShutdown:
"""测试后台 DB 写入线程的优雅停止"""
......@@ -213,6 +235,72 @@ class TestExistingLyricsCache:
assert records[0].lyrics == '缓存里的歌词\n第二行'
assert stats == {'cached': 1, 'downloaded': 1, 'failed': 0}
def test_load_existing_l2_candidates_persists_successes_before_later_download_exception(self, tmp_path):
cache_path = tmp_path / 'lyrics_cache.jsonl'
rows = [
{
'id': 100,
'lyrics_url': 'https://oss.example.test/100.txt',
'name': '第一首',
'singer': '歌手',
'lyricist': '词',
'composer': '曲',
},
{
'id': 101,
'lyrics_url': 'https://oss.example.test/101.txt',
'name': '第二首',
'singer': '歌手',
'lyricist': '词',
'composer': '曲',
},
]
def downloader(url, timeout=10):
if url.endswith('/101.txt'):
raise RuntimeError('network interrupted')
return '已下载歌词'.encode('utf-8'), 'text/plain'
with pytest.raises(RuntimeError, match='network interrupted'):
load_existing_l2_candidates(rows, cache_path, downloader=downloader)
cache_lines = cache_path.read_text(encoding='utf-8').splitlines()
assert len(cache_lines) == 1
cached = json.loads(cache_lines[0])
assert cached['id'] == '100'
assert cached['lyrics'] == '已下载歌词'
def test_sync_inserted_lyrics_cache_adds_committed_inserted_rows(self, tmp_path):
cache_path = tmp_path / 'lyrics_cache.jsonl'
cache_path.write_text(
json.dumps({
'id': '100',
'url': 'https://oss.example.test/100.txt',
'lyrics': '已有歌词',
}, ensure_ascii=False) + '\n',
encoding='utf-8',
)
tuple_row = [None] * 45
tuple_row[0] = 200
tuple_row[1] = '新歌'
tuple_row[2] = '新词'
tuple_row[3] = '新曲'
tuple_row[8] = 'https://oss.example.test/200.txt'
tuple_row[33] = '新歌手'
source_row = {'lyrics_txt_content': '新歌词正文'}
added = sync_inserted_lyrics_cache(cache_path, [(tuple(tuple_row), source_row)])
cache_items = [
json.loads(line)
for line in cache_path.read_text(encoding='utf-8').splitlines()
]
assert added == 1
assert len(cache_items) == 2
assert cache_items[1]['id'] == '200'
assert cache_items[1]['url'] == 'https://oss.example.test/200.txt'
assert cache_items[1]['lyrics'] == '新歌词正文'
# ====================================================================
# L1 元数据去重 check_l1
......