Skip to content
Toggle navigation
Toggle navigation
This project
Loading...
Sign in
沈秋雨
/
import_hk_songs
Go to a project
Toggle navigation
Toggle navigation pinning
Projects
Groups
Snippets
Help
Project
Activity
Repository
Pipelines
Graphs
Issues
0
Merge Requests
0
Wiki
Network
Create a new issue
Builds
Commits
Issue Boards
Files
Commits
Network
Compare
Branches
Tags
Commit
bbb6ce14
...
bbb6ce1447e193340855290021205a90c07891fe
authored
2026-07-20 14:56:01 +0800
by
沈秋雨
Browse Files
Options
Browse Files
Tag
Download
Email Patches
Plain Diff
feat(review): 优化审核结果提交和重复数据合并逻辑
1 parent
2393078b
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
151 additions
and
29 deletions
l2_review_dashboard.html
serve_l2_dashboard.py
test_serve_l2_dashboard.py
l2_review_dashboard.html
View file @
bbb6ce1
...
...
@@ -646,6 +646,7 @@
<option
value=
"unsure"
>
待确认
</option>
<option
value=
"reviewed"
>
已审核
</option>
<option
value=
"submitted"
>
已提交
</option>
<option
value=
"deleted"
>
已删除
</option>
<option
value=
""
>
全部
</option>
<option
value=
"l1_hit"
>
L1 命中
</option>
<option
value=
"conflict"
>
L1/L2 冲突
</option>
...
...
@@ -661,7 +662,7 @@
<option
value=
"200"
>
200
</option>
</select>
<button
id=
"exportBtn"
>
导出标注
</button>
<button
id=
"importReviewedBtn"
>
入库审核通过
</button>
<button
id=
"importReviewedBtn"
>
提交审核结果
</button>
<div
class=
"toolbar-right"
>
<span>
审核人:
</span><span
id=
"reviewerDisplay"
></span>
<button
id=
"logoutBtn"
class=
"secondary"
>
登出
</button>
...
...
@@ -1002,7 +1003,7 @@
group
.
review_claim_expires_at
=
record
.
review_claim_expires_at
||
''
;
group
.
claimed_by_me
=
group
.
review_claimed_by
===
state
.
reviewer
;
group
.
reviewed
=
[
'approved_import'
,
'rejected_duplicate'
,
'deleted'
].
includes
(
record
.
biz_review_status
);
group
.
submitted
=
group
.
reviewed
&&
record
.
staging_status
===
'imported'
;
group
.
submitted
=
group
.
reviewed
&&
[
'imported'
,
'skipped'
].
includes
(
record
.
staging_status
)
;
}
function
currentQuery
()
{
...
...
@@ -1059,12 +1060,12 @@
metric
(
'待审核'
,
row
.
pending_review_count
||
'0'
),
metric
(
'已审核'
,
row
.
reviewed_count
||
'0'
),
metric
(
'已提交'
,
row
.
submitted_count
||
'0'
),
metric
(
'确认
入库
'
,
row
.
approved_count
||
'0'
),
metric
(
'确认
不入库
'
,
row
.
rejected_count
||
'0'
),
metric
(
'确认
不重复
'
,
row
.
approved_count
||
'0'
),
metric
(
'确认
重复
'
,
row
.
rejected_count
||
'0'
),
metric
(
'已删除'
,
row
.
deleted_count
||
'0'
),
metric
(
'
new
'
,
row
.
new_count
||
'-'
),
metric
(
'
merge
'
,
row
.
duplicate_count
||
'0'
),
metric
(
'
skip
'
,
row
.
skip_count
||
'0'
)
metric
(
'
新歌
'
,
row
.
new_count
||
'-'
),
metric
(
'
重复
'
,
row
.
duplicate_count
||
'0'
),
metric
(
'
已跳过
'
,
row
.
skip_count
||
'0'
)
].
join
(
''
);
}
...
...
@@ -1182,20 +1183,21 @@
</div>`
;
}
async
function
updateStagingReview
(
query
,
decision
,
note
)
{
async
function
updateStagingReview
(
query
,
decision
,
note
,
candidateId
=
''
)
{
return
reviewPostJSON
(
'/api/staging-review'
,
{
staging_id
:
String
(
query
.
staging_id
),
expected_version
:
Number
(
query
.
review_version
||
0
),
client_token
:
state
.
clientToken
,
reviewer
:
state
.
reviewer
,
decision
,
note
note
,
candidate_id
:
candidateId
});
}
const
_DB_REVIEW_LABELS
=
{
approved_import
:
'确认
入库
(待入库)'
,
rejected_duplicate
:
'确认
不入库
'
,
approved_import
:
'确认
不重复
(待入库)'
,
rejected_duplicate
:
'确认
重复(待合并作者)
'
,
unsure
:
'待确认'
,
deleted
:
'已删除'
,
};
...
...
@@ -1293,8 +1295,9 @@
const
needsManualReview
=
rowDecision
(
query
)
===
'review'
;
const
hasFinalReview
=
Boolean
(
query
.
review_status
)
&&
!
[
'pending'
,
'not_required'
,
'unsure'
].
includes
(
query
.
review_status
);
const
canUndoReview
=
hasFinalReview
&&
query
.
staging_status
!==
'imported'
;
const
canDeleteRecord
=
query
.
staging_status
!==
'imported'
;
const
isSubmitted
=
[
'imported'
,
'skipped'
].
includes
(
query
.
staging_status
);
const
canUndoReview
=
hasFinalReview
&&
query
.
staging_status
===
'staged'
;
const
canDeleteRecord
=
!
[
'imported'
,
'skipped'
].
includes
(
query
.
staging_status
);
// Sticky area: 新入库歌词 + 召回候选(各含音频播放器)
els
.
stickyInfo
.
innerHTML
=
`
...
...
@@ -1339,7 +1342,7 @@
<div class="section-head">
<h2 class="section-title">人工标注</h2>
${
hasFinalReview
?
`<span class="pill
${
query
.
review_status
===
'approved_import'
?
'new'
:
query
.
review_status
===
'rejected_duplicate'
?
'duplicate'
:
query
.
review_status
===
'deleted'
?
'duplicate'
:
''
}
">
${
query
.
review_status
===
'deleted'
?
'已删除'
:
query
.
staging_status
===
'imported'
?
'已提交'
:
'已审核'
}
</span>`
?
`<span class="pill
${
query
.
review_status
===
'approved_import'
?
'new'
:
query
.
review_status
===
'rejected_duplicate'
?
'duplicate'
:
query
.
review_status
===
'deleted'
?
'duplicate'
:
''
}
">
${
query
.
review_status
===
'deleted'
?
'已删除'
:
isSubmitted
?
'已提交'
:
'已审核'
}
</span>`
:
''
}
</div>
<div class="section-body">
...
...
@@ -1347,8 +1350,10 @@
<div class="review-bar" style="align-items:center">
<span style="flex:1">
<b>
${
esc
(
query
.
review_status
===
'approved_import'
&&
query
.
staging_status
===
'imported'
?
'确认入库(已提交)'
:
(
_DB_REVIEW_LABELS
[
query
.
review_status
]
||
query
.
review_status
))}
</b>
?
'确认不重复(已入库)'
:
query
.
review_status
===
'rejected_duplicate'
&&
query
.
staging_status
===
'skipped'
?
'确认重复(作者已合并)'
:
(
_DB_REVIEW_LABELS
[
query
.
review_status
]
||
query
.
review_status
))}
</b>
${
query
.
reviewed_by
?
` · 审核人
${
esc
(
query
.
reviewed_by
)}
`
:
''
}
${
query
.
reviewed_at
?
` ·
${
esc
(
query
.
reviewed_at
)}
`
:
''
}
${
query
.
review_note
?
` · 备注:
${
esc
(
query
.
review_note
)}
`
:
''
}
...
...
@@ -1359,7 +1364,7 @@
<div class="review-bar">
${[
'duplicate'
,
'not_duplicate'
,
'unsure'
].
map
(
value
=>
`
<button class="review-choice
${
selectedReview
.
final_decision
===
value
?
'active'
:
''
}
" data-review="
${
value
}
">
${
value
===
'duplicate'
?
'确认
不入库'
:
value
===
'not_duplicate'
?
'确认入库
'
:
'待确认'
}
${
value
===
'duplicate'
?
'确认
重复'
:
value
===
'not_duplicate'
?
'确认不重复
'
:
'待确认'
}
</button>`
).
join
(
''
)}
<input id="reviewNote" value="
${
esc
(
selectedReview
.
note
||
''
)}
" placeholder="人工备注">
<button id="deleteReviewBtn" class="secondary" style="color:#c0392b;border-color:#c0392b">删除</button>
...
...
@@ -1411,7 +1416,9 @@
renderDetail
();
return
;
}
const
payload
=
await
updateStagingReview
(
query
,
dbDecision
,
note
);
const
payload
=
await
updateStagingReview
(
query
,
dbDecision
,
note
,
localDecision
===
'duplicate'
?
(
candidate
?.
candidate_id
||
''
)
:
''
);
for
(
const
row
of
state
.
queryRows
)
applyReviewSnapshot
(
row
,
payload
.
record
);
// 刷新统计和加载列表并行
await
Promise
.
all
([
reloadSummary
(),
loadGroups
()]);
...
...
@@ -1530,12 +1537,12 @@
alert('历史报表仅供查看,请在 staging-db 中执行入库。');
return;
}
if (!confirm('确认
入库数据库中所有"确认入库(待入库)"样本?
')) return;
if (!confirm('确认
提交所有审核结果?“确认不重复”将入库,“确认重复”将把词/曲作者增量合并到已有记录。
')) return;
els.importReviewedBtn.disabled = true;
els.importReviewedBtn.textContent = '入库中...';
try {
const payload = await reviewPostJSON('/api/import-reviewed');
alert(`
入库完成:
$
{
payload
.
inserted_count
}
条。结果文件:
$
{
payload
.
review_csv
}
`);
alert(`
提交完成:不重复入库
$
{
payload
.
inserted_count
}
条,重复样本
$
{
payload
.
duplicate_count
||
0
}
条,其中作者有增量
$
{
payload
.
author_merged_count
||
0
}
条。结果文件:
$
{
payload
.
review_csv
}
`);
// 刷新统计和加载列表并行
await Promise.all([reloadSummary(), loadGroups()]);
renderAll();
...
...
@@ -1543,14 +1550,17 @@
if (err.status === 409 && err.payload?.code === 'import_conflict') {
const conflicts = err.payload.conflicts || [];
const details = conflicts.slice(0, 20).map((row, index) => {
const types = (row.conflict_types || []).map(type =>
type === 'primary_key' ? '主键 ID 冲突' : '来源键冲突'
).join(' + ') || '唯一键冲突';
const typeLabels = {
primary_key: '主键 ID 冲突',
source_key: '来源键冲突',
duplicate_target_missing: '未找到已入库的重复目标'
};
const types = (row.conflict_types || []).map(type => typeLabels[type] || type).join(' + ') || '数据冲突';
const targetIds = [row.id_conflict_target_id, row.source_conflict_target_id]
.filter(value => value !== null && value !== undefined)
.filter((value, i, values) => values.indexOf(value) === i)
.join('/');
return `
$
{
index
+
1
}.
$
{
row
.
name
||
'(无歌名)'
}
|
source
=
$
{
row
.
source_table_name
||
'-'
}:
$
{
row
.
source_song_id
||
'-'
}
| staging_id=
${
row
.
staging_id
}
| 待入库ID=
${
row
.
incoming_id
||
'-'
}
|
${
types
}
| 正式表ID=
${
targetIds
||
'-'
}
`
;
return `
$
{
index
+
1
}.
$
{
row
.
name
||
'(无歌名)'
}
|
source
=
$
{
row
.
source_table_name
||
'-'
}:
$
{
row
.
source_song_id
||
'-'
}
| staging_id=
${
row
.
staging_id
}
| 待入库ID=
${
row
.
incoming_id
||
'-'
}
|
重复候选=
${
row
.
matched_song_id
||
'-'
}
|
${
types
}
| 正式表ID=
${
targetIds
||
'-'
}
`
;
});
if
(
conflicts
.
length
>
20
)
details
.
push
(
`……另有
${
conflicts
.
length
-
20
}
条未展开`
);
alert
([
...
...
@@ -1565,7 +1575,7 @@
}
}
finally
{
els
.
importReviewedBtn
.
disabled
=
false
;
els
.
importReviewedBtn
.
textContent
=
'
入库审核通过
'
;
els
.
importReviewedBtn
.
textContent
=
'
提交审核结果
'
;
}
}
...
...
@@ -1659,7 +1669,7 @@
group
.
review_claim_expires_at
=
remote
.
review_claim_expires_at
||
''
;
group
.
claimed_by_me
=
Boolean
(
remote
.
claimed_by_me
);
group
.
reviewed
=
[
'approved_import'
,
'rejected_duplicate'
,
'deleted'
].
includes
(
remote
.
biz_review_status
);
group
.
submitted
=
group
.
reviewed
&&
remote
.
staging_status
===
'imported'
;
group
.
submitted
=
group
.
reviewed
&&
[
'imported'
,
'skipped'
].
includes
(
remote
.
staging_status
)
;
}
const
query
=
currentQuery
();
const
selectedRemote
=
query
?
statuses
.
get
(
String
(
query
.
staging_id
))
:
null
;
...
...
serve_l2_dashboard.py
View file @
bbb6ce1
This diff is collapsed.
Click to expand it.
test_serve_l2_dashboard.py
View file @
bbb6ce1
...
...
@@ -96,6 +96,45 @@ def test_stale_review_returns_conflict_without_overwrite():
assert
not
conn
.
committed
def
test_rejected_duplicate_persists_the_selected_candidate
():
snapshot
=
{
"staging_id"
:
42
,
"matched_song_id"
:
None
,
"recalled_candidates"
:
'[{"id":"existing-song"}]'
,
"biz_review_status"
:
"rejected_duplicate"
,
"review_version"
:
8
,
}
cursor
=
FakeCursor
(
update_rowcount
=
1
,
snapshot
=
snapshot
)
conn
=
FakeConnection
(
cursor
)
with
patch
.
object
(
dashboard
,
"_target_conn"
,
return_value
=
conn
):
dashboard
.
_update_staging_review
(
42
,
"rejected_duplicate"
,
"same"
,
"alice"
,
7
,
"browser-token"
,
"existing-song"
)
update_sql
,
params
=
cursor
.
executions
[
1
]
assert
"matched_song_id =
%
s"
in
update_sql
assert
params
[:
3
]
==
[
"rejected_duplicate"
,
"alice"
,
"existing-song"
]
assert
conn
.
committed
def
test_rejected_duplicate_rejects_a_candidate_outside_recall_results
():
cursor
=
FakeCursor
(
update_rowcount
=
1
,
snapshot
=
{
"staging_id"
:
42
,
"matched_song_id"
:
"candidate-a"
,
"recalled_candidates"
:
"[]"
},
)
conn
=
FakeConnection
(
cursor
)
with
patch
.
object
(
dashboard
,
"_target_conn"
,
return_value
=
conn
):
with
pytest
.
raises
(
ValueError
,
match
=
"不在当前召回结果"
):
dashboard
.
_update_staging_review
(
42
,
"rejected_duplicate"
,
""
,
"alice"
,
7
,
"browser-token"
,
"candidate-b"
)
assert
conn
.
rolled_back
assert
not
any
(
sql
.
lstrip
()
.
startswith
(
"UPDATE"
)
for
sql
,
_params
in
cursor
.
executions
)
def
test_pending_review_restores_a_deleted_staging_row
():
snapshot
=
{
"staging_id"
:
42
,
...
...
@@ -185,7 +224,7 @@ class ImportCursor(FakeCursor):
def
__init__
(
self
)
->
None
:
super
()
.
__init__
(
update_rowcount
=
1
,
snapshot
=
{})
self
.
result_sets
=
[
[{
"staging_id"
:
42
,
"source_song_id"
:
"song-1"
}],
[{
"staging_id"
:
42
,
"source_song_id"
:
"song-1"
,
"biz_review_status"
:
"approved_import"
}],
[],
[{
"source_song_id"
:
"song-1"
,
"target_id"
:
9001
}],
]
...
...
@@ -217,7 +256,7 @@ def test_import_locks_only_unclaimed_staged_rows():
def
test_import_conflict_rolls_back_and_exposes_the_staging_row
():
cursor
=
ImportCursor
()
cursor
.
result_sets
=
[
[{
"staging_id"
:
42
,
"source_song_id"
:
"song-1"
}],
[{
"staging_id"
:
42
,
"source_song_id"
:
"song-1"
,
"biz_review_status"
:
"approved_import"
}],
[{
"staging_id"
:
42
,
"source_song_id"
:
"song-1"
,
...
...
@@ -241,6 +280,69 @@ def test_import_conflict_rolls_back_and_exposes_the_staging_row():
assert
not
any
(
sql
.
lstrip
()
.
startswith
(
"INSERT"
)
for
sql
,
_params
in
cursor
.
executions
)
class
DuplicateImportCursor
:
def
__init__
(
self
)
->
None
:
self
.
rowcount
=
0
self
.
executions
:
list
[
tuple
[
str
,
object
]]
=
[]
self
.
current
=
None
def
__enter__
(
self
):
return
self
def
__exit__
(
self
,
*
_args
):
return
False
def
execute
(
self
,
sql
,
params
=
None
):
self
.
executions
.
append
((
sql
,
params
))
compact
=
" "
.
join
(
sql
.
split
())
self
.
rowcount
=
0
if
"SELECT s.staging_id"
in
compact
and
"FOR UPDATE"
in
compact
:
self
.
current
=
[{
"staging_id"
:
42
,
"source_song_id"
:
"new-song"
,
"source_table_name"
:
"hk_song_platform"
,
"name"
:
"same song"
,
"lyricist"
:
"Alice/Bob"
,
"composer"
:
"Composer B"
,
"biz_review_status"
:
"rejected_duplicate"
,
"matched_song_id"
:
"existing-song"
,
}]
elif
"SELECT id FROM"
in
compact
and
"source_table_name =
%
s"
in
compact
:
self
.
current
=
{
"id"
:
9001
}
elif
"SELECT id, lyricist, composer"
in
compact
:
self
.
current
=
{
"id"
:
9001
,
"lyricist"
:
"Alice"
,
"composer"
:
"Composer A"
}
elif
compact
.
startswith
(
"UPDATE"
):
self
.
current
=
None
self
.
rowcount
=
1
else
:
raise
AssertionError
(
f
"unexpected SQL: {compact}"
)
def
fetchall
(
self
):
return
self
.
current
or
[]
def
fetchone
(
self
):
return
self
.
current
def
test_confirmed_duplicate_merges_authors_without_inserting_a_new_song
():
cursor
=
DuplicateImportCursor
()
conn
=
FakeConnection
(
cursor
)
with
patch
.
object
(
dashboard
,
"_target_conn"
,
return_value
=
conn
):
result
=
dashboard
.
_import_approved_staging
()
sql_params
=
[(
" "
.
join
(
sql
.
split
()),
params
)
for
sql
,
params
in
cursor
.
executions
]
assert
not
any
(
sql
.
startswith
(
"INSERT"
)
for
sql
,
_params
in
sql_params
)
assert
any
(
"SET lyricist =
%
s"
in
sql
and
params
==
(
"Alice、Bob"
,
9001
)
for
sql
,
params
in
sql_params
)
assert
any
(
"SET composer =
%
s"
in
sql
and
params
==
(
"Composer A、Composer B"
,
9001
)
for
sql
,
params
in
sql_params
)
assert
any
(
"SET staging_status = 'skipped'"
in
sql
for
sql
,
_params
in
sql_params
)
assert
result
[
"duplicate_count"
]
==
1
assert
result
[
"author_merged_count"
]
==
1
assert
result
[
"inserted_count"
]
==
0
assert
result
[
"duplicate_source_ids"
]
==
[
"new-song"
]
assert
conn
.
committed
def
test_browsing_a_list_item_can_auto_claim_without_reloading_groups
():
html
=
Path
(
"l2_review_dashboard.html"
)
.
read_text
(
encoding
=
"utf-8"
)
start
=
html
.
index
(
"for (const btn of els.queryList.querySelectorAll('.query-item'))"
)
...
...
@@ -274,7 +376,17 @@ def test_dashboard_distinguishes_reviewed_from_submitted():
assert
row
[
"staging_status"
]
==
"imported"
assert
'<option value="reviewed">已审核</option>'
in
html
assert
'<option value="submitted">已提交</option>'
in
html
assert
"hasFinalReview && query.staging_status !== 'imported'"
in
html
assert
"hasFinalReview && query.staging_status === 'staged'"
in
html
def
test_dashboard_has_a_dedicated_deleted_filter
():
html
=
Path
(
"l2_review_dashboard.html"
)
.
read_text
(
encoding
=
"utf-8"
)
assert
'<option value="deleted">已删除</option>'
in
html
source
=
Path
(
"serve_l2_dashboard.py"
)
.
read_text
(
encoding
=
"utf-8"
)
assert
"elif decision == 'deleted':"
in
source
assert
"s.biz_review_status = 'deleted'"
in
source
assert
"s.staging_status = 'deleted'"
in
source
def
test_review_access_code_issues_and_validates_edit_session
():
...
...
Please
register
or
sign in
to post a comment