feat(import): 增加入库冲突检测和详细冲突报告
Showing
3 changed files
with
137 additions
and
10 deletions
| ... | @@ -1518,7 +1518,29 @@ | ... | @@ -1518,7 +1518,29 @@ |
| 1518 | await Promise.all([reloadSummary(), loadGroups()]); | 1518 | await Promise.all([reloadSummary(), loadGroups()]); |
| 1519 | renderAll(); | 1519 | renderAll(); |
| 1520 | } catch (err) { | 1520 | } catch (err) { |
| 1521 | alert(`入库失败:${err.message}`); | 1521 | if (err.status === 409 && err.payload?.code === 'import_conflict') { |
| 1522 | const conflicts = err.payload.conflicts || []; | ||
| 1523 | const details = conflicts.slice(0, 20).map((row, index) => { | ||
| 1524 | const types = (row.conflict_types || []).map(type => | ||
| 1525 | type === 'primary_key' ? '主键 ID 冲突' : '来源键冲突' | ||
| 1526 | ).join(' + ') || '唯一键冲突'; | ||
| 1527 | const targetIds = [row.id_conflict_target_id, row.source_conflict_target_id] | ||
| 1528 | .filter(value => value !== null && value !== undefined) | ||
| 1529 | .filter((value, i, values) => values.indexOf(value) === i) | ||
| 1530 | .join('/'); | ||
| 1531 | return `${index + 1}. ${row.name || '(无歌名)'} | source=${row.source_table_name || '-'}:${row.source_song_id || '-'} | staging_id=${row.staging_id} | 待入库ID=${row.incoming_id || '-'} | ${types} | 正式表ID=${targetIds || '-'}`; | ||
| 1532 | }); | ||
| 1533 | if (conflicts.length > 20) details.push(`……另有 ${conflicts.length - 20} 条未展开`); | ||
| 1534 | alert([ | ||
| 1535 | `入库失败:${err.message}`, | ||
| 1536 | '', | ||
| 1537 | ...details, | ||
| 1538 | '', | ||
| 1539 | err.payload.advice || '请将上述冲突数据反馈给开发人员处理。' | ||
| 1540 | ].join('\n')); | ||
| 1541 | } else { | ||
| 1542 | alert(`入库失败:${err.message}\n\n请保留当前页面信息并反馈给开发人员。已有审核结果不会丢失。`); | ||
| 1543 | } | ||
| 1522 | } finally { | 1544 | } finally { |
| 1523 | els.importReviewedBtn.disabled = false; | 1545 | els.importReviewedBtn.disabled = false; |
| 1524 | els.importReviewedBtn.textContent = '入库审核通过'; | 1546 | els.importReviewedBtn.textContent = '入库审核通过'; | ... | ... |
| ... | @@ -93,6 +93,20 @@ class ReviewAuthError(Exception): | ... | @@ -93,6 +93,20 @@ class ReviewAuthError(Exception): |
| 93 | """The caller does not have a valid review editing session.""" | 93 | """The caller does not have a valid review editing session.""" |
| 94 | 94 | ||
| 95 | 95 | ||
| 96 | class ImportConflictError(Exception): | ||
| 97 | """Approved staging rows conflict with records already in the target table.""" | ||
| 98 | |||
| 99 | def __init__( | ||
| 100 | self, | ||
| 101 | message: str, | ||
| 102 | conflicts: list[dict[str, object]] | None = None, | ||
| 103 | database_error: str = "", | ||
| 104 | ) -> None: | ||
| 105 | super().__init__(message) | ||
| 106 | self.conflicts = conflicts or [] | ||
| 107 | self.database_error = database_error | ||
| 108 | |||
| 109 | |||
| 96 | def _review_login( | 110 | def _review_login( |
| 97 | reviewer: str, | 111 | reviewer: str, |
| 98 | access_code: str = "", | 112 | access_code: str = "", |
| ... | @@ -744,15 +758,32 @@ def _import_approved_staging(source_ids: list[str] | None = None) -> dict[str, o | ... | @@ -744,15 +758,32 @@ def _import_approved_staging(source_ids: list[str] | None = None) -> dict[str, o |
| 744 | staging_ids = [row["staging_id"] for row in approved_rows] | 758 | staging_ids = [row["staging_id"] for row in approved_rows] |
| 745 | approved_source_ids = [str(row["source_song_id"]) for row in approved_rows] | 759 | approved_source_ids = [str(row["source_song_id"]) for row in approved_rows] |
| 746 | staging_placeholders = ",".join(["%s"] * len(staging_ids)) | 760 | staging_placeholders = ",".join(["%s"] * len(staging_ids)) |
| 747 | cursor.execute( | 761 | conflicts = _find_import_conflicts(cursor, staging_ids) |
| 748 | f""" | 762 | if conflicts: |
| 749 | INSERT INTO {TARGET_TABLE_NAME} ({columns}) | 763 | conn.rollback() |
| 750 | SELECT {select_columns} | 764 | raise ImportConflictError( |
| 751 | FROM {TARGET_TABLE_NAME_TMP} s | 765 | f"发现 {len(conflicts)} 条待入库数据与正式表已有记录冲突,本次未写入任何数据", |
| 752 | WHERE s.staging_id IN ({staging_placeholders}) | 766 | conflicts, |
| 753 | """, | 767 | ) |
| 754 | staging_ids, | 768 | try: |
| 755 | ) | 769 | cursor.execute( |
| 770 | f""" | ||
| 771 | INSERT INTO {TARGET_TABLE_NAME} ({columns}) | ||
| 772 | SELECT {select_columns} | ||
| 773 | FROM {TARGET_TABLE_NAME_TMP} s | ||
| 774 | WHERE s.staging_id IN ({staging_placeholders}) | ||
| 775 | """, | ||
| 776 | staging_ids, | ||
| 777 | ) | ||
| 778 | except pymysql.err.IntegrityError as exc: | ||
| 779 | # 预检与 INSERT 之间仍可能有其他程序写入,回滚后再查一次以返回具体行。 | ||
| 780 | conn.rollback() | ||
| 781 | conflicts = _find_import_conflicts(cursor, staging_ids) | ||
| 782 | raise ImportConflictError( | ||
| 783 | "入库时发生唯一键或主键冲突,本次未写入任何数据", | ||
| 784 | conflicts, | ||
| 785 | str(exc), | ||
| 786 | ) from exc | ||
| 756 | inserted_count = cursor.rowcount | 787 | inserted_count = cursor.rowcount |
| 757 | source_placeholders = ",".join(["%s"] * len(approved_source_ids)) | 788 | source_placeholders = ",".join(["%s"] * len(approved_source_ids)) |
| 758 | cursor.execute( | 789 | cursor.execute( |
| ... | @@ -789,6 +820,40 @@ def _import_approved_staging(source_ids: list[str] | None = None) -> dict[str, o | ... | @@ -789,6 +820,40 @@ def _import_approved_staging(source_ids: list[str] | None = None) -> dict[str, o |
| 789 | } | 820 | } |
| 790 | 821 | ||
| 791 | 822 | ||
| 823 | def _find_import_conflicts(cursor, staging_ids: list[object]) -> list[dict[str, object]]: | ||
| 824 | """Return target-table key conflicts for the selected staging rows.""" | ||
| 825 | if not staging_ids: | ||
| 826 | return [] | ||
| 827 | placeholders = ",".join(["%s"] * len(staging_ids)) | ||
| 828 | cursor.execute( | ||
| 829 | f""" | ||
| 830 | SELECT s.staging_id, s.source_song_id, s.source_table_name, s.name, | ||
| 831 | s.id AS incoming_id, | ||
| 832 | by_id.id AS id_conflict_target_id, | ||
| 833 | by_source.id AS source_conflict_target_id | ||
| 834 | FROM {TARGET_TABLE_NAME_TMP} s | ||
| 835 | LEFT JOIN {TARGET_TABLE_NAME} by_id | ||
| 836 | ON by_id.id = s.id | ||
| 837 | LEFT JOIN {TARGET_TABLE_NAME} by_source | ||
| 838 | ON by_source.source_table_name = s.source_table_name | ||
| 839 | AND by_source.source_song_id = s.source_song_id | ||
| 840 | WHERE s.staging_id IN ({placeholders}) | ||
| 841 | AND (by_id.id IS NOT NULL OR by_source.id IS NOT NULL) | ||
| 842 | ORDER BY s.staging_id | ||
| 843 | """, | ||
| 844 | staging_ids, | ||
| 845 | ) | ||
| 846 | conflicts = [] | ||
| 847 | for row in cursor.fetchall(): | ||
| 848 | conflict_types = [] | ||
| 849 | if row.get("id_conflict_target_id") is not None: | ||
| 850 | conflict_types.append("primary_key") | ||
| 851 | if row.get("source_conflict_target_id") is not None: | ||
| 852 | conflict_types.append("source_key") | ||
| 853 | conflicts.append({**row, "conflict_types": conflict_types}) | ||
| 854 | return conflicts | ||
| 855 | |||
| 856 | |||
| 792 | def _report_runs() -> list[dict[str, str]]: | 857 | def _report_runs() -> list[dict[str, str]]: |
| 793 | summaries = { | 858 | summaries = { |
| 794 | path.name.replace("l2_topk_benchmark_summary_", "").removesuffix(".csv"): path | 859 | path.name.replace("l2_topk_benchmark_summary_", "").removesuffix(".csv"): path |
| ... | @@ -1250,6 +1315,18 @@ class Handler(BaseHTTPRequestHandler): | ... | @@ -1250,6 +1315,18 @@ class Handler(BaseHTTPRequestHandler): |
| 1250 | _json_response(self, {"review_csv": str(review_csv), **result}) | 1315 | _json_response(self, {"review_csv": str(review_csv), **result}) |
| 1251 | except ReviewAuthError as exc: | 1316 | except ReviewAuthError as exc: |
| 1252 | _error(self, str(exc), status=401) | 1317 | _error(self, str(exc), status=401) |
| 1318 | except ImportConflictError as exc: | ||
| 1319 | _json_response( | ||
| 1320 | self, | ||
| 1321 | { | ||
| 1322 | "error": str(exc), | ||
| 1323 | "code": "import_conflict", | ||
| 1324 | "conflicts": exc.conflicts, | ||
| 1325 | "database_error": exc.database_error, | ||
| 1326 | "advice": "请核对以下冲突数据,并将信息反馈给开发人员处理后再次提交。已有审核结果不会丢失。", | ||
| 1327 | }, | ||
| 1328 | status=409, | ||
| 1329 | ) | ||
| 1253 | except Exception as exc: # noqa: BLE001 - local diagnostic API | 1330 | except Exception as exc: # noqa: BLE001 - local diagnostic API |
| 1254 | _error(self, str(exc), status=400) | 1331 | _error(self, str(exc), status=400) |
| 1255 | 1332 | ... | ... |
| ... | @@ -186,6 +186,7 @@ class ImportCursor(FakeCursor): | ... | @@ -186,6 +186,7 @@ class ImportCursor(FakeCursor): |
| 186 | super().__init__(update_rowcount=1, snapshot={}) | 186 | super().__init__(update_rowcount=1, snapshot={}) |
| 187 | self.result_sets = [ | 187 | self.result_sets = [ |
| 188 | [{"staging_id": 42, "source_song_id": "song-1"}], | 188 | [{"staging_id": 42, "source_song_id": "song-1"}], |
| 189 | [], | ||
| 189 | [{"source_song_id": "song-1", "target_id": 9001}], | 190 | [{"source_song_id": "song-1", "target_id": 9001}], |
| 190 | ] | 191 | ] |
| 191 | 192 | ||
| ... | @@ -213,6 +214,33 @@ def test_import_locks_only_unclaimed_staged_rows(): | ... | @@ -213,6 +214,33 @@ def test_import_locks_only_unclaimed_staged_rows(): |
| 213 | assert conn.committed | 214 | assert conn.committed |
| 214 | 215 | ||
| 215 | 216 | ||
| 217 | def test_import_conflict_rolls_back_and_exposes_the_staging_row(): | ||
| 218 | cursor = ImportCursor() | ||
| 219 | cursor.result_sets = [ | ||
| 220 | [{"staging_id": 42, "source_song_id": "song-1"}], | ||
| 221 | [{ | ||
| 222 | "staging_id": 42, | ||
| 223 | "source_song_id": "song-1", | ||
| 224 | "source_table_name": "hk_song_platform", | ||
| 225 | "name": "same song", | ||
| 226 | "incoming_id": 9002, | ||
| 227 | "id_conflict_target_id": None, | ||
| 228 | "source_conflict_target_id": 9001, | ||
| 229 | }], | ||
| 230 | ] | ||
| 231 | conn = FakeConnection(cursor) | ||
| 232 | |||
| 233 | with patch.object(dashboard, "_target_conn", return_value=conn): | ||
| 234 | with pytest.raises(dashboard.ImportConflictError) as raised: | ||
| 235 | dashboard._import_approved_staging() | ||
| 236 | |||
| 237 | assert raised.value.conflicts[0]["staging_id"] == 42 | ||
| 238 | assert raised.value.conflicts[0]["conflict_types"] == ["source_key"] | ||
| 239 | assert conn.rolled_back | ||
| 240 | assert not conn.committed | ||
| 241 | assert not any(sql.lstrip().startswith("INSERT") for sql, _params in cursor.executions) | ||
| 242 | |||
| 243 | |||
| 216 | def test_browsing_a_list_item_can_auto_claim_without_reloading_groups(): | 244 | def test_browsing_a_list_item_can_auto_claim_without_reloading_groups(): |
| 217 | html = Path("l2_review_dashboard.html").read_text(encoding="utf-8") | 245 | html = Path("l2_review_dashboard.html").read_text(encoding="utf-8") |
| 218 | start = html.index("for (const btn of els.queryList.querySelectorAll('.query-item'))") | 246 | start = html.index("for (const btn of els.queryList.querySelectorAll('.query-item'))") | ... | ... |
-
Please register or sign in to post a comment