Skip to content

沈秋雨 / import_hk_songs

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 59e59a59 59e59a596ddbb01f0958e833e2188e5f9fd9e7cd by 沈秋雨
Options

fix(reader): 排除音眼自制版权ID的录音数据 

- 在聚合查询SQL中加入排除音眼自制版权ID的条件
- 更新相关的SQL查询语句，过滤掉指定版权ID的记录
- 修改测试用例，确保音眼自制版权ID的录音被正确排除
- .gitignore新增docs目录的忽略规则
1 parent 7a8fed26
Inline Side-by-side
Showing 5 changed files with 46 additions and 17 deletions
......@@ -21,3 +21,5 @@ output/
*.csv
*.sql
*.xlsx
docs/
......
......@@ -3,6 +3,9 @@ import pymysql
from .config import PLATFORMS, TARGET_TABLE_NAME
YINYAN_SELF_MADE_COPYRIGHT_ID = '1871126451002978305'
_HK_SONGS_QUERY = f"""
SELECT id, name, lyricist, composer, audio_url, lyrics_url,
cover_url, singer, issue_time, source_song_id, song_time
......@@ -28,7 +31,7 @@ WHERE deleted = '0'
ORDER BY id
"""
_PLATFORM_QUERY = """
_PLATFORM_QUERY = f"""
SELECT
sar.song_id AS source_song_id,
mr.id AS record_id,
......@@ -42,11 +45,13 @@ SELECT
mr.record_name
FROM hk_song_and_record sar
JOIN hk_music_record mr ON mr.id = sar.record_id
WHERE sar.song_id IN ({placeholders})
WHERE sar.song_id IN ({{placeholders}})
AND mr.platform IN ('1','2','4')
AND mr.platform_unique_key IS NOT NULL
AND mr.platform_unique_key != ''
AND mr.deleted = 0
AND COALESCE(NULLIF(TRIM(mr.copyright_id), ''), '')
<> '{YINYAN_SELF_MADE_COPYRIGHT_ID}'
ORDER BY sar.song_id,
COALESCE(sar.is_main_version, 0) DESC,
COALESCE(mr.is_high, 0) DESC,
......@@ -55,7 +60,7 @@ ORDER BY sar.song_id,
mr.id ASC
"""
_PLATFORM_BY_RECORD_IDS_QUERY = """
_PLATFORM_BY_RECORD_IDS_QUERY = f"""
SELECT
sar.song_id AS source_song_id,
mr.id AS record_id,
......@@ -69,18 +74,20 @@ SELECT
mr.record_name
FROM hk_song_and_record sar
JOIN hk_music_record mr ON mr.id = sar.record_id
WHERE mr.id IN ({placeholders})
WHERE mr.id IN ({{placeholders}})
AND mr.platform IN ('1','2','4')
AND mr.platform_unique_key IS NOT NULL
AND mr.platform_unique_key != ''
AND mr.deleted = 0
AND COALESCE(NULLIF(TRIM(mr.copyright_id), ''), '')
<> '{YINYAN_SELF_MADE_COPYRIGHT_ID}'
"""
# 同 _PLATFORM_QUERY，但不做 per-platform 去重，保留同平台全部录音供 singer fallback 遍历
_ALL_PLATFORM_RECORDS_QUERY = _PLATFORM_QUERY
# records2 只接收具备必要录音元数据的关联关系，pub_time 允许为空。
_RECORDS2_RELATIONS_QUERY = """
_RECORDS2_RELATIONS_QUERY = f"""
SELECT
sar.song_id AS source_song_id,
mr.id AS record_id,
......@@ -94,11 +101,13 @@ SELECT
mr.record_name
FROM hk_song_and_record sar
JOIN hk_music_record mr ON mr.id = sar.record_id
WHERE sar.song_id IN ({placeholders})
WHERE sar.song_id IN ({{placeholders}})
AND mr.platform IN ('1','2','4')
AND mr.platform_unique_key IS NOT NULL
AND TRIM(mr.platform_unique_key) != ''
AND mr.deleted = 0
AND COALESCE(NULLIF(TRIM(mr.copyright_id), ''), '')
<> '{YINYAN_SELF_MADE_COPYRIGHT_ID}'
AND mr.record_name IS NOT NULL AND TRIM(mr.record_name) != ''
AND mr.duration IS NOT NULL AND mr.duration > 0
AND mr.singer_name IS NOT NULL AND TRIM(mr.singer_name) != ''
......
......@@ -94,7 +94,10 @@ OSS_CONFIG = {
# ==================== 聚合查询 SQL ====================
AGGREGATE_SQL = """
YINYAN_SELF_MADE_COPYRIGHT_ID = '1871126451002978305'
AGGREGATE_SQL = f"""
SELECT
sp.id AS source_id,
COALESCE(NULLIF(TRIM(sp.song_name), ''),
......@@ -168,19 +171,25 @@ SELECT
sr_cnt.record_count
FROM hk_song_platform sp
INNER JOIN (
SELECT song_id,
SELECT sar.song_id,
COALESCE(
MIN(CASE WHEN is_main_version = 1 THEN record_id END),
MIN(record_id)
MIN(CASE WHEN sar.is_main_version = 1 THEN sar.record_id END),
MIN(sar.record_id)
) AS record_id
FROM hk_song_and_record
GROUP BY song_id
FROM hk_song_and_record sar
JOIN hk_music_record candidate_r ON candidate_r.id = sar.record_id
WHERE COALESCE(NULLIF(TRIM(candidate_r.copyright_id), ''), '')
<> '{YINYAN_SELF_MADE_COPYRIGHT_ID}'
GROUP BY sar.song_id
) sr
ON sr.song_id = sp.id
LEFT JOIN (
SELECT song_id, COUNT(record_id) AS record_count
FROM hk_song_and_record
GROUP BY song_id
SELECT sar.song_id, COUNT(sar.record_id) AS record_count
FROM hk_song_and_record sar
JOIN hk_music_record candidate_r ON candidate_r.id = sar.record_id
WHERE COALESCE(NULLIF(TRIM(candidate_r.copyright_id), ''), '')
<> '{YINYAN_SELF_MADE_COPYRIGHT_ID}'
GROUP BY sar.song_id
) sr_cnt
ON sr_cnt.song_id = sp.id
LEFT JOIN hk_music_record r
......
......@@ -61,14 +61,16 @@ from lyric_dedup.checker import CandidateMatch
class TestAggregateSqlRules:
"""测试源库聚合查询中的业务过滤规则"""
def test_excludes_liancheng_xiaorui_compositions_but_not_yinyan_self_made_compositions(self):
def test_excludes_liancheng_xiaorui_and_yinyan_self_made_records(self):
assert '1871475560046465025' in AGGREGATE_SQL
assert '连城小睿音乐工作室' in AGGREGATE_SQL
assert '1871126451002978305' in AGGREGATE_SQL
where_clause = AGGREGATE_SQL.split('WHERE', 1)[1]
assert 'sp.copyright_id' in where_clause
assert 'sp.copyright_name' in where_clause
assert '音眼自制' not in where_clause
assert 'candidate_r.copyright_id' in where_clause
assert '音眼自制' not in AGGREGATE_SQL
# ====================================================================
......
......@@ -58,6 +58,9 @@ def test_fetch_platform_records_keeps_one_record_per_song_and_platform():
result = fetch_platform_records(conn, [10])
sql = conn.cursor.return_value.execute.call_args.args[0]
assert '1871126451002978305' in sql
assert 'mr.copyright_id' in sql
assert len(result) == 3
assert {row['record_id'] for row in result} == {100, 101, 102}
......@@ -83,6 +86,8 @@ def test_fetch_platform_records_by_record_ids_queries_exact_records_once():
cur = conn.cursor.return_value
sql, params = cur.execute.call_args[0]
assert 'WHERE mr.id IN (%s)' in sql
assert '1871126451002978305' in sql
assert 'mr.copyright_id' in sql
assert params == [200]
assert result[0]['record_id'] == 200
assert result[0]['platform'] == '2'
......@@ -132,6 +137,8 @@ def test_fetch_all_song_record_relations_keeps_all_records_for_a_song():
assert 'mr.platform_index_url IS NOT NULL' in sql
assert 'mr.storage_url IS NOT NULL' in sql
assert 'mr.platform_play_url IS NOT NULL' in sql
assert '1871126451002978305' in sql
assert 'mr.copyright_id' in sql
assert 'mr.pub_time IS NOT NULL' not in sql
assert params == [10]
assert [(row['record_id'], row['platform']) for row in result] == [
......