kim/clean-broken-tgvideos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove suffix-based stage2 escalation by default

Browse Source
This commit is contained in:
kim
2026-03-30 16:30:00 +08:00
parent cec0920f0c
commit fef2e57cf4
6 changed files with 16 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3 -3
View File
@@ -157,7 +157,7 @@ mp4,mov,m4v,mkv,webm,avi,flv,f4v,ts,m2ts,mts,mpg,mpeg,mpe,mpv,m2v,wmv,asf,3gp,3g
说明：
- `tg-hdd` / `tg-fast` 仍保持 Telegram 常见容器的保守高风险复核范围，避免因为格式表扩大导致 stage2 backlog 暴涨
- `tg-hdd` / `tg-fast` 默认不再因为文件后缀直接升级到第二轮，只有 `ffprobe` 已经发现异常信号时才进入 `pending_stage2`
- 如果你希望对全部已支持格式都走更激进的二次校验，使用 `--profile deep-verify`
### `--paths-file` 规则
@@ -194,7 +194,7 @@ tgvideo-check verify-stage2 --db ".\small-test.db" --export-jsonl ".\small-test-
重点看：
- `confirmed_broken` 是否命中了你故意造坏的样本
- `pending_stage2` 数量是否符合你对高风险格式/可疑样本的预期
- `pending_stage2` 数量是否符合你对可疑样本的预期
- `passed_stage2` 里是否混进了你明知已坏的文件
- 是否误伤了你确认正常的视频
@@ -229,7 +229,7 @@ tgvideo-check verify-stage2 --db ".\small-test.db" --export-jsonl ".\small-test-
| `missing_video_stream` | 没有检测到视频流 | 文件不是有效视频 |
| `missing_format_name` | 无法识别容器格式 | 容器头部或格式识别失败 |
| `invalid_duration` | 时长字段异常，因此升级到第二轮验证 | 容器信息不完整或异常 |
| `high_risk_extension` | 高风险扩展名，因此升级到第二轮验证 | 根据 profile 规则升级复核 |
| `high_risk_extension` | 根据 profile 规则直接升级到第二轮验证 | 主要出现在显式使用 `deep-verify` 这类激进配置时 |
| `passed_stage1` | 容器级快速校验通过 | 第一轮结束即可放行 |
| `passed_stage2` | 第二轮采样或确认校验通过 | 第二轮也没发现问题 |
| `command_error` | 外部命令执行异常 | 环境问题，不应直接删文件 |
src/tgvideo_check/profiles.py
+2 -5
View File
@@ -5,9 +5,6 @@ from dataclasses import dataclass, replace
from tgvideo_check.config import DEFAULT_VIDEO_EXTENSIONS, ScanConfig
TG_HIGH_RISK_EXTENSIONS: tuple[str, ...] = ("mp4", "mov", "m4v", "mkv", "webm")
@dataclass(frozen=True)
class Profile:
name: str
@@ -23,14 +20,14 @@ PROFILES: dict[str, Profile] = {
stage1_workers=1,
stage2_workers=1,
sample_duration=3,
high_risk_extensions=TG_HIGH_RISK_EXTENSIONS,
high_risk_extensions=(),
),
"tg-fast": Profile(
name="tg-fast",
stage1_workers=2,
stage2_workers=1,
sample_duration=3,
high_risk_extensions=TG_HIGH_RISK_EXTENSIONS,
high_risk_extensions=(),
),
"deep-verify": Profile(
name="deep-verify",
tests/test_cli.py
+6 -6
View File
@@ -86,7 +86,7 @@ def test_scan_command_leaves_stage2_pending_by_default(tmp_path, monkeypatch) ->
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"N/A"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
@@ -120,7 +120,7 @@ def test_verify_stage2_command_runs_pending_stage2_files(tmp_path, monkeypatch)
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"N/A"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
@@ -178,7 +178,7 @@ def test_scan_command_reports_discovery_and_stage1_progress(tmp_path, monkeypatc
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"N/A"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
@@ -218,7 +218,7 @@ def test_verify_stage2_command_reports_heartbeat_while_waiting(tmp_path, monkeyp
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"N/A"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
@@ -334,7 +334,7 @@ def test_scan_resume_does_not_reuse_previous_stage2_results(tmp_path, monkeypatc
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"N/A"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
@@ -390,7 +390,7 @@ def test_scan_resume_does_not_reuse_previous_stage2_results(tmp_path, monkeypatc
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"N/A"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
tests/test_integration_resume.py
+2 -2
View File
@@ -32,7 +32,7 @@ def test_scan_reuses_previous_results_without_rerunning_probes(
],
)
assert first_result.exit_code == 0
assert "pending_stage2=1" in first_result.stdout
assert "passed_stage1=1" in first_result.stdout
def fail_ffprobe(*args, **kwargs):
raise AssertionError("run_ffprobe should not be called during resume")
@@ -50,7 +50,7 @@ def test_scan_reuses_previous_results_without_rerunning_probes(
)
assert second_result.exit_code == 0
assert "pending_stage2=1" in second_result.stdout
assert "passed_stage1=1" in second_result.stdout
def _generate_test_video(path: Path, *, duration: int) -> None:
tests/test_integration_truncated_mkv.py
+2
View File
@@ -32,6 +32,8 @@ def test_verify_stage2_detects_truncated_mkv(tmp_path: Path) -> None:
[
"scan",
str(root),
"--profile",
"deep-verify",
"--db",
str(db_path),
],
tests/test_profiles.py
+1 -1
View File
@@ -9,7 +9,7 @@ def test_tg_hdd_profile_uses_conservative_defaults() -> None:
assert profile.stage1_workers == 1
assert profile.stage2_workers == 1
assert profile.sample_duration == 3
assert profile.high_risk_extensions == ("mp4", "mov", "m4v", "mkv", "webm")
assert profile.high_risk_extensions == ()
def test_default_extensions_cover_broad_common_video_formats() -> None: