kim/clean-broken-tgvideos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Separate stage1 scan from stage2 verification

Browse Source
This commit is contained in:
kim
2026-03-30 15:53:53 +08:00
parent 7e74de3e08
commit cec0920f0c
9 changed files with 420 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_cli.py
+160 -19
View File
@@ -15,6 +15,7 @@ def test_help_shows_scan_command() -> None:
assert result.exit_code == 0
assert "scan" in result.stdout
assert "verify-stage2" in result.stdout
def test_scan_command_processes_stage1_results_and_exports_reports(
@@ -74,7 +75,7 @@ def test_scan_command_processes_stage1_results_and_exports_reports(
assert csv_path.exists()
def test_scan_command_runs_stage2_for_pending_files(tmp_path, monkeypatch) -> None:
def test_scan_command_leaves_stage2_pending_by_default(tmp_path, monkeypatch) -> None:
root = tmp_path / "videos"
root.mkdir()
risky = root / "risky.mp4"
@@ -90,19 +91,7 @@ def test_scan_command_runs_stage2_for_pending_files(tmp_path, monkeypatch) -> No
elapsed_ms=1,
)
def fake_run_ffmpeg(path, **kwargs):
return FfmpegExecutionResult(
status="completed",
command=("ffmpeg", str(path)),
exit_code=0,
stdout_text="frame=10",
stderr_text="",
elapsed_ms=2,
reason_code=None,
)
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
monkeypatch.setattr("tgvideo_check.cli.run_ffmpeg_verification", fake_run_ffmpeg)
result = runner.invoke(
app,
@@ -116,14 +105,15 @@ def test_scan_command_runs_stage2_for_pending_files(tmp_path, monkeypatch) -> No
assert result.exit_code == 0
assert "Scanned 1 file(s)" in result.stdout
assert "passed_stage2=1" in result.stdout
assert "pending_stage2=1" in result.stdout
def test_scan_command_reports_discovery_and_stage_progress(tmp_path, monkeypatch) -> None:
def test_verify_stage2_command_runs_pending_stage2_files(tmp_path, monkeypatch) -> None:
root = tmp_path / "videos"
root.mkdir()
risky = root / "risky.mp4"
risky.write_bytes(b"risky")
db_path = tmp_path / "scan_state.db"
def fake_run_ffprobe(path, **kwargs):
return ProbeExecutionResult(
@@ -149,6 +139,52 @@ def test_scan_command_reports_discovery_and_stage_progress(tmp_path, monkeypatch
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
monkeypatch.setattr("tgvideo_check.cli.run_ffmpeg_verification", fake_run_ffmpeg)
scan_result = runner.invoke(
app,
[
"scan",
str(root),
"--db",
str(db_path),
],
)
assert scan_result.exit_code == 0
assert "pending_stage2=1" in scan_result.stdout
verify_result = runner.invoke(
app,
[
"verify-stage2",
"--db",
str(db_path),
],
)
assert verify_result.exit_code == 0
assert "Stage2 source: run_id=1 profile=tg-hdd" in verify_result.stdout
assert "Stage2 progress: 1/1 (100%) passed_stage2=1" in verify_result.stdout
assert "Verified stage2 for run 1. passed_stage2=1" in verify_result.stdout
def test_scan_command_reports_discovery_and_stage1_progress(tmp_path, monkeypatch) -> None:
root = tmp_path / "videos"
root.mkdir()
risky = root / "risky.mp4"
risky.write_bytes(b"risky")
def fake_run_ffprobe(path, **kwargs):
return ProbeExecutionResult(
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
result = runner.invoke(
app,
[
@@ -166,14 +202,16 @@ def test_scan_command_reports_discovery_and_stage_progress(tmp_path, monkeypatch
in result.stdout
)
assert "Stage1 progress: 1/1 (100%) pending_stage2=1" in result.stdout
assert "Stage2 progress: 1/1 (100%) passed_stage2=1" in result.stdout
assert "Stage2 progress:" not in result.stdout
assert "Stage2: no pending files" not in result.stdout
def test_scan_command_reports_stage2_heartbeat_while_waiting(tmp_path, monkeypatch) -> None:
def test_verify_stage2_command_reports_heartbeat_while_waiting(tmp_path, monkeypatch) -> None:
root = tmp_path / "videos"
root.mkdir()
for index in range(60):
(root / f"risky-{index:03d}.mp4").write_bytes(b"risky")
db_path = tmp_path / "scan_state.db"
def fake_run_ffprobe(path, **kwargs):
return ProbeExecutionResult(
@@ -201,13 +239,25 @@ def test_scan_command_reports_stage2_heartbeat_while_waiting(tmp_path, monkeypat
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
monkeypatch.setattr("tgvideo_check.cli.run_ffmpeg_verification", fake_run_ffmpeg)
result = runner.invoke(
scan_result = runner.invoke(
app,
[
"scan",
str(root),
"--db",
str(tmp_path / "scan_state.db"),
str(db_path),
],
)
assert scan_result.exit_code == 0
assert "pending_stage2=60" in scan_result.stdout
result = runner.invoke(
app,
[
"verify-stage2",
"--db",
str(db_path),
],
)
@@ -272,6 +322,97 @@ def test_scan_resume_preserves_reason_code_in_exported_report(tmp_path, monkeypa
assert '"reason_code":"ffprobe_exit_nonzero"' in second_jsonl.read_text(encoding="utf-8")
def test_scan_resume_does_not_reuse_previous_stage2_results(tmp_path, monkeypatch) -> None:
root = tmp_path / "videos"
root.mkdir()
risky = root / "risky.mp4"
risky.write_bytes(b"risky")
db_path = tmp_path / "scan_state.db"
def fake_run_ffprobe(path, **kwargs):
return ProbeExecutionResult(
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
def fake_run_ffmpeg(path, **kwargs):
return FfmpegExecutionResult(
status="completed",
command=("ffmpeg", str(path)),
exit_code=0,
stdout_text="frame=10",
stderr_text="",
elapsed_ms=2,
reason_code=None,
)
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
monkeypatch.setattr("tgvideo_check.cli.run_ffmpeg_verification", fake_run_ffmpeg)
first_result = runner.invoke(
app,
[
"scan",
str(root),
"--db",
str(db_path),
],
)
assert first_result.exit_code == 0
assert "pending_stage2=1" in first_result.stdout
verify_result = runner.invoke(
app,
[
"verify-stage2",
"--db",
str(db_path),
],
)
assert verify_result.exit_code == 0
assert "passed_stage2=1" in verify_result.stdout
def fail_ffmpeg(*args, **kwargs):
raise AssertionError("stage2 should not run during default scan resume")
monkeypatch.setattr("tgvideo_check.cli.run_ffmpeg_verification", fail_ffmpeg)
probe_calls = 0
def second_run_ffprobe(path, **kwargs):
nonlocal probe_calls
probe_calls += 1
return ProbeExecutionResult(
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", second_run_ffprobe)
second_result = runner.invoke(
app,
[
"scan",
str(root),
"--db",
str(db_path),
],
)
assert second_result.exit_code == 0
assert "pending_stage2=1" in second_result.stdout
assert "passed_stage2=1" not in second_result.stdout
assert probe_calls == 1
def test_scan_no_resume_forces_probe_execution(tmp_path, monkeypatch) -> None:
root = tmp_path / "videos"
root.mkdir()
tests/test_integration_resume.py
+2 -6
View File
@@ -32,16 +32,12 @@ def test_scan_reuses_previous_results_without_rerunning_probes(
],
)
assert first_result.exit_code == 0
assert "passed_stage2=1" in first_result.stdout
assert "pending_stage2=1" in first_result.stdout
def fail_ffprobe(*args, **kwargs):
raise AssertionError("run_ffprobe should not be called during resume")
def fail_ffmpeg(*args, **kwargs):
raise AssertionError("run_ffmpeg_verification should not be called during resume")
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fail_ffprobe)
monkeypatch.setattr("tgvideo_check.cli.run_ffmpeg_verification", fail_ffmpeg)
second_result = runner.invoke(
app,
@@ -54,7 +50,7 @@ def test_scan_reuses_previous_results_without_rerunning_probes(
)
assert second_result.exit_code == 0
assert "passed_stage2=1" in second_result.stdout
assert "pending_stage2=1" in second_result.stdout
def _generate_test_video(path: Path, *, duration: int) -> None:
tests/test_integration_truncated_mkv.py
+15 -3
View File
@@ -14,7 +14,7 @@ HAS_FFMPEG = shutil.which("ffmpeg") is not None and shutil.which("ffprobe") is n
@pytest.mark.skipif(not HAS_FFMPEG, reason="ffmpeg/ffprobe not installed")
def test_scan_detects_truncated_mkv_during_stage2(tmp_path: Path) -> None:
def test_verify_stage2_detects_truncated_mkv(tmp_path: Path) -> None:
root = tmp_path / "videos"
root.mkdir()
valid_path = root / "valid.mkv"
@@ -27,7 +27,7 @@ def test_scan_detects_truncated_mkv_during_stage2(tmp_path: Path) -> None:
)
db_path = tmp_path / "scan_state.db"
result = runner.invoke(
scan_result = runner.invoke(
app,
[
"scan",
@@ -37,7 +37,19 @@ def test_scan_detects_truncated_mkv_during_stage2(tmp_path: Path) -> None:
],
)
assert result.exit_code == 0
assert scan_result.exit_code == 0
assert "pending_stage2=2" in scan_result.stdout
verify_result = runner.invoke(
app,
[
"verify-stage2",
"--db",
str(db_path),
],
)
assert verify_result.exit_code == 0
store = ScanStore(db_path)
rows = store.list_files(1)
tests/test_perf_smoke.py
-2
View File
@@ -43,8 +43,6 @@ def _run_scan(root: Path, db_path: Path, *, stage1_workers: int) -> float:
str(db_path),
"--stage1-workers",
str(stage1_workers),
"--stage2-workers",
"1",
],
)
elapsed = time.perf_counter() - start
tests/test_resume_logic.py
+35 -1
View File
@@ -70,6 +70,41 @@ def test_upsert_files_reuses_previous_status_when_identity_matches(tmp_path: Pat
assert resumed.resumed_from_run_id == first_run
def test_upsert_files_can_skip_reusing_previous_stage2_results(tmp_path: Path) -> None:
store = ScanStore(tmp_path / "scan_state.db")
store.initialize()
first_run = store.create_run(profile="tg-hdd", root_paths=(tmp_path,))
file_path = tmp_path / "clip.mp4"
candidate = make_candidate(file_path, order=0, size=12, mtime_ns=999)
[stored] = store.upsert_files(first_run, [candidate])
store.record_stage1_status(
stored.file_id,
stage1_status="pending_stage2",
final_status=None,
reason_code="high_risk_extension",
)
store.record_stage2_status(
stored.file_id,
stage2_status="passed_stage2",
final_status="passed_stage2",
reason_code="passed_stage2",
)
second_run = store.create_run(profile="tg-hdd", root_paths=(tmp_path,))
[resumed] = store.upsert_files(
second_run,
[candidate],
reuse_stage2_results=False,
)
assert resumed.stage1_status == "pending_stage1"
assert resumed.stage2_status is None
assert resumed.final_status is None
assert resumed.reason_code is None
assert resumed.resumed_from_run_id is None
def test_upsert_files_does_not_reuse_changed_file(tmp_path: Path) -> None:
store = ScanStore(tmp_path / "scan_state.db")
store.initialize()
@@ -97,4 +132,3 @@ def test_upsert_files_does_not_reuse_changed_file(tmp_path: Path) -> None:
assert fresh.stage1_status == "pending_stage1"
assert fresh.final_status is None
assert fresh.resumed_from_run_id is None