kim/clean-broken-tgvideos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix resume DB growth and add compaction command

Browse Source
This commit is contained in:
kim
2026-03-30 19:55:29 +08:00
parent 834577dafc
commit 6a599ae5d4
5 changed files with 586 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_cli.py
+131
View File
@@ -1,3 +1,4 @@
import sqlite3
import time
from pathlib import Path
@@ -490,6 +491,66 @@ def test_scan_resume_preserves_reason_code_in_exported_report(tmp_path, monkeypa
assert '"reason_code":"ffprobe_exit_nonzero"' in second_jsonl.read_text(encoding="utf-8")
def test_scan_resume_reuses_incomplete_run_without_duplication(tmp_path: Path, monkeypatch) -> None:
root = tmp_path / "videos"
root.mkdir()
video = root / "video.avi"
video.write_bytes(b"video")
db_path = tmp_path / "scan_state.db"
store = ScanStore(db_path)
store.initialize()
run_id = store.create_run(profile="tg-hdd", root_paths=(root,))
candidate = FileCandidate(
path=video,
size=video.stat().st_size,
mtime_ns=video.stat().st_mtime_ns,
extension="avi",
discovery_order=0,
)
[stored] = store.upsert_files(run_id, [candidate])
store.record_stage1_result(
stored.file_id,
exit_code=0,
stdout_text='{"format":{"format_name":"avi","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
store.record_stage1_status(
stored.file_id,
stage1_status="passed_stage1",
final_status="passed_stage1",
reason_code="passed_stage1",
)
def fail_ffprobe(*args, **kwargs):
raise AssertionError("resume within same run should not call ffprobe")
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fail_ffprobe)
result = runner.invoke(
app,
[
"scan",
str(root),
"--db",
str(db_path),
],
)
assert result.exit_code == 0
assert "passed_stage1=1" in result.stdout
with sqlite3.connect(db_path) as connection:
run_count = connection.execute("SELECT COUNT(*) FROM scan_runs").fetchone()[0]
file_count = connection.execute("SELECT COUNT(*) FROM files").fetchone()[0]
stage1_count = connection.execute("SELECT COUNT(*) FROM stage1_results").fetchone()[0]
assert run_count == 1
assert file_count == 1
assert stage1_count == 1
def test_scan_resume_does_not_reuse_previous_stage2_results(tmp_path, monkeypatch) -> None:
root = tmp_path / "videos"
root.mkdir()
@@ -581,6 +642,76 @@ def test_scan_resume_does_not_reuse_previous_stage2_results(tmp_path, monkeypatc
assert probe_calls == 1
def test_compact_db_command_keeps_selected_run_and_removes_duplicate_runs(tmp_path: Path) -> None:
root = tmp_path / "videos"
root.mkdir()
video = root / "video.mp4"
video.write_bytes(b"video")
db_path = tmp_path / "scan_state.db"
store = ScanStore(db_path)
store.initialize()
candidate = FileCandidate(
path=video,
size=video.stat().st_size,
mtime_ns=video.stat().st_mtime_ns,
extension="mp4",
discovery_order=0,
)
first_run = store.create_run(profile="tg-hdd", root_paths=(root,))
[first_stored] = store.upsert_files(first_run, [candidate])
store.record_stage1_result(
first_stored.file_id,
exit_code=0,
stdout_text='{"format":{"format_name":"mp4","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
store.record_stage1_status(
first_stored.file_id,
stage1_status="passed_stage1",
final_status="passed_stage1",
reason_code="passed_stage1",
)
store.update_run_status(first_run, status="failed")
second_run = store.create_run(profile="tg-hdd", root_paths=(root,))
[second_stored] = store.upsert_files(second_run, [candidate])
store.update_run_status(second_run, status="running")
before_size = db_path.stat().st_size
result = runner.invoke(
app,
[
"compact-db",
"--db",
str(db_path),
"--run-id",
str(second_run),
"--no-dry-run",
],
)
assert result.exit_code == 0
assert f"keep_run_id={second_run}" in result.stdout
assert "deleted_runs=1" in result.stdout
with sqlite3.connect(db_path) as connection:
run_count = connection.execute("SELECT COUNT(*) FROM scan_runs").fetchone()[0]
file_count = connection.execute("SELECT COUNT(*) FROM files").fetchone()[0]
stage1_count = connection.execute("SELECT COUNT(*) FROM stage1_results").fetchone()[0]
remaining_run_id = connection.execute("SELECT id FROM scan_runs").fetchone()[0]
assert second_stored.file_id != first_stored.file_id
assert run_count == 1
assert file_count == 1
assert stage1_count == 1
assert remaining_run_id == second_run
assert db_path.stat().st_size <= before_size
def test_scan_no_resume_forces_probe_execution(tmp_path, monkeypatch) -> None:
root = tmp_path / "videos"
root.mkdir()
tests/test_resume_logic.py
+97
View File
@@ -132,3 +132,100 @@ def test_upsert_files_does_not_reuse_changed_file(tmp_path: Path) -> None:
assert fresh.stage1_status == "pending_stage1"
assert fresh.final_status is None
assert fresh.resumed_from_run_id is None
def test_upsert_files_same_run_preserves_existing_status_without_duplication(
tmp_path: Path,
) -> None:
store = ScanStore(tmp_path / "scan_state.db")
store.initialize()
run_id = store.create_run(profile="tg-hdd", root_paths=(tmp_path,))
file_path = tmp_path / "clip.mp4"
candidate = make_candidate(file_path, order=0, size=12, mtime_ns=999)
[stored] = store.upsert_files(run_id, [candidate])
store.record_stage1_result(
stored.file_id,
exit_code=0,
stdout_text='{"format":{"format_name":"mp4","duration":"2.0"}}',
stderr_text="",
elapsed_ms=10,
)
store.record_stage1_status(
stored.file_id,
stage1_status="passed_stage1",
final_status="passed_stage1",
reason_code="passed_stage1",
)
[resumed] = store.upsert_files(run_id, [candidate])
assert resumed.file_id == stored.file_id
assert resumed.stage1_status == "passed_stage1"
assert resumed.final_status == "passed_stage1"
assert resumed.resumed_from_run_id is None
with sqlite3.connect(store.db_path) as connection:
file_count = connection.execute(
"SELECT COUNT(*) FROM files WHERE run_id = ?",
(run_id,),
).fetchone()[0]
stage1_count = connection.execute(
"SELECT COUNT(*) FROM stage1_results WHERE file_id = ?",
(stored.file_id,),
).fetchone()[0]
assert file_count == 1
assert stage1_count == 1
def test_upsert_files_same_run_changed_file_clears_stale_stage_results(
tmp_path: Path,
) -> None:
store = ScanStore(tmp_path / "scan_state.db")
store.initialize()
run_id = store.create_run(profile="tg-hdd", root_paths=(tmp_path,))
file_path = tmp_path / "clip.mp4"
original = make_candidate(file_path, order=0, size=12, mtime_ns=999)
[stored] = store.upsert_files(run_id, [original])
store.record_stage1_result(
stored.file_id,
exit_code=0,
stdout_text='{"format":{"format_name":"mp4","duration":"2.0"}}',
stderr_text="",
elapsed_ms=10,
)
store.record_stage1_status(
stored.file_id,
stage1_status="passed_stage1",
final_status="passed_stage1",
reason_code="passed_stage1",
)
changed = FileCandidate(
path=file_path,
size=13,
mtime_ns=1000,
extension="mp4",
discovery_order=0,
)
[refreshed] = store.upsert_files(run_id, [changed])
assert refreshed.file_id == stored.file_id
assert refreshed.stage1_status == "pending_stage1"
assert refreshed.final_status is None
assert refreshed.reason_code is None
with sqlite3.connect(store.db_path) as connection:
stage1_count = connection.execute(
"SELECT COUNT(*) FROM stage1_results WHERE file_id = ?",
(stored.file_id,),
).fetchone()[0]
stage2_count = connection.execute(
"SELECT COUNT(*) FROM stage2_results WHERE file_id = ?",
(stored.file_id,),
).fetchone()[0]
assert stage1_count == 0
assert stage2_count == 0