kim/clean-broken-tgvideos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate scan store to shared entities

Browse Source
This commit is contained in:
kim
2026-03-30 21:48:16 +08:00
parent 6a599ae5d4
commit fc6b99227b
6 changed files with 1122 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_cli.py
+163
View File
@@ -2,6 +2,7 @@ import sqlite3
import time
from pathlib import Path
import orjson
from typer.testing import CliRunner
from tgvideo_check.cli import app
@@ -642,6 +643,168 @@ def test_scan_resume_does_not_reuse_previous_stage2_results(tmp_path, monkeypatc
assert probe_calls == 1
def test_scan_compacts_covered_historical_run_when_parent_root_is_scanned(
tmp_path: Path, monkeypatch
) -> None:
parent = tmp_path / "a"
nested = parent / "b"
sibling = parent / "c"
nested.mkdir(parents=True)
sibling.mkdir()
legacy_video = nested / "legacy.mp4"
fresh_video = sibling / "fresh.mp4"
legacy_video.write_bytes(b"legacy")
fresh_video.write_bytes(b"fresh")
db_path = tmp_path / "scan_state.db"
store = ScanStore(db_path)
store.initialize()
legacy_run = store.create_run(profile="tg-hdd", root_paths=(nested,))
legacy_candidate = FileCandidate(
path=legacy_video,
size=legacy_video.stat().st_size,
mtime_ns=legacy_video.stat().st_mtime_ns,
extension="mp4",
discovery_order=0,
)
[legacy_stored] = store.upsert_files(legacy_run, [legacy_candidate])
store.record_stage1_result(
legacy_stored.file_id,
exit_code=0,
stdout_text='{"format":{"format_name":"mp4","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
store.record_stage1_status(
legacy_stored.file_id,
stage1_status="passed_stage1",
final_status="passed_stage1",
reason_code="passed_stage1",
)
store.update_run_status(legacy_run, status="completed")
probe_calls: list[Path] = []
def fake_ffprobe(path: Path) -> ProbeExecutionResult:
probe_calls.append(path)
return ProbeExecutionResult(
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mp4","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_ffprobe)
result = runner.invoke(
app,
[
"scan",
str(parent),
"--db",
str(db_path),
],
)
assert result.exit_code == 0
with sqlite3.connect(db_path) as connection:
run_count = connection.execute("SELECT COUNT(*) FROM scan_runs").fetchone()[0]
file_count = connection.execute("SELECT COUNT(*) FROM files").fetchone()[0]
stage1_count = connection.execute("SELECT COUNT(*) FROM stage1_results").fetchone()[0]
remaining_root_paths_json = connection.execute(
"SELECT root_paths_json FROM scan_runs"
).fetchone()[0]
legacy_row = connection.execute(
"""
SELECT stage1_status, final_status
FROM files
WHERE path = ?
""",
(str(legacy_video),),
).fetchone()
assert run_count == 1
assert file_count == 2
assert stage1_count == 2
assert orjson.loads(remaining_root_paths_json) == [str(parent)]
assert legacy_row == ("passed_stage1", "passed_stage1")
assert probe_calls == [fresh_video]
def test_scan_does_not_compact_historical_run_when_nested_file_changed(
tmp_path: Path, monkeypatch
) -> None:
parent = tmp_path / "a"
nested = parent / "b"
nested.mkdir(parents=True)
legacy_video = nested / "legacy.mp4"
legacy_video.write_bytes(b"legacy")
db_path = tmp_path / "scan_state.db"
store = ScanStore(db_path)
store.initialize()
legacy_run = store.create_run(profile="tg-hdd", root_paths=(nested,))
legacy_candidate = FileCandidate(
path=legacy_video,
size=legacy_video.stat().st_size,
mtime_ns=legacy_video.stat().st_mtime_ns,
extension="mp4",
discovery_order=0,
)
[legacy_stored] = store.upsert_files(legacy_run, [legacy_candidate])
store.record_stage1_result(
legacy_stored.file_id,
exit_code=0,
stdout_text='{"format":{"format_name":"mp4","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
store.record_stage1_status(
legacy_stored.file_id,
stage1_status="passed_stage1",
final_status="passed_stage1",
reason_code="passed_stage1",
)
store.update_run_status(legacy_run, status="completed")
time.sleep(0.01)
legacy_video.write_bytes(b"legacy-changed")
def fake_ffprobe(path: Path) -> ProbeExecutionResult:
return ProbeExecutionResult(
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mp4","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_ffprobe)
result = runner.invoke(
app,
[
"scan",
str(parent),
"--db",
str(db_path),
],
)
assert result.exit_code == 0
with sqlite3.connect(db_path) as connection:
run_count = connection.execute("SELECT COUNT(*) FROM scan_runs").fetchone()[0]
stage1_count = connection.execute("SELECT COUNT(*) FROM stage1_results").fetchone()[0]
assert run_count == 2
assert stage1_count == 2
def test_compact_db_command_keeps_selected_run_and_removes_duplicate_runs(tmp_path: Path) -> None:
root = tmp_path / "videos"
root.mkdir()
tests/test_resume_logic.py
+260 -10
View File
@@ -16,6 +16,55 @@ def make_candidate(path: Path, *, order: int, size: int = 10, mtime_ns: int = 10
)
def create_legacy_schema(db_path: Path) -> None:
with sqlite3.connect(db_path) as connection:
connection.executescript(
"""
CREATE TABLE scan_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
profile TEXT NOT NULL,
root_paths_json TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'running'
);
CREATE TABLE files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id INTEGER NOT NULL,
path TEXT NOT NULL,
size INTEGER NOT NULL,
mtime_ns INTEGER NOT NULL,
extension TEXT NOT NULL,
discovery_order INTEGER NOT NULL,
stage1_status TEXT NOT NULL,
stage2_status TEXT,
final_status TEXT,
resumed_from_run_id INTEGER,
reason_code TEXT,
UNIQUE (run_id, path)
);
CREATE TABLE stage1_results (
file_id INTEGER PRIMARY KEY,
exit_code INTEGER,
stdout_text TEXT,
stderr_text TEXT,
elapsed_ms INTEGER
);
CREATE TABLE stage2_results (
file_id INTEGER PRIMARY KEY,
mode TEXT,
sample_points_json TEXT,
exit_code INTEGER,
stderr_text TEXT,
reason_code TEXT,
elapsed_ms INTEGER
);
"""
)
def test_initialize_creates_expected_tables(tmp_path: Path) -> None:
store = ScanStore(tmp_path / "scan_state.db")
@@ -29,7 +78,7 @@ def test_initialize_creates_expected_tables(tmp_path: Path) -> None:
)
}
assert {"scan_runs", "files", "stage1_results", "stage2_results"} <= table_names
assert {"scan_runs", "files", "file_entities", "stage1_results", "stage2_results"} <= table_names
def test_upsert_files_persists_candidates_in_discovery_order(tmp_path: Path) -> None:
@@ -69,6 +118,13 @@ def test_upsert_files_reuses_previous_status_when_identity_matches(tmp_path: Pat
assert resumed.final_status == "passed_stage1"
assert resumed.resumed_from_run_id == first_run
with sqlite3.connect(store.db_path) as connection:
stage1_count = connection.execute(
"SELECT COUNT(*) FROM stage1_results"
).fetchone()[0]
assert stage1_count == 0
def test_upsert_files_can_skip_reusing_previous_stage2_results(tmp_path: Path) -> None:
store = ScanStore(tmp_path / "scan_state.db")
@@ -170,12 +226,16 @@ def test_upsert_files_same_run_preserves_existing_status_without_duplication(
"SELECT COUNT(*) FROM files WHERE run_id = ?",
(run_id,),
).fetchone()[0]
entity_count = connection.execute(
"SELECT COUNT(DISTINCT entity_id) FROM files WHERE run_id = ?",
(run_id,),
).fetchone()[0]
stage1_count = connection.execute(
"SELECT COUNT(*) FROM stage1_results WHERE file_id = ?",
(stored.file_id,),
"SELECT COUNT(*) FROM stage1_results",
).fetchone()[0]
assert file_count == 1
assert entity_count == 1
assert stage1_count == 1
@@ -202,6 +262,11 @@ def test_upsert_files_same_run_changed_file_clears_stale_stage_results(
final_status="passed_stage1",
reason_code="passed_stage1",
)
with sqlite3.connect(store.db_path) as connection:
original_entity_id = connection.execute(
"SELECT entity_id FROM files WHERE id = ?",
(stored.file_id,),
).fetchone()[0]
changed = FileCandidate(
path=file_path,
@@ -218,14 +283,199 @@ def test_upsert_files_same_run_changed_file_clears_stale_stage_results(
assert refreshed.reason_code is None
with sqlite3.connect(store.db_path) as connection:
stage1_count = connection.execute(
"SELECT COUNT(*) FROM stage1_results WHERE file_id = ?",
current_entity_id = connection.execute(
"SELECT entity_id FROM files WHERE id = ?",
(stored.file_id,),
).fetchone()[0]
stage2_count = connection.execute(
"SELECT COUNT(*) FROM stage2_results WHERE file_id = ?",
(stored.file_id,),
new_entity_stage1_count = connection.execute(
"SELECT COUNT(*) FROM stage1_results WHERE entity_id = ?",
(current_entity_id,),
).fetchone()[0]
old_entity_stage1_count = connection.execute(
"SELECT COUNT(*) FROM stage1_results WHERE entity_id = ?",
(original_entity_id,),
).fetchone()[0]
new_entity_stage2_count = connection.execute(
"SELECT COUNT(*) FROM stage2_results WHERE entity_id = ?",
(current_entity_id,),
).fetchone()[0]
assert stage1_count == 0
assert stage2_count == 0
assert current_entity_id != original_entity_id
assert new_entity_stage1_count == 0
assert old_entity_stage1_count == 1
assert new_entity_stage2_count == 0
def test_initialize_migrates_legacy_schema_to_entity_storage_without_duplicate_stage_results(
tmp_path: Path,
) -> None:
db_path = tmp_path / "scan_state.db"
create_legacy_schema(db_path)
with sqlite3.connect(db_path) as connection:
connection.execute(
"""
INSERT INTO scan_runs(id, profile, root_paths_json, status)
VALUES (1, 'tg-hdd', '["root"]', 'failed')
"""
)
connection.execute(
"""
INSERT INTO scan_runs(id, profile, root_paths_json, status)
VALUES (2, 'tg-hdd', '["root"]', 'running')
"""
)
connection.execute(
"""
INSERT INTO files(
id, run_id, path, size, mtime_ns, extension, discovery_order,
stage1_status, stage2_status, final_status, resumed_from_run_id, reason_code
)
VALUES
(11, 1, ?, 12, 999, 'mp4', 0, 'passed_stage1', NULL, 'passed_stage1', NULL, 'passed_stage1'),
(12, 2, ?, 12, 999, 'mp4', 0, 'passed_stage1', NULL, 'passed_stage1', 1, 'passed_stage1')
""",
(str(tmp_path / "clip.mp4"), str(tmp_path / "clip.mp4")),
)
connection.execute(
"""
INSERT INTO stage1_results(file_id, exit_code, stdout_text, stderr_text, elapsed_ms)
VALUES
(11, 0, '{"legacy":"first"}', '', 10),
(12, 0, '{"legacy":"latest"}', '', 12)
"""
)
store = ScanStore(db_path)
store.initialize()
with sqlite3.connect(db_path) as connection:
table_names = {
row[0]
for row in connection.execute(
"SELECT name FROM sqlite_master WHERE type = 'table'"
)
}
entity_count = connection.execute(
"SELECT COUNT(*) FROM file_entities"
).fetchone()[0]
file_rows = connection.execute(
"SELECT COUNT(*) FROM files"
).fetchone()[0]
stage1_rows = connection.execute(
"SELECT COUNT(*) FROM stage1_results"
).fetchone()[0]
migrated_stdout = connection.execute(
"SELECT stdout_text FROM stage1_results"
).fetchone()[0]
entity_links = connection.execute(
"SELECT COUNT(DISTINCT entity_id) FROM files"
).fetchone()[0]
assert "file_entities" in table_names
assert entity_count == 1
assert file_rows == 2
assert stage1_rows == 1
assert migrated_stdout == '{"legacy":"latest"}'
assert entity_links == 1
def test_stage1_results_are_shared_across_runs_by_entity(tmp_path: Path) -> None:
store = ScanStore(tmp_path / "scan_state.db")
store.initialize()
first_run = store.create_run(profile="tg-hdd", root_paths=(tmp_path,))
candidate = make_candidate(tmp_path / "clip.mp4", order=0, size=12, mtime_ns=999)
[first] = store.upsert_files(first_run, [candidate])
store.record_stage1_result(
first.file_id,
exit_code=0,
stdout_text='{"shared":"yes"}',
stderr_text="",
elapsed_ms=10,
)
store.record_stage1_status(
first.file_id,
stage1_status="passed_stage1",
final_status="passed_stage1",
reason_code="passed_stage1",
)
second_run = store.create_run(profile="tg-hdd", root_paths=(tmp_path,))
[second] = store.upsert_files(second_run, [candidate])
assert second.stage1_status == "passed_stage1"
assert second.final_status == "passed_stage1"
with sqlite3.connect(store.db_path) as connection:
stage1_rows = connection.execute(
"SELECT COUNT(*) FROM stage1_results"
).fetchone()[0]
distinct_entities = connection.execute(
"SELECT COUNT(DISTINCT entity_id) FROM files"
).fetchone()[0]
assert stage1_rows == 1
assert distinct_entities == 1
def test_compact_duplicate_runs_removes_orphan_entities_and_stage_results(
tmp_path: Path,
) -> None:
store = ScanStore(tmp_path / "scan_state.db")
store.initialize()
first_run = store.create_run(profile="tg-hdd", root_paths=(tmp_path,))
common = make_candidate(tmp_path / "common.mp4", order=0, size=12, mtime_ns=999)
stale_only = make_candidate(tmp_path / "stale.mp4", order=1, size=13, mtime_ns=1000)
first_rows = store.upsert_files(first_run, [common, stale_only])
for item, stdout_text in zip(first_rows, ('{"file":"common"}', '{"file":"stale"}'), strict=True):
store.record_stage1_result(
item.file_id,
exit_code=0,
stdout_text=stdout_text,
stderr_text="",
elapsed_ms=10,
)
store.record_stage1_status(
item.file_id,
stage1_status="passed_stage1",
final_status="passed_stage1",
reason_code="passed_stage1",
)
second_run = store.create_run(profile="tg-hdd", root_paths=(tmp_path,))
[second_common] = store.upsert_files(second_run, [common])
store.record_stage1_status(
second_common.file_id,
stage1_status="passed_stage1",
final_status="passed_stage1",
reason_code="passed_stage1",
)
summary = store.compact_duplicate_runs(second_run, dry_run=False, vacuum=False)
with sqlite3.connect(store.db_path) as connection:
remaining_runs = connection.execute(
"SELECT COUNT(*) FROM scan_runs"
).fetchone()[0]
remaining_files = connection.execute(
"SELECT COUNT(*) FROM files"
).fetchone()[0]
remaining_entities = connection.execute(
"SELECT COUNT(*) FROM file_entities"
).fetchone()[0]
remaining_stage1 = connection.execute(
"SELECT COUNT(*) FROM stage1_results"
).fetchone()[0]
remaining_stdout = {
row[0]
for row in connection.execute("SELECT stdout_text FROM stage1_results")
}
assert summary.deleted_run_count == 1
assert remaining_runs == 1
assert remaining_files == 1
assert remaining_entities == 1
assert remaining_stage1 == 1
assert remaining_stdout == {'{"file":"common"}'}