feat: add txt path import and prune command

This commit is contained in:
kim
2026-03-30 10:15:49 +08:00
parent 3b2d3a5df4
commit 79370ed542
30 changed files with 432 additions and 7 deletions
+37
View File
@@ -19,11 +19,48 @@ uv run tgvideo-check scan D:\Exports\ChatA D:\Exports\ChatB --profile tg-hdd
常用参数:
- `--db`:指定 SQLite 状态库
- `--paths-file`:从 txt 文件读取目录列表,一行一个目录
- `--export-jsonl` / `--export-csv`:导出结果报告
- `--stage1-workers` / `--stage2-workers`:覆盖 profile 默认 worker 数
- `--queue-limit`:限制单阶段批次大小,避免把待处理文件一次性堆进内存
- `--full-decode-confirm`:对 stage2 强制使用完整解码确认
`--paths-file` 规则:
- 按 `UTF-8` / `UTF-8 BOM` 读取
- 一行一个目录
- 空行自动跳过
- 行内空格、中文、方括号、感叹号等字符按字面值保留
- 如果整行外面包了一层单引号或双引号,会自动去掉外层引号
### 删除已确认损坏文件
先预览:
```powershell
tgvideo-check prune --db .\scan_state.db
```
默认行为:
- 默认 `--dry-run`
- 默认只处理 `confirmed_broken`
- 输出匹配数量、删除数量、总字节数、原因统计
确认后执行真实删除:
```powershell
tgvideo-check prune --db .\scan_state.db --no-dry-run
```
如果后续你明确要扩大删除范围,可以传:
```powershell
tgvideo-check prune --db .\scan_state.db --delete-status confirmed_broken,suspicious
```
但这不是默认推荐,因为 `suspicious` 仍然代表边界文件,不应默认直接删。
### 性能 Smoke
仓库内提供了 `tests/test_perf_smoke.py`,默认跳过,不把环境波动硬编码成普通单测。
@@ -20,3 +20,6 @@ dev = [
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/tgvideo_check"]
@@ -35,14 +35,19 @@ def root() -> None:
@app.command()
def scan(
paths: list[Path] = typer.Argument(
...,
exists=True,
file_okay=False,
dir_okay=True,
readable=True,
paths: list[str] | None = typer.Argument(
None,
help="One or more root directories to scan.",
),
paths_file: Path | None = typer.Option(
None,
"--paths-file",
exists=True,
file_okay=True,
dir_okay=False,
readable=True,
help="Text file with one directory path per line.",
),
profile: str = typer.Option("tg-hdd", "--profile"),
recursive: bool = typer.Option(True, "--recursive/--no-recursive"),
extensions: str = typer.Option(
@@ -64,7 +69,7 @@ def scan(
config = resolve_config(
ScanConfig(
roots=tuple(paths),
roots=_resolve_scan_roots(paths or [], paths_file),
profile=profile,
recursive=recursive,
extensions=_parse_extensions(extensions),
@@ -88,6 +93,79 @@ def scan(
typer.echo(f"Scanned {len(rows)} file(s). {summary_text}".strip())
@app.command()
def prune(
db_path: Path = typer.Option(Path("scan_state.db"), "--db"),
run_id: int | None = typer.Option(None, "--run-id"),
delete_status: str = typer.Option(
"confirmed_broken",
"--delete-status",
help="Comma-separated final statuses eligible for deletion.",
),
dry_run: bool = typer.Option(
True,
"--dry-run/--no-dry-run",
help="Preview deletion targets without removing files.",
),
) -> None:
"""Delete selected files from the latest scan result and print stats."""
store = ScanStore(db_path)
store.initialize()
selected_run_id = run_id or store.get_latest_run_id()
if selected_run_id is None:
raise typer.BadParameter(f"No scan runs found in database: {db_path}")
statuses = {
item.strip()
for item in delete_status.split(",")
if item.strip()
}
if not statuses:
raise typer.BadParameter("At least one delete status must be provided.")
files = store.list_files(selected_run_id)
matched_rows = _build_report_rows(files, {})
matched_rows = [
row for row in matched_rows if row.final_status in statuses
]
matched_size = sum(row.size for row in matched_rows)
deleted = 0
deleted_size = 0
missing = 0
failed = 0
if not dry_run:
for row in matched_rows:
try:
if not row.path.exists():
missing += 1
continue
if not row.path.is_file():
failed += 1
continue
row.path.unlink()
deleted += 1
deleted_size += row.size
except OSError:
failed += 1
summary_label = "Dry-run" if dry_run else "Prune"
typer.echo(
f"{summary_label}: run_id={selected_run_id} matched={len(matched_rows)} "
f"deleted={deleted} missing={missing} failed={failed} bytes={matched_size}"
)
if not dry_run:
typer.echo(f"Deleted bytes={deleted_size}")
reason_counts = Counter(row.reason_code for row in matched_rows)
if reason_counts:
reason_text = " ".join(
f"{reason}={reason_counts[reason]}" for reason in sorted(reason_counts)
)
typer.echo(f"Reasons: {reason_text}")
def main() -> None:
app()
@@ -360,3 +438,67 @@ def _parse_extensions(raw_extensions: str) -> tuple[str, ...]:
def _dump_json(values: tuple[float, ...]) -> str:
return orjson.dumps(values).decode("utf-8")
def _resolve_scan_roots(
positional_paths: list[str],
paths_file: Path | None,
) -> tuple[Path, ...]:
roots: list[Path] = []
seen: set[str] = set()
for raw_path in positional_paths:
_append_scan_root(
Path(raw_path),
roots=roots,
seen=seen,
source="command argument",
)
if paths_file is not None:
for line_number, raw_line in enumerate(
paths_file.read_text(encoding="utf-8-sig").splitlines(),
start=1,
):
candidate = raw_line.rstrip("\r\n")
if not candidate.strip():
continue
_append_scan_root(
Path(_strip_optional_wrapping_quotes(candidate)),
roots=roots,
seen=seen,
source=f"{paths_file}:{line_number}",
)
if not roots:
raise typer.BadParameter(
"Provide at least one directory path or use --paths-file.",
)
return tuple(roots)
def _append_scan_root(
path: Path,
*,
roots: list[Path],
seen: set[str],
source: str,
) -> None:
if not path.exists():
raise typer.BadParameter(f"Path from {source} does not exist: {path}")
if not path.is_dir():
raise typer.BadParameter(f"Path from {source} is not a directory: {path}")
normalized = str(path.resolve())
if normalized in seen:
return
seen.add(normalized)
roots.append(path)
def _strip_optional_wrapping_quotes(value: str) -> str:
if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
return value[1:-1]
return value
@@ -90,6 +90,34 @@ class ScanStore:
)
return int(cursor.lastrowid)
def get_latest_run_id(self, *, prefer_completed: bool = True) -> int | None:
with self._connect() as connection:
if prefer_completed:
row = connection.execute(
"""
SELECT id
FROM scan_runs
WHERE status = 'completed'
ORDER BY id DESC
LIMIT 1
"""
).fetchone()
if row is not None:
return int(row["id"])
row = connection.execute(
"""
SELECT id
FROM scan_runs
ORDER BY id DESC
LIMIT 1
"""
).fetchone()
if row is None:
return None
return int(row["id"])
def upsert_files(
self,
run_id: int,
@@ -234,3 +234,218 @@ def test_scan_no_resume_forces_probe_execution(tmp_path, monkeypatch) -> None:
assert second_result.exit_code == 0
assert probe_calls == 1
def test_scan_can_load_directories_from_paths_file(tmp_path, monkeypatch) -> None:
root_a = tmp_path / "Dir With Space"
root_b = tmp_path / "[Special]!目录"
root_a.mkdir()
root_b.mkdir()
(root_a / "a.avi").write_bytes(b"a")
(root_b / "b.avi").write_bytes(b"b")
paths_file = tmp_path / "paths.txt"
paths_file.write_text(
f'{root_a}\n"{root_b}"\n',
encoding="utf-8",
)
probe_calls: list[str] = []
def fake_run_ffprobe(path, **kwargs):
probe_calls.append(path.name)
return ProbeExecutionResult(
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"avi","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
result = runner.invoke(
app,
[
"scan",
"--paths-file",
str(paths_file),
"--db",
str(tmp_path / "scan_state.db"),
],
)
assert result.exit_code == 0
assert "Scanned 2 file(s)" in result.stdout
assert sorted(probe_calls) == ["a.avi", "b.avi"]
def test_scan_merges_positional_paths_and_paths_file(tmp_path, monkeypatch) -> None:
direct_root = tmp_path / "direct"
listed_root = tmp_path / "listed"
direct_root.mkdir()
listed_root.mkdir()
(direct_root / "direct.avi").write_bytes(b"a")
(listed_root / "listed.avi").write_bytes(b"b")
paths_file = tmp_path / "paths.txt"
paths_file.write_text(f"{listed_root}\n", encoding="utf-8")
probe_calls: list[str] = []
def fake_run_ffprobe(path, **kwargs):
probe_calls.append(path.name)
return ProbeExecutionResult(
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"avi","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
result = runner.invoke(
app,
[
"scan",
str(direct_root),
"--paths-file",
str(paths_file),
"--db",
str(tmp_path / "scan_state.db"),
],
)
assert result.exit_code == 0
assert "Scanned 2 file(s)" in result.stdout
assert sorted(probe_calls) == ["direct.avi", "listed.avi"]
def test_prune_dry_run_reports_confirmed_broken_without_deleting(tmp_path, monkeypatch) -> None:
root = tmp_path / "videos"
root.mkdir()
broken = root / "broken.mp4"
passed = root / "passed.avi"
broken.write_bytes(b"broken")
passed.write_bytes(b"passed")
db_path = tmp_path / "scan_state.db"
def fake_run_ffprobe(path, **kwargs):
if path == broken:
return ProbeExecutionResult(
status="completed",
command=("ffprobe", str(path)),
exit_code=1,
stdout_text="",
stderr_text="moov atom not found",
elapsed_ms=1,
)
return ProbeExecutionResult(
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"avi","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
scan_result = runner.invoke(
app,
[
"scan",
str(root),
"--db",
str(db_path),
],
)
assert scan_result.exit_code == 0
prune_result = runner.invoke(
app,
[
"prune",
"--db",
str(db_path),
],
)
assert prune_result.exit_code == 0
assert "Dry-run" in prune_result.stdout
assert "matched=1" in prune_result.stdout
assert "deleted=0" in prune_result.stdout
assert broken.exists()
assert passed.exists()
def test_prune_no_dry_run_deletes_confirmed_broken_files(tmp_path, monkeypatch) -> None:
root = tmp_path / "videos"
root.mkdir()
broken = root / "broken.mp4"
suspicious = root / "suspicious.mp4"
broken.write_bytes(b"broken")
suspicious.write_bytes(b"suspicious")
db_path = tmp_path / "scan_state.db"
def fake_run_ffprobe(path, **kwargs):
if path == broken:
return ProbeExecutionResult(
status="completed",
command=("ffprobe", str(path)),
exit_code=1,
stdout_text="",
stderr_text="moov atom not found",
elapsed_ms=1,
)
return ProbeExecutionResult(
status="completed",
command=("ffprobe", str(path)),
exit_code=0,
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
stderr_text="",
elapsed_ms=1,
)
def fake_run_ffmpeg(path, **kwargs):
return FfmpegExecutionResult(
status="completed",
command=("ffmpeg", str(path)),
exit_code=0,
stdout_text="frame=10",
stderr_text="",
elapsed_ms=2,
reason_code=None,
)
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
monkeypatch.setattr("tgvideo_check.cli.run_ffmpeg_verification", fake_run_ffmpeg)
scan_result = runner.invoke(
app,
[
"scan",
str(root),
"--db",
str(db_path),
],
)
assert scan_result.exit_code == 0
prune_result = runner.invoke(
app,
[
"prune",
"--db",
str(db_path),
"--no-dry-run",
],
)
assert prune_result.exit_code == 0
assert "deleted=1" in prune_result.stdout
assert not broken.exists()
assert suspicious.exists()
View File