feat: add txt path import and prune command
This commit is contained in:
@@ -19,11 +19,48 @@ uv run tgvideo-check scan D:\Exports\ChatA D:\Exports\ChatB --profile tg-hdd
|
||||
常用参数:
|
||||
|
||||
- `--db`:指定 SQLite 状态库
|
||||
- `--paths-file`:从 txt 文件读取目录列表,一行一个目录
|
||||
- `--export-jsonl` / `--export-csv`:导出结果报告
|
||||
- `--stage1-workers` / `--stage2-workers`:覆盖 profile 默认 worker 数
|
||||
- `--queue-limit`:限制单阶段批次大小,避免把待处理文件一次性堆进内存
|
||||
- `--full-decode-confirm`:对 stage2 强制使用完整解码确认
|
||||
|
||||
`--paths-file` 规则:
|
||||
|
||||
- 按 `UTF-8` / `UTF-8 BOM` 读取
|
||||
- 一行一个目录
|
||||
- 空行自动跳过
|
||||
- 行内空格、中文、方括号、感叹号等字符按字面值保留
|
||||
- 如果整行外面包了一层单引号或双引号,会自动去掉外层引号
|
||||
|
||||
### 删除已确认损坏文件
|
||||
|
||||
先预览:
|
||||
|
||||
```powershell
|
||||
tgvideo-check prune --db .\scan_state.db
|
||||
```
|
||||
|
||||
默认行为:
|
||||
|
||||
- 默认 `--dry-run`
|
||||
- 默认只处理 `confirmed_broken`
|
||||
- 输出匹配数量、删除数量、总字节数、原因统计
|
||||
|
||||
确认后执行真实删除:
|
||||
|
||||
```powershell
|
||||
tgvideo-check prune --db .\scan_state.db --no-dry-run
|
||||
```
|
||||
|
||||
如果后续你明确要扩大删除范围,可以传:
|
||||
|
||||
```powershell
|
||||
tgvideo-check prune --db .\scan_state.db --delete-status confirmed_broken,suspicious
|
||||
```
|
||||
|
||||
但这不是默认推荐,因为 `suspicious` 仍然代表边界文件,不应默认直接删。
|
||||
|
||||
### 性能 Smoke
|
||||
|
||||
仓库内提供了 `tests/test_perf_smoke.py`,默认跳过,不把环境波动硬编码成普通单测。
|
||||
@@ -20,3 +20,6 @@ dev = [
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/tgvideo_check"]
|
||||
@@ -35,14 +35,19 @@ def root() -> None:
|
||||
|
||||
@app.command()
|
||||
def scan(
|
||||
paths: list[Path] = typer.Argument(
|
||||
...,
|
||||
exists=True,
|
||||
file_okay=False,
|
||||
dir_okay=True,
|
||||
readable=True,
|
||||
paths: list[str] | None = typer.Argument(
|
||||
None,
|
||||
help="One or more root directories to scan.",
|
||||
),
|
||||
paths_file: Path | None = typer.Option(
|
||||
None,
|
||||
"--paths-file",
|
||||
exists=True,
|
||||
file_okay=True,
|
||||
dir_okay=False,
|
||||
readable=True,
|
||||
help="Text file with one directory path per line.",
|
||||
),
|
||||
profile: str = typer.Option("tg-hdd", "--profile"),
|
||||
recursive: bool = typer.Option(True, "--recursive/--no-recursive"),
|
||||
extensions: str = typer.Option(
|
||||
@@ -64,7 +69,7 @@ def scan(
|
||||
|
||||
config = resolve_config(
|
||||
ScanConfig(
|
||||
roots=tuple(paths),
|
||||
roots=_resolve_scan_roots(paths or [], paths_file),
|
||||
profile=profile,
|
||||
recursive=recursive,
|
||||
extensions=_parse_extensions(extensions),
|
||||
@@ -88,6 +93,79 @@ def scan(
|
||||
typer.echo(f"Scanned {len(rows)} file(s). {summary_text}".strip())
|
||||
|
||||
|
||||
@app.command()
|
||||
def prune(
|
||||
db_path: Path = typer.Option(Path("scan_state.db"), "--db"),
|
||||
run_id: int | None = typer.Option(None, "--run-id"),
|
||||
delete_status: str = typer.Option(
|
||||
"confirmed_broken",
|
||||
"--delete-status",
|
||||
help="Comma-separated final statuses eligible for deletion.",
|
||||
),
|
||||
dry_run: bool = typer.Option(
|
||||
True,
|
||||
"--dry-run/--no-dry-run",
|
||||
help="Preview deletion targets without removing files.",
|
||||
),
|
||||
) -> None:
|
||||
"""Delete selected files from the latest scan result and print stats."""
|
||||
|
||||
store = ScanStore(db_path)
|
||||
store.initialize()
|
||||
selected_run_id = run_id or store.get_latest_run_id()
|
||||
if selected_run_id is None:
|
||||
raise typer.BadParameter(f"No scan runs found in database: {db_path}")
|
||||
|
||||
statuses = {
|
||||
item.strip()
|
||||
for item in delete_status.split(",")
|
||||
if item.strip()
|
||||
}
|
||||
if not statuses:
|
||||
raise typer.BadParameter("At least one delete status must be provided.")
|
||||
|
||||
files = store.list_files(selected_run_id)
|
||||
matched_rows = _build_report_rows(files, {})
|
||||
matched_rows = [
|
||||
row for row in matched_rows if row.final_status in statuses
|
||||
]
|
||||
matched_size = sum(row.size for row in matched_rows)
|
||||
deleted = 0
|
||||
deleted_size = 0
|
||||
missing = 0
|
||||
failed = 0
|
||||
|
||||
if not dry_run:
|
||||
for row in matched_rows:
|
||||
try:
|
||||
if not row.path.exists():
|
||||
missing += 1
|
||||
continue
|
||||
if not row.path.is_file():
|
||||
failed += 1
|
||||
continue
|
||||
row.path.unlink()
|
||||
deleted += 1
|
||||
deleted_size += row.size
|
||||
except OSError:
|
||||
failed += 1
|
||||
|
||||
summary_label = "Dry-run" if dry_run else "Prune"
|
||||
typer.echo(
|
||||
f"{summary_label}: run_id={selected_run_id} matched={len(matched_rows)} "
|
||||
f"deleted={deleted} missing={missing} failed={failed} bytes={matched_size}"
|
||||
)
|
||||
if not dry_run:
|
||||
typer.echo(f"Deleted bytes={deleted_size}")
|
||||
|
||||
reason_counts = Counter(row.reason_code for row in matched_rows)
|
||||
if reason_counts:
|
||||
reason_text = " ".join(
|
||||
f"{reason}={reason_counts[reason]}" for reason in sorted(reason_counts)
|
||||
)
|
||||
typer.echo(f"Reasons: {reason_text}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
app()
|
||||
|
||||
@@ -360,3 +438,67 @@ def _parse_extensions(raw_extensions: str) -> tuple[str, ...]:
|
||||
|
||||
def _dump_json(values: tuple[float, ...]) -> str:
|
||||
return orjson.dumps(values).decode("utf-8")
|
||||
|
||||
|
||||
def _resolve_scan_roots(
|
||||
positional_paths: list[str],
|
||||
paths_file: Path | None,
|
||||
) -> tuple[Path, ...]:
|
||||
roots: list[Path] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for raw_path in positional_paths:
|
||||
_append_scan_root(
|
||||
Path(raw_path),
|
||||
roots=roots,
|
||||
seen=seen,
|
||||
source="command argument",
|
||||
)
|
||||
|
||||
if paths_file is not None:
|
||||
for line_number, raw_line in enumerate(
|
||||
paths_file.read_text(encoding="utf-8-sig").splitlines(),
|
||||
start=1,
|
||||
):
|
||||
candidate = raw_line.rstrip("\r\n")
|
||||
if not candidate.strip():
|
||||
continue
|
||||
_append_scan_root(
|
||||
Path(_strip_optional_wrapping_quotes(candidate)),
|
||||
roots=roots,
|
||||
seen=seen,
|
||||
source=f"{paths_file}:{line_number}",
|
||||
)
|
||||
|
||||
if not roots:
|
||||
raise typer.BadParameter(
|
||||
"Provide at least one directory path or use --paths-file.",
|
||||
)
|
||||
|
||||
return tuple(roots)
|
||||
|
||||
|
||||
def _append_scan_root(
|
||||
path: Path,
|
||||
*,
|
||||
roots: list[Path],
|
||||
seen: set[str],
|
||||
source: str,
|
||||
) -> None:
|
||||
if not path.exists():
|
||||
raise typer.BadParameter(f"Path from {source} does not exist: {path}")
|
||||
if not path.is_dir():
|
||||
raise typer.BadParameter(f"Path from {source} is not a directory: {path}")
|
||||
|
||||
normalized = str(path.resolve())
|
||||
if normalized in seen:
|
||||
return
|
||||
|
||||
seen.add(normalized)
|
||||
roots.append(path)
|
||||
|
||||
|
||||
def _strip_optional_wrapping_quotes(value: str) -> str:
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
|
||||
return value[1:-1]
|
||||
return value
|
||||
@@ -90,6 +90,34 @@ class ScanStore:
|
||||
)
|
||||
return int(cursor.lastrowid)
|
||||
|
||||
def get_latest_run_id(self, *, prefer_completed: bool = True) -> int | None:
|
||||
with self._connect() as connection:
|
||||
if prefer_completed:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM scan_runs
|
||||
WHERE status = 'completed'
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
return int(row["id"])
|
||||
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM scan_runs
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
|
||||
if row is None:
|
||||
return None
|
||||
return int(row["id"])
|
||||
|
||||
def upsert_files(
|
||||
self,
|
||||
run_id: int,
|
||||
@@ -234,3 +234,218 @@ def test_scan_no_resume_forces_probe_execution(tmp_path, monkeypatch) -> None:
|
||||
|
||||
assert second_result.exit_code == 0
|
||||
assert probe_calls == 1
|
||||
|
||||
|
||||
def test_scan_can_load_directories_from_paths_file(tmp_path, monkeypatch) -> None:
|
||||
root_a = tmp_path / "Dir With Space"
|
||||
root_b = tmp_path / "[Special]!目录"
|
||||
root_a.mkdir()
|
||||
root_b.mkdir()
|
||||
(root_a / "a.avi").write_bytes(b"a")
|
||||
(root_b / "b.avi").write_bytes(b"b")
|
||||
|
||||
paths_file = tmp_path / "paths.txt"
|
||||
paths_file.write_text(
|
||||
f'{root_a}\n"{root_b}"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
probe_calls: list[str] = []
|
||||
|
||||
def fake_run_ffprobe(path, **kwargs):
|
||||
probe_calls.append(path.name)
|
||||
return ProbeExecutionResult(
|
||||
status="completed",
|
||||
command=("ffprobe", str(path)),
|
||||
exit_code=0,
|
||||
stdout_text='{"format":{"format_name":"avi","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
|
||||
stderr_text="",
|
||||
elapsed_ms=1,
|
||||
)
|
||||
|
||||
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"scan",
|
||||
"--paths-file",
|
||||
str(paths_file),
|
||||
"--db",
|
||||
str(tmp_path / "scan_state.db"),
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "Scanned 2 file(s)" in result.stdout
|
||||
assert sorted(probe_calls) == ["a.avi", "b.avi"]
|
||||
|
||||
|
||||
def test_scan_merges_positional_paths_and_paths_file(tmp_path, monkeypatch) -> None:
|
||||
direct_root = tmp_path / "direct"
|
||||
listed_root = tmp_path / "listed"
|
||||
direct_root.mkdir()
|
||||
listed_root.mkdir()
|
||||
(direct_root / "direct.avi").write_bytes(b"a")
|
||||
(listed_root / "listed.avi").write_bytes(b"b")
|
||||
|
||||
paths_file = tmp_path / "paths.txt"
|
||||
paths_file.write_text(f"{listed_root}\n", encoding="utf-8")
|
||||
|
||||
probe_calls: list[str] = []
|
||||
|
||||
def fake_run_ffprobe(path, **kwargs):
|
||||
probe_calls.append(path.name)
|
||||
return ProbeExecutionResult(
|
||||
status="completed",
|
||||
command=("ffprobe", str(path)),
|
||||
exit_code=0,
|
||||
stdout_text='{"format":{"format_name":"avi","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
|
||||
stderr_text="",
|
||||
elapsed_ms=1,
|
||||
)
|
||||
|
||||
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"scan",
|
||||
str(direct_root),
|
||||
"--paths-file",
|
||||
str(paths_file),
|
||||
"--db",
|
||||
str(tmp_path / "scan_state.db"),
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "Scanned 2 file(s)" in result.stdout
|
||||
assert sorted(probe_calls) == ["direct.avi", "listed.avi"]
|
||||
|
||||
|
||||
def test_prune_dry_run_reports_confirmed_broken_without_deleting(tmp_path, monkeypatch) -> None:
|
||||
root = tmp_path / "videos"
|
||||
root.mkdir()
|
||||
broken = root / "broken.mp4"
|
||||
passed = root / "passed.avi"
|
||||
broken.write_bytes(b"broken")
|
||||
passed.write_bytes(b"passed")
|
||||
db_path = tmp_path / "scan_state.db"
|
||||
|
||||
def fake_run_ffprobe(path, **kwargs):
|
||||
if path == broken:
|
||||
return ProbeExecutionResult(
|
||||
status="completed",
|
||||
command=("ffprobe", str(path)),
|
||||
exit_code=1,
|
||||
stdout_text="",
|
||||
stderr_text="moov atom not found",
|
||||
elapsed_ms=1,
|
||||
)
|
||||
return ProbeExecutionResult(
|
||||
status="completed",
|
||||
command=("ffprobe", str(path)),
|
||||
exit_code=0,
|
||||
stdout_text='{"format":{"format_name":"avi","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
|
||||
stderr_text="",
|
||||
elapsed_ms=1,
|
||||
)
|
||||
|
||||
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
|
||||
|
||||
scan_result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"scan",
|
||||
str(root),
|
||||
"--db",
|
||||
str(db_path),
|
||||
],
|
||||
)
|
||||
assert scan_result.exit_code == 0
|
||||
|
||||
prune_result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"prune",
|
||||
"--db",
|
||||
str(db_path),
|
||||
],
|
||||
)
|
||||
|
||||
assert prune_result.exit_code == 0
|
||||
assert "Dry-run" in prune_result.stdout
|
||||
assert "matched=1" in prune_result.stdout
|
||||
assert "deleted=0" in prune_result.stdout
|
||||
assert broken.exists()
|
||||
assert passed.exists()
|
||||
|
||||
|
||||
def test_prune_no_dry_run_deletes_confirmed_broken_files(tmp_path, monkeypatch) -> None:
|
||||
root = tmp_path / "videos"
|
||||
root.mkdir()
|
||||
broken = root / "broken.mp4"
|
||||
suspicious = root / "suspicious.mp4"
|
||||
broken.write_bytes(b"broken")
|
||||
suspicious.write_bytes(b"suspicious")
|
||||
db_path = tmp_path / "scan_state.db"
|
||||
|
||||
def fake_run_ffprobe(path, **kwargs):
|
||||
if path == broken:
|
||||
return ProbeExecutionResult(
|
||||
status="completed",
|
||||
command=("ffprobe", str(path)),
|
||||
exit_code=1,
|
||||
stdout_text="",
|
||||
stderr_text="moov atom not found",
|
||||
elapsed_ms=1,
|
||||
)
|
||||
return ProbeExecutionResult(
|
||||
status="completed",
|
||||
command=("ffprobe", str(path)),
|
||||
exit_code=0,
|
||||
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
|
||||
stderr_text="",
|
||||
elapsed_ms=1,
|
||||
)
|
||||
|
||||
def fake_run_ffmpeg(path, **kwargs):
|
||||
return FfmpegExecutionResult(
|
||||
status="completed",
|
||||
command=("ffmpeg", str(path)),
|
||||
exit_code=0,
|
||||
stdout_text="frame=10",
|
||||
stderr_text="",
|
||||
elapsed_ms=2,
|
||||
reason_code=None,
|
||||
)
|
||||
|
||||
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
|
||||
monkeypatch.setattr("tgvideo_check.cli.run_ffmpeg_verification", fake_run_ffmpeg)
|
||||
|
||||
scan_result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"scan",
|
||||
str(root),
|
||||
"--db",
|
||||
str(db_path),
|
||||
],
|
||||
)
|
||||
assert scan_result.exit_code == 0
|
||||
|
||||
prune_result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"prune",
|
||||
"--db",
|
||||
str(db_path),
|
||||
"--no-dry-run",
|
||||
],
|
||||
)
|
||||
|
||||
assert prune_result.exit_code == 0
|
||||
assert "deleted=1" in prune_result.stdout
|
||||
assert not broken.exists()
|
||||
assert suspicious.exists()
|
||||
Generated
Reference in New Issue
Block a user