feat: add txt path import and prune command
This commit is contained in:
@@ -19,11 +19,48 @@ uv run tgvideo-check scan D:\Exports\ChatA D:\Exports\ChatB --profile tg-hdd
|
|||||||
常用参数:
|
常用参数:
|
||||||
|
|
||||||
- `--db`:指定 SQLite 状态库
|
- `--db`:指定 SQLite 状态库
|
||||||
|
- `--paths-file`:从 txt 文件读取目录列表,一行一个目录
|
||||||
- `--export-jsonl` / `--export-csv`:导出结果报告
|
- `--export-jsonl` / `--export-csv`:导出结果报告
|
||||||
- `--stage1-workers` / `--stage2-workers`:覆盖 profile 默认 worker 数
|
- `--stage1-workers` / `--stage2-workers`:覆盖 profile 默认 worker 数
|
||||||
- `--queue-limit`:限制单阶段批次大小,避免把待处理文件一次性堆进内存
|
- `--queue-limit`:限制单阶段批次大小,避免把待处理文件一次性堆进内存
|
||||||
- `--full-decode-confirm`:对 stage2 强制使用完整解码确认
|
- `--full-decode-confirm`:对 stage2 强制使用完整解码确认
|
||||||
|
|
||||||
|
`--paths-file` 规则:
|
||||||
|
|
||||||
|
- 按 `UTF-8` / `UTF-8 BOM` 读取
|
||||||
|
- 一行一个目录
|
||||||
|
- 空行自动跳过
|
||||||
|
- 行内空格、中文、方括号、感叹号等字符按字面值保留
|
||||||
|
- 如果整行外面包了一层单引号或双引号,会自动去掉外层引号
|
||||||
|
|
||||||
|
### 删除已确认损坏文件
|
||||||
|
|
||||||
|
先预览:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
tgvideo-check prune --db .\scan_state.db
|
||||||
|
```
|
||||||
|
|
||||||
|
默认行为:
|
||||||
|
|
||||||
|
- 默认 `--dry-run`
|
||||||
|
- 默认只处理 `confirmed_broken`
|
||||||
|
- 输出匹配数量、删除数量、总字节数、原因统计
|
||||||
|
|
||||||
|
确认后执行真实删除:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
tgvideo-check prune --db .\scan_state.db --no-dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
如果后续你明确要扩大删除范围,可以传:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
tgvideo-check prune --db .\scan_state.db --delete-status confirmed_broken,suspicious
|
||||||
|
```
|
||||||
|
|
||||||
|
但这不是默认推荐,因为 `suspicious` 仍然代表边界文件,不应默认直接删。
|
||||||
|
|
||||||
### 性能 Smoke
|
### 性能 Smoke
|
||||||
|
|
||||||
仓库内提供了 `tests/test_perf_smoke.py`,默认跳过,不把环境波动硬编码成普通单测。
|
仓库内提供了 `tests/test_perf_smoke.py`,默认跳过,不把环境波动硬编码成普通单测。
|
||||||
@@ -20,3 +20,6 @@ dev = [
|
|||||||
[build-system]
|
[build-system]
|
||||||
requires = ["hatchling"]
|
requires = ["hatchling"]
|
||||||
build-backend = "hatchling.build"
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[tool.hatch.build.targets.wheel]
|
||||||
|
packages = ["src/tgvideo_check"]
|
||||||
@@ -35,14 +35,19 @@ def root() -> None:
|
|||||||
|
|
||||||
@app.command()
|
@app.command()
|
||||||
def scan(
|
def scan(
|
||||||
paths: list[Path] = typer.Argument(
|
paths: list[str] | None = typer.Argument(
|
||||||
...,
|
None,
|
||||||
exists=True,
|
|
||||||
file_okay=False,
|
|
||||||
dir_okay=True,
|
|
||||||
readable=True,
|
|
||||||
help="One or more root directories to scan.",
|
help="One or more root directories to scan.",
|
||||||
),
|
),
|
||||||
|
paths_file: Path | None = typer.Option(
|
||||||
|
None,
|
||||||
|
"--paths-file",
|
||||||
|
exists=True,
|
||||||
|
file_okay=True,
|
||||||
|
dir_okay=False,
|
||||||
|
readable=True,
|
||||||
|
help="Text file with one directory path per line.",
|
||||||
|
),
|
||||||
profile: str = typer.Option("tg-hdd", "--profile"),
|
profile: str = typer.Option("tg-hdd", "--profile"),
|
||||||
recursive: bool = typer.Option(True, "--recursive/--no-recursive"),
|
recursive: bool = typer.Option(True, "--recursive/--no-recursive"),
|
||||||
extensions: str = typer.Option(
|
extensions: str = typer.Option(
|
||||||
@@ -64,7 +69,7 @@ def scan(
|
|||||||
|
|
||||||
config = resolve_config(
|
config = resolve_config(
|
||||||
ScanConfig(
|
ScanConfig(
|
||||||
roots=tuple(paths),
|
roots=_resolve_scan_roots(paths or [], paths_file),
|
||||||
profile=profile,
|
profile=profile,
|
||||||
recursive=recursive,
|
recursive=recursive,
|
||||||
extensions=_parse_extensions(extensions),
|
extensions=_parse_extensions(extensions),
|
||||||
@@ -88,6 +93,79 @@ def scan(
|
|||||||
typer.echo(f"Scanned {len(rows)} file(s). {summary_text}".strip())
|
typer.echo(f"Scanned {len(rows)} file(s). {summary_text}".strip())
|
||||||
|
|
||||||
|
|
||||||
|
@app.command()
|
||||||
|
def prune(
|
||||||
|
db_path: Path = typer.Option(Path("scan_state.db"), "--db"),
|
||||||
|
run_id: int | None = typer.Option(None, "--run-id"),
|
||||||
|
delete_status: str = typer.Option(
|
||||||
|
"confirmed_broken",
|
||||||
|
"--delete-status",
|
||||||
|
help="Comma-separated final statuses eligible for deletion.",
|
||||||
|
),
|
||||||
|
dry_run: bool = typer.Option(
|
||||||
|
True,
|
||||||
|
"--dry-run/--no-dry-run",
|
||||||
|
help="Preview deletion targets without removing files.",
|
||||||
|
),
|
||||||
|
) -> None:
|
||||||
|
"""Delete selected files from the latest scan result and print stats."""
|
||||||
|
|
||||||
|
store = ScanStore(db_path)
|
||||||
|
store.initialize()
|
||||||
|
selected_run_id = run_id or store.get_latest_run_id()
|
||||||
|
if selected_run_id is None:
|
||||||
|
raise typer.BadParameter(f"No scan runs found in database: {db_path}")
|
||||||
|
|
||||||
|
statuses = {
|
||||||
|
item.strip()
|
||||||
|
for item in delete_status.split(",")
|
||||||
|
if item.strip()
|
||||||
|
}
|
||||||
|
if not statuses:
|
||||||
|
raise typer.BadParameter("At least one delete status must be provided.")
|
||||||
|
|
||||||
|
files = store.list_files(selected_run_id)
|
||||||
|
matched_rows = _build_report_rows(files, {})
|
||||||
|
matched_rows = [
|
||||||
|
row for row in matched_rows if row.final_status in statuses
|
||||||
|
]
|
||||||
|
matched_size = sum(row.size for row in matched_rows)
|
||||||
|
deleted = 0
|
||||||
|
deleted_size = 0
|
||||||
|
missing = 0
|
||||||
|
failed = 0
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
for row in matched_rows:
|
||||||
|
try:
|
||||||
|
if not row.path.exists():
|
||||||
|
missing += 1
|
||||||
|
continue
|
||||||
|
if not row.path.is_file():
|
||||||
|
failed += 1
|
||||||
|
continue
|
||||||
|
row.path.unlink()
|
||||||
|
deleted += 1
|
||||||
|
deleted_size += row.size
|
||||||
|
except OSError:
|
||||||
|
failed += 1
|
||||||
|
|
||||||
|
summary_label = "Dry-run" if dry_run else "Prune"
|
||||||
|
typer.echo(
|
||||||
|
f"{summary_label}: run_id={selected_run_id} matched={len(matched_rows)} "
|
||||||
|
f"deleted={deleted} missing={missing} failed={failed} bytes={matched_size}"
|
||||||
|
)
|
||||||
|
if not dry_run:
|
||||||
|
typer.echo(f"Deleted bytes={deleted_size}")
|
||||||
|
|
||||||
|
reason_counts = Counter(row.reason_code for row in matched_rows)
|
||||||
|
if reason_counts:
|
||||||
|
reason_text = " ".join(
|
||||||
|
f"{reason}={reason_counts[reason]}" for reason in sorted(reason_counts)
|
||||||
|
)
|
||||||
|
typer.echo(f"Reasons: {reason_text}")
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
app()
|
app()
|
||||||
|
|
||||||
@@ -360,3 +438,67 @@ def _parse_extensions(raw_extensions: str) -> tuple[str, ...]:
|
|||||||
|
|
||||||
def _dump_json(values: tuple[float, ...]) -> str:
|
def _dump_json(values: tuple[float, ...]) -> str:
|
||||||
return orjson.dumps(values).decode("utf-8")
|
return orjson.dumps(values).decode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_scan_roots(
|
||||||
|
positional_paths: list[str],
|
||||||
|
paths_file: Path | None,
|
||||||
|
) -> tuple[Path, ...]:
|
||||||
|
roots: list[Path] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
|
||||||
|
for raw_path in positional_paths:
|
||||||
|
_append_scan_root(
|
||||||
|
Path(raw_path),
|
||||||
|
roots=roots,
|
||||||
|
seen=seen,
|
||||||
|
source="command argument",
|
||||||
|
)
|
||||||
|
|
||||||
|
if paths_file is not None:
|
||||||
|
for line_number, raw_line in enumerate(
|
||||||
|
paths_file.read_text(encoding="utf-8-sig").splitlines(),
|
||||||
|
start=1,
|
||||||
|
):
|
||||||
|
candidate = raw_line.rstrip("\r\n")
|
||||||
|
if not candidate.strip():
|
||||||
|
continue
|
||||||
|
_append_scan_root(
|
||||||
|
Path(_strip_optional_wrapping_quotes(candidate)),
|
||||||
|
roots=roots,
|
||||||
|
seen=seen,
|
||||||
|
source=f"{paths_file}:{line_number}",
|
||||||
|
)
|
||||||
|
|
||||||
|
if not roots:
|
||||||
|
raise typer.BadParameter(
|
||||||
|
"Provide at least one directory path or use --paths-file.",
|
||||||
|
)
|
||||||
|
|
||||||
|
return tuple(roots)
|
||||||
|
|
||||||
|
|
||||||
|
def _append_scan_root(
|
||||||
|
path: Path,
|
||||||
|
*,
|
||||||
|
roots: list[Path],
|
||||||
|
seen: set[str],
|
||||||
|
source: str,
|
||||||
|
) -> None:
|
||||||
|
if not path.exists():
|
||||||
|
raise typer.BadParameter(f"Path from {source} does not exist: {path}")
|
||||||
|
if not path.is_dir():
|
||||||
|
raise typer.BadParameter(f"Path from {source} is not a directory: {path}")
|
||||||
|
|
||||||
|
normalized = str(path.resolve())
|
||||||
|
if normalized in seen:
|
||||||
|
return
|
||||||
|
|
||||||
|
seen.add(normalized)
|
||||||
|
roots.append(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_optional_wrapping_quotes(value: str) -> str:
|
||||||
|
if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
|
||||||
|
return value[1:-1]
|
||||||
|
return value
|
||||||
@@ -90,6 +90,34 @@ class ScanStore:
|
|||||||
)
|
)
|
||||||
return int(cursor.lastrowid)
|
return int(cursor.lastrowid)
|
||||||
|
|
||||||
|
def get_latest_run_id(self, *, prefer_completed: bool = True) -> int | None:
|
||||||
|
with self._connect() as connection:
|
||||||
|
if prefer_completed:
|
||||||
|
row = connection.execute(
|
||||||
|
"""
|
||||||
|
SELECT id
|
||||||
|
FROM scan_runs
|
||||||
|
WHERE status = 'completed'
|
||||||
|
ORDER BY id DESC
|
||||||
|
LIMIT 1
|
||||||
|
"""
|
||||||
|
).fetchone()
|
||||||
|
if row is not None:
|
||||||
|
return int(row["id"])
|
||||||
|
|
||||||
|
row = connection.execute(
|
||||||
|
"""
|
||||||
|
SELECT id
|
||||||
|
FROM scan_runs
|
||||||
|
ORDER BY id DESC
|
||||||
|
LIMIT 1
|
||||||
|
"""
|
||||||
|
).fetchone()
|
||||||
|
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
return int(row["id"])
|
||||||
|
|
||||||
def upsert_files(
|
def upsert_files(
|
||||||
self,
|
self,
|
||||||
run_id: int,
|
run_id: int,
|
||||||
@@ -234,3 +234,218 @@ def test_scan_no_resume_forces_probe_execution(tmp_path, monkeypatch) -> None:
|
|||||||
|
|
||||||
assert second_result.exit_code == 0
|
assert second_result.exit_code == 0
|
||||||
assert probe_calls == 1
|
assert probe_calls == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_can_load_directories_from_paths_file(tmp_path, monkeypatch) -> None:
|
||||||
|
root_a = tmp_path / "Dir With Space"
|
||||||
|
root_b = tmp_path / "[Special]!目录"
|
||||||
|
root_a.mkdir()
|
||||||
|
root_b.mkdir()
|
||||||
|
(root_a / "a.avi").write_bytes(b"a")
|
||||||
|
(root_b / "b.avi").write_bytes(b"b")
|
||||||
|
|
||||||
|
paths_file = tmp_path / "paths.txt"
|
||||||
|
paths_file.write_text(
|
||||||
|
f'{root_a}\n"{root_b}"\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
probe_calls: list[str] = []
|
||||||
|
|
||||||
|
def fake_run_ffprobe(path, **kwargs):
|
||||||
|
probe_calls.append(path.name)
|
||||||
|
return ProbeExecutionResult(
|
||||||
|
status="completed",
|
||||||
|
command=("ffprobe", str(path)),
|
||||||
|
exit_code=0,
|
||||||
|
stdout_text='{"format":{"format_name":"avi","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
|
||||||
|
stderr_text="",
|
||||||
|
elapsed_ms=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
|
||||||
|
|
||||||
|
result = runner.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"scan",
|
||||||
|
"--paths-file",
|
||||||
|
str(paths_file),
|
||||||
|
"--db",
|
||||||
|
str(tmp_path / "scan_state.db"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "Scanned 2 file(s)" in result.stdout
|
||||||
|
assert sorted(probe_calls) == ["a.avi", "b.avi"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_merges_positional_paths_and_paths_file(tmp_path, monkeypatch) -> None:
|
||||||
|
direct_root = tmp_path / "direct"
|
||||||
|
listed_root = tmp_path / "listed"
|
||||||
|
direct_root.mkdir()
|
||||||
|
listed_root.mkdir()
|
||||||
|
(direct_root / "direct.avi").write_bytes(b"a")
|
||||||
|
(listed_root / "listed.avi").write_bytes(b"b")
|
||||||
|
|
||||||
|
paths_file = tmp_path / "paths.txt"
|
||||||
|
paths_file.write_text(f"{listed_root}\n", encoding="utf-8")
|
||||||
|
|
||||||
|
probe_calls: list[str] = []
|
||||||
|
|
||||||
|
def fake_run_ffprobe(path, **kwargs):
|
||||||
|
probe_calls.append(path.name)
|
||||||
|
return ProbeExecutionResult(
|
||||||
|
status="completed",
|
||||||
|
command=("ffprobe", str(path)),
|
||||||
|
exit_code=0,
|
||||||
|
stdout_text='{"format":{"format_name":"avi","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
|
||||||
|
stderr_text="",
|
||||||
|
elapsed_ms=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
|
||||||
|
|
||||||
|
result = runner.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"scan",
|
||||||
|
str(direct_root),
|
||||||
|
"--paths-file",
|
||||||
|
str(paths_file),
|
||||||
|
"--db",
|
||||||
|
str(tmp_path / "scan_state.db"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "Scanned 2 file(s)" in result.stdout
|
||||||
|
assert sorted(probe_calls) == ["direct.avi", "listed.avi"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_prune_dry_run_reports_confirmed_broken_without_deleting(tmp_path, monkeypatch) -> None:
|
||||||
|
root = tmp_path / "videos"
|
||||||
|
root.mkdir()
|
||||||
|
broken = root / "broken.mp4"
|
||||||
|
passed = root / "passed.avi"
|
||||||
|
broken.write_bytes(b"broken")
|
||||||
|
passed.write_bytes(b"passed")
|
||||||
|
db_path = tmp_path / "scan_state.db"
|
||||||
|
|
||||||
|
def fake_run_ffprobe(path, **kwargs):
|
||||||
|
if path == broken:
|
||||||
|
return ProbeExecutionResult(
|
||||||
|
status="completed",
|
||||||
|
command=("ffprobe", str(path)),
|
||||||
|
exit_code=1,
|
||||||
|
stdout_text="",
|
||||||
|
stderr_text="moov atom not found",
|
||||||
|
elapsed_ms=1,
|
||||||
|
)
|
||||||
|
return ProbeExecutionResult(
|
||||||
|
status="completed",
|
||||||
|
command=("ffprobe", str(path)),
|
||||||
|
exit_code=0,
|
||||||
|
stdout_text='{"format":{"format_name":"avi","duration":"2.0"},"streams":[{"codec_type":"video"}]}',
|
||||||
|
stderr_text="",
|
||||||
|
elapsed_ms=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
|
||||||
|
|
||||||
|
scan_result = runner.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"scan",
|
||||||
|
str(root),
|
||||||
|
"--db",
|
||||||
|
str(db_path),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert scan_result.exit_code == 0
|
||||||
|
|
||||||
|
prune_result = runner.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"prune",
|
||||||
|
"--db",
|
||||||
|
str(db_path),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert prune_result.exit_code == 0
|
||||||
|
assert "Dry-run" in prune_result.stdout
|
||||||
|
assert "matched=1" in prune_result.stdout
|
||||||
|
assert "deleted=0" in prune_result.stdout
|
||||||
|
assert broken.exists()
|
||||||
|
assert passed.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_prune_no_dry_run_deletes_confirmed_broken_files(tmp_path, monkeypatch) -> None:
|
||||||
|
root = tmp_path / "videos"
|
||||||
|
root.mkdir()
|
||||||
|
broken = root / "broken.mp4"
|
||||||
|
suspicious = root / "suspicious.mp4"
|
||||||
|
broken.write_bytes(b"broken")
|
||||||
|
suspicious.write_bytes(b"suspicious")
|
||||||
|
db_path = tmp_path / "scan_state.db"
|
||||||
|
|
||||||
|
def fake_run_ffprobe(path, **kwargs):
|
||||||
|
if path == broken:
|
||||||
|
return ProbeExecutionResult(
|
||||||
|
status="completed",
|
||||||
|
command=("ffprobe", str(path)),
|
||||||
|
exit_code=1,
|
||||||
|
stdout_text="",
|
||||||
|
stderr_text="moov atom not found",
|
||||||
|
elapsed_ms=1,
|
||||||
|
)
|
||||||
|
return ProbeExecutionResult(
|
||||||
|
status="completed",
|
||||||
|
command=("ffprobe", str(path)),
|
||||||
|
exit_code=0,
|
||||||
|
stdout_text='{"format":{"format_name":"mov,mp4,m4a,3gp,3g2,mj2","duration":"20.0"},"streams":[{"codec_type":"video"}]}',
|
||||||
|
stderr_text="",
|
||||||
|
elapsed_ms=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
def fake_run_ffmpeg(path, **kwargs):
|
||||||
|
return FfmpegExecutionResult(
|
||||||
|
status="completed",
|
||||||
|
command=("ffmpeg", str(path)),
|
||||||
|
exit_code=0,
|
||||||
|
stdout_text="frame=10",
|
||||||
|
stderr_text="",
|
||||||
|
elapsed_ms=2,
|
||||||
|
reason_code=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr("tgvideo_check.cli.run_ffprobe", fake_run_ffprobe)
|
||||||
|
monkeypatch.setattr("tgvideo_check.cli.run_ffmpeg_verification", fake_run_ffmpeg)
|
||||||
|
|
||||||
|
scan_result = runner.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"scan",
|
||||||
|
str(root),
|
||||||
|
"--db",
|
||||||
|
str(db_path),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert scan_result.exit_code == 0
|
||||||
|
|
||||||
|
prune_result = runner.invoke(
|
||||||
|
app,
|
||||||
|
[
|
||||||
|
"prune",
|
||||||
|
"--db",
|
||||||
|
str(db_path),
|
||||||
|
"--no-dry-run",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert prune_result.exit_code == 0
|
||||||
|
assert "deleted=1" in prune_result.stdout
|
||||||
|
assert not broken.exists()
|
||||||
|
assert suspicious.exists()
|
||||||
Generated
Reference in New Issue
Block a user