kim/clean-broken-tgvideos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve scan progress reporting

Browse Source
This commit is contained in:
kim
2026-03-30 14:46:52 +08:00
parent cd4965f46a
commit 6125a9389c
6 changed files with 226 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tgvideo_check/cli.py
+113 -12
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from collections import Counter
from dataclasses import dataclass, field
from pathlib import Path
import orjson
@@ -15,7 +16,7 @@ from tgvideo_check.profiles import resolve_config
from tgvideo_check.reporting import ReportRow, export_csv_summary, export_jsonl
from tgvideo_check.scheduler import (
build_stage_plan,
execute_stage_batch,
iter_stage_batch_results,
select_pending_stage1_files,
select_pending_stage2_files,
)
@@ -27,6 +28,82 @@ app = typer.Typer(
help="Batch detector for truncated or damaged Telegram-exported videos.",
)
DISCOVERY_PROGRESS_EVERY = 100
@dataclass
class _StageProgress:
label: str
total: int
counts: Counter[str] = field(default_factory=Counter)
last_reported: int = 0
def record(self, status: str) -> None:
self.counts[status] += 1
completed = sum(self.counts.values())
if not self._should_report(completed):
return
percent = (completed * 100) // self.total if self.total else 100
status_text = _format_progress_counts(self.label, self.counts)
typer.echo(
f"{self.label} progress: {completed}/{self.total} ({percent}%) {status_text}".rstrip()
)
self.last_reported = completed
def _should_report(self, completed: int) -> bool:
if completed == self.total or completed == 1:
return True
report_interval = max(1, self.total // 20)
return completed - self.last_reported >= report_interval
class ScanProgressReporter:
def discovery_started(self, roots: tuple[Path, ...]) -> None:
typer.echo(f"Discovery: scanning {len(roots)} root(s)")
def discovery_found(self, discovered_count: int) -> None:
if discovered_count == 1 or discovered_count % DISCOVERY_PROGRESS_EVERY == 0:
typer.echo(f"Discovery progress: found={discovered_count}")
def discovery_completed(self, stored_files: list) -> None:
pending_stage1 = sum(
1 for item in stored_files if item.stage1_status == "pending_stage1"
)
pending_stage2 = sum(
1
for item in stored_files
if item.stage1_status == "pending_stage2" and item.stage2_status is None
)
completed = sum(1 for item in stored_files if item.final_status is not None)
reused = sum(1 for item in stored_files if item.resumed_from_run_id is not None)
typer.echo(
"Discovery summary: "
f"found={len(stored_files)} reused={reused} "
f"pending_stage1={pending_stage1} pending_stage2={pending_stage2} "
f"completed={completed}"
)
def stage_started(self, *, stage_name: str, total: int, workers: int) -> _StageProgress | None:
label = stage_name.capitalize()
if total == 0:
typer.echo(f"{label}: no pending files")
return None
typer.echo(f"{label}: total={total} workers={workers}")
return _StageProgress(label=label, total=total)
def _format_progress_counts(label: str, counts: Counter[str]) -> str:
if label == "Stage1":
ordered_keys = ("pending_stage2", "confirmed_broken", "passed_stage1", "command_error")
else:
ordered_keys = ("passed_stage2", "confirmed_broken", "command_error")
return " ".join(
f"{key}={counts[key]}"
for key in ordered_keys
if counts[key]
)
@app.callback()
def root() -> None:
@@ -176,32 +253,43 @@ def _execute_scan(config: ScanConfig) -> list[ReportRow]:
run_id = store.create_run(profile=config.profile, root_paths=config.roots)
reasons: dict[int, str] = {}
stage1_payloads: dict[int, dict[str, object]] = {}
progress = ScanProgressReporter()
try:
progress.discovery_started(config.roots)
candidates = discover_files(
config.roots,
extensions=config.extensions,
recursive=config.recursive,
on_discovered=lambda candidate: progress.discovery_found(
candidate.discovery_order + 1
),
)
stored_files = store.upsert_files(
run_id,
candidates,
reuse_previous_results=config.resume,
)
progress.discovery_completed(stored_files)
stage1_files = select_pending_stage1_files(stored_files)
stage1_batches = build_stage_plan(
stage1_files=select_pending_stage1_files(stored_files),
stage1_files=stage1_files,
stage2_files=[],
stage1_workers=config.stage1_workers or 1,
stage2_workers=config.stage2_workers or 1,
queue_limit=config.queue_limit,
)
stage1_progress = progress.stage_started(
stage_name="stage1",
total=len(stage1_files),
workers=config.stage1_workers or 1,
)
for batch in stage1_batches:
probe_results = execute_stage_batch(
for stored_file, probe_result in iter_stage_batch_results(
batch,
lambda item: run_ffprobe(item.path),
)
for stored_file, probe_result in zip(batch.files, probe_results):
):
store.record_stage1_result(
stored_file.file_id,
exit_code=probe_result.exit_code,
@@ -211,24 +299,30 @@ def _execute_scan(config: ScanConfig) -> list[ReportRow]:
)
if probe_result.status != "completed":
stage1_status = "command_error"
store.record_stage1_status(
stored_file.file_id,
stage1_status="command_error",
stage1_status=stage1_status,
final_status="command_error",
reason_code=probe_result.status,
)
reasons[stored_file.file_id] = probe_result.status
if stage1_progress is not None:
stage1_progress.record(stage1_status)
continue
payload = _parse_probe_payload(probe_result.stdout_text)
if probe_result.stdout_text and payload is None:
stage1_status = "command_error"
store.record_stage1_status(
stored_file.file_id,
stage1_status="command_error",
stage1_status=stage1_status,
final_status="command_error",
reason_code="invalid_probe_output",
)
reasons[stored_file.file_id] = "invalid_probe_output"
if stage1_progress is not None:
stage1_progress.record(stage1_status)
continue
classification = classify_stage1_result(
@@ -252,17 +346,25 @@ def _execute_scan(config: ScanConfig) -> list[ReportRow]:
reasons[stored_file.file_id] = classification.reason_code
if payload is not None:
stage1_payloads[stored_file.file_id] = payload
if stage1_progress is not None:
stage1_progress.record(classification.status)
stored_files = store.list_files(run_id)
stage2_files = select_pending_stage2_files(stored_files)
stage2_batches = build_stage_plan(
stage1_files=[],
stage2_files=select_pending_stage2_files(stored_files),
stage2_files=stage2_files,
stage1_workers=config.stage1_workers or 1,
stage2_workers=config.stage2_workers or 1,
queue_limit=config.queue_limit,
)
stage2_progress = progress.stage_started(
stage_name="stage2",
total=len(stage2_files),
workers=config.stage2_workers or 1,
)
for batch in stage2_batches:
stage2_outputs = execute_stage_batch(
for stored_file, (stage2_status, reason_code, stage2_result) in iter_stage_batch_results(
batch,
lambda item: _run_stage2_for_file(
item.path,
@@ -272,9 +374,6 @@ def _execute_scan(config: ScanConfig) -> list[ReportRow]:
sample_duration=config.sample_duration or 3,
full_decode_confirm=config.full_decode_confirm,
),
)
for stored_file, (stage2_status, reason_code, stage2_result) in zip(
batch.files, stage2_outputs
):
store.record_stage2_result(
stored_file.file_id,
@@ -292,6 +391,8 @@ def _execute_scan(config: ScanConfig) -> list[ReportRow]:
reason_code=reason_code,
)
reasons[stored_file.file_id] = reason_code
if stage2_progress is not None:
stage2_progress.record(stage2_status)
rows = _build_report_rows(store.list_files(run_id), reasons)
if config.export_jsonl is not None:
src/tgvideo_check/discovery.py
+14 -8
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from collections.abc import Callable
from pathlib import Path
from tgvideo_check.config import normalize_extension
@@ -11,6 +12,7 @@ def discover_files(
*,
extensions: tuple[str, ...],
recursive: bool = True,
on_discovered: Callable[[FileCandidate], None] | None = None,
) -> list[FileCandidate]:
normalized_extensions = {normalize_extension(item) for item in extensions}
discovered: list[FileCandidate] = []
@@ -21,6 +23,7 @@ def discover_files(
recursive=recursive,
extensions=normalized_extensions,
discovered=discovered,
on_discovered=on_discovered,
)
return discovered
@@ -32,6 +35,7 @@ def _discover_root(
recursive: bool,
extensions: set[str],
discovered: list[FileCandidate],
on_discovered: Callable[[FileCandidate], None] | None,
) -> None:
entries: list[tuple[Path, bool]] = []
try:
@@ -61,6 +65,7 @@ def _discover_root(
recursive=recursive,
extensions=extensions,
discovered=discovered,
on_discovered=on_discovered,
)
continue
@@ -73,12 +78,13 @@ def _discover_root(
except OSError:
continue
discovered.append(
FileCandidate(
path=entry,
size=stat.st_size,
mtime_ns=stat.st_mtime_ns,
extension=extension,
discovery_order=len(discovered),
)
candidate = FileCandidate(
path=entry,
size=stat.st_size,
mtime_ns=stat.st_mtime_ns,
extension=extension,
discovery_order=len(discovered),
)
discovered.append(candidate)
if on_discovered is not None:
on_discovered(candidate)
src/tgvideo_check/scheduler.py
+22 -2
View File
@@ -1,8 +1,9 @@
from __future__ import annotations
from collections.abc import Callable, Iterator
from dataclasses import dataclass
from concurrent.futures import ThreadPoolExecutor
from typing import Callable, TypeVar
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import TypeVar
from tgvideo_check.models import StoredFile
@@ -80,6 +81,25 @@ def execute_stage_batch(
return tuple(executor.map(worker, batch.files))
def iter_stage_batch_results(
batch: StageBatch,
worker: Callable[[StoredFile], T],
) -> Iterator[tuple[StoredFile, T]]:
if len(batch.files) <= 1 or batch.worker_count <= 1:
for item in batch.files:
yield item, worker(item)
return
max_workers = min(batch.worker_count, len(batch.files))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = {
executor.submit(worker, item): item
for item in batch.files
}
for future in as_completed(futures):
yield futures[future], future.result()
def _chunk_by_limit(
files: list[StoredFile], queue_limit: int
) -> tuple[tuple[StoredFile, ...], ...]: