254 lines
10 KiB
Python
254 lines
10 KiB
Python
"""Retención diaria de respaldos aplicados para no saturar el disco del servidor.
|
|
|
|
Dos limpiezas independientes sobre las carpetas locales de CloudRestoreAS:
|
|
|
|
- ``Procesados/``: por NODO. Para cada nodo se toma su restauración más reciente como
|
|
referencia y se conservan las de los últimos ``days``; se borran las anteriores (respaldos
|
|
ya aplicados y obsoletos). Nunca se borra la más reciente ni nodos con una sola restauración.
|
|
- ``Fallados/``: por ANTIGÜEDAD absoluta. Se borran los ZIP cuya carpeta-fecha sea anterior a
|
|
``hoy - failed_days`` (los fallos no tienen semántica de "última restauración exitosa por nodo").
|
|
|
|
Salvaguardas: solo ``unlink`` de archivos dentro de la carpeta configurada; nunca ``rmtree``;
|
|
no sigue symlinks; y jamás toca la carpeta-fecha de la restauración más reciente de un nodo.
|
|
El borrado físico se correlaciona con la tabla ``jobs`` (la BD no guarda la ruta destino), por
|
|
lo que la carpeta-fecha se deriva de ``finished_at`` con tolerancia de ±1 día por el desfase
|
|
UTC/local del momento del movimiento.
|
|
"""
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from typing import Callable, Optional
|
|
|
|
from ..db.event_repository import EventRepository
|
|
from ..db.job_repository import Job, JobRepository
|
|
from ..extract.seven_zip import SevenZipExtractor
|
|
from ..utils.logger import app_logger
|
|
|
|
|
|
@dataclass
|
|
class RetentionResult:
|
|
"""Resumen de una corrida de retención."""
|
|
|
|
dry_run: bool
|
|
deleted_files: int = 0
|
|
freed_bytes: int = 0
|
|
purged_jobs: int = 0
|
|
missing_files: int = 0
|
|
errors: int = 0
|
|
|
|
|
|
class RetentionCleaner:
|
|
"""Aplica la política de retención sobre las carpetas Procesados/ y Fallados/."""
|
|
|
|
def __init__(
|
|
self,
|
|
config: dict,
|
|
*,
|
|
dry_run: Optional[bool] = None,
|
|
clock: Callable[[], datetime] = datetime.now,
|
|
) -> None:
|
|
retention = config.get("retention", {})
|
|
paths = config.get("paths", {})
|
|
|
|
processed_raw = (paths.get("processed_folder") or "").strip()
|
|
failed_raw = (paths.get("failed_folder") or "").strip()
|
|
self._processed_folder = Path(processed_raw) if processed_raw else None
|
|
self._failed_folder = Path(failed_raw) if failed_raw else None
|
|
|
|
self._days = int(retention.get("days", 2))
|
|
self._failed_days = int(retention.get("failed_days", 7))
|
|
# El dry_run explícito (p.ej. botón manual) gana sobre la config.
|
|
self._dry_run = bool(retention.get("dry_run", True)) if dry_run is None else dry_run
|
|
self._clock = clock
|
|
|
|
# -- Orquestación ----------------------------------------------------------------
|
|
|
|
def run(self) -> RetentionResult:
|
|
"""Ejecuta ambas limpiezas y devuelve el resumen."""
|
|
result = RetentionResult(dry_run=self._dry_run)
|
|
self._clean_processed(result)
|
|
self._clean_failed(result)
|
|
|
|
mode = "SIMULACRO" if self._dry_run else "real"
|
|
freed_mb = result.freed_bytes / (1024 * 1024)
|
|
verb = "se borrarían" if self._dry_run else "borrados"
|
|
message = (
|
|
f"Retención ({mode}): {result.deleted_files} archivo(s) {verb} "
|
|
f"({freed_mb:.1f} MB), {result.purged_jobs} job(s) marcados, "
|
|
f"{result.missing_files} no hallado(s), {result.errors} error(es)"
|
|
)
|
|
app_logger.info(message)
|
|
EventRepository.create("INFO", message)
|
|
return result
|
|
|
|
# -- Procesados (por nodo) -------------------------------------------------------
|
|
|
|
def _clean_processed(self, result: RetentionResult) -> None:
|
|
if not self._processed_folder or not self._processed_folder.is_dir():
|
|
return
|
|
|
|
obsolete = JobRepository.get_obsolete_completed_by_node(self._days)
|
|
if not obsolete:
|
|
return
|
|
|
|
# Fecha-carpeta local de la restauración más reciente de cada nodo: intocable.
|
|
ref_dates = {
|
|
node: self._local_date_from_iso(finished_at)
|
|
for node, finished_at in JobRepository.get_latest_completed_per_node().items()
|
|
}
|
|
|
|
for job in obsolete:
|
|
local_date = self._local_date_from_iso(job.finished_at)
|
|
if local_date is None:
|
|
continue
|
|
|
|
ref_date = ref_dates.get(job.node_name)
|
|
parts, date_folder = self._resolve_processed_paths(job, local_date, ref_date)
|
|
|
|
if not parts:
|
|
# El archivo ya no está en disco (o el nombre no coincide): lo damos por
|
|
# purgado para no re-escanearlo indefinidamente.
|
|
result.missing_files += 1
|
|
self._mark_purged(job, result)
|
|
continue
|
|
|
|
self._delete_paths(parts, self._processed_folder, result)
|
|
self._mark_purged(job, result)
|
|
if not self._dry_run and date_folder is not None:
|
|
self._cleanup_empty_dir(date_folder)
|
|
|
|
def _resolve_processed_paths(
|
|
self, job: Job, local_date: date, ref_date: Optional[date]
|
|
) -> tuple[list[Path], Optional[Path]]:
|
|
"""Localiza el/los archivo(s) del job en Procesados/ derivando la carpeta-fecha.
|
|
|
|
Devuelve (partes_existentes, carpeta_fecha) o ([], None) si no se localizó. Nunca
|
|
considera la carpeta-fecha de la restauración más reciente del nodo (``ref_date``).
|
|
"""
|
|
for candidate in self._candidate_dates(local_date):
|
|
if ref_date is not None and candidate == ref_date:
|
|
continue
|
|
date_folder = self._processed_folder / candidate.isoformat()
|
|
if not date_folder.is_dir():
|
|
continue
|
|
parts = self._collect_parts_in_folder(date_folder, job.source_name)
|
|
if parts:
|
|
return parts, date_folder
|
|
return [], None
|
|
|
|
# -- Fallados (por antigüedad absoluta) ------------------------------------------
|
|
|
|
def _clean_failed(self, result: RetentionResult) -> None:
|
|
if not self._failed_folder or not self._failed_folder.is_dir():
|
|
return
|
|
|
|
cutoff = self._clock().date() - timedelta(days=self._failed_days)
|
|
for date_folder in sorted(self._failed_folder.iterdir()):
|
|
if not date_folder.is_dir():
|
|
continue
|
|
folder_date = self._parse_date_folder(date_folder.name)
|
|
if folder_date is None:
|
|
# Carpeta con nombre que no es una fecha: no la tocamos.
|
|
continue
|
|
if folder_date >= cutoff:
|
|
continue
|
|
|
|
files = [p for p in date_folder.iterdir() if p.is_file()]
|
|
self._delete_paths(files, self._failed_folder, result)
|
|
if not self._dry_run:
|
|
self._cleanup_empty_dir(date_folder)
|
|
|
|
# -- Helpers de borrado ----------------------------------------------------------
|
|
|
|
def _delete_paths(
|
|
self, paths: list[Path], root: Path, result: RetentionResult
|
|
) -> None:
|
|
for path in paths:
|
|
try:
|
|
if path.is_symlink():
|
|
app_logger.warning(f"Retención: se omite symlink {path}")
|
|
continue
|
|
if not self._is_inside(path, root):
|
|
app_logger.warning(
|
|
f"Retención: se omite ruta fuera de {root}: {path}"
|
|
)
|
|
continue
|
|
if not path.is_file():
|
|
continue
|
|
|
|
size = path.stat().st_size
|
|
if self._dry_run:
|
|
app_logger.info(f"[SIMULACRO] Se borraría {path} ({size} bytes)")
|
|
else:
|
|
path.unlink()
|
|
app_logger.info(f"Retención: borrado {path} ({size} bytes)")
|
|
result.deleted_files += 1
|
|
result.freed_bytes += size
|
|
except FileNotFoundError:
|
|
result.missing_files += 1
|
|
except OSError as e:
|
|
app_logger.warning(f"Retención: no se pudo borrar {path}: {e}")
|
|
result.errors += 1
|
|
|
|
def _cleanup_empty_dir(self, folder: Path) -> None:
|
|
"""Borra la carpeta-fecha solo si quedó vacía (best-effort, nunca rmtree)."""
|
|
try:
|
|
if folder.is_dir() and not any(folder.iterdir()):
|
|
folder.rmdir()
|
|
app_logger.info(f"Retención: carpeta vacía eliminada {folder}")
|
|
except OSError as e:
|
|
app_logger.warning(f"Retención: no se pudo eliminar carpeta {folder}: {e}")
|
|
|
|
def _mark_purged(self, job: Job, result: RetentionResult) -> None:
|
|
if self._dry_run:
|
|
return
|
|
JobRepository.mark_purged(job.job_id)
|
|
result.purged_jobs += 1
|
|
|
|
# -- Helpers puros ---------------------------------------------------------------
|
|
|
|
@staticmethod
|
|
def _collect_parts_in_folder(date_folder: Path, source_name: str) -> list[Path]:
|
|
"""Rutas del archivo (y sus partes multipart) dentro de una carpeta-fecha."""
|
|
if SevenZipExtractor.is_multipart(source_name):
|
|
base_name = Path(source_name).stem.split(".zip")[0]
|
|
return sorted(date_folder.glob(f"{base_name}.zip.*"))
|
|
candidate = date_folder / source_name
|
|
return [candidate] if candidate.exists() else []
|
|
|
|
@staticmethod
|
|
def _candidate_dates(local_date: date) -> list[date]:
|
|
"""Fecha exacta y ±1 día, para absorber el desfase de medianoche/zona horaria."""
|
|
return [local_date, local_date - timedelta(days=1), local_date + timedelta(days=1)]
|
|
|
|
@staticmethod
|
|
def _local_date_from_iso(finished_at: Optional[str]) -> Optional[date]:
|
|
"""Convierte un finished_at (ISO UTC naive) a la fecha local del movimiento."""
|
|
if not finished_at:
|
|
return None
|
|
try:
|
|
dt = datetime.fromisoformat(finished_at)
|
|
except ValueError:
|
|
return None
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
return dt.astimezone().date()
|
|
|
|
@staticmethod
|
|
def _parse_date_folder(name: str) -> Optional[date]:
|
|
try:
|
|
return datetime.strptime(name, "%Y-%m-%d").date()
|
|
except ValueError:
|
|
return None
|
|
|
|
@staticmethod
|
|
def _is_inside(path: Path, root: Path) -> bool:
|
|
"""True si `path` resuelve dentro de `root` (anti path-traversal)."""
|
|
try:
|
|
resolved = path.resolve()
|
|
root_resolved = root.resolve()
|
|
except OSError:
|
|
return False
|
|
return resolved == root_resolved or root_resolved in resolved.parents
|