Initial commit: CloudRestoreAS v1.0.0 - Aplicación completa de restauración automática SQL Server

This commit is contained in:
2026-01-25 16:53:00 -07:00
commit 854ffa116f
43 changed files with 6357 additions and 0 deletions

1
app/engine/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""Motor de procesamiento."""

310
app/engine/engine.py Normal file
View File

@@ -0,0 +1,310 @@
"""Motor principal de la aplicación."""
from typing import Optional
from pathlib import Path
from PySide6.QtCore import QObject, Signal, QThreadPool
from .file_watcher import FileWatcher, FileStabilityChecker, calculate_file_hash
from .restore_worker import RestoreWorker
from ..db.job_repository import JobRepository
from ..db.event_repository import EventRepository
from ..db.config_repository import ConfigRepository
from ..constants import JobStatus, DEFAULT_CONFIG
from ..utils.logger import app_logger
class EngineSignals(QObject):
"""Señales del motor."""
job_created = Signal(str) # job_id
stats_updated = Signal(dict) # stats
config_loaded = Signal(dict) # config
class RestoreEngine(QObject):
"""Motor principal de restauración."""
def __init__(self):
"""Inicializa el motor."""
super().__init__()
self.signals = EngineSignals()
# Thread pool para workers
self._thread_pool = QThreadPool.globalInstance()
# File watcher
self._file_watcher: Optional[FileWatcher] = None
# Estado
self._running = False
self._paused = False
# Configuración
self._config = self._load_config()
app_logger.info("RestoreEngine inicializado")
def _load_config(self) -> dict:
"""Carga la configuración desde la base de datos."""
config = ConfigRepository.get("app_config", DEFAULT_CONFIG.copy())
# Asegurar que tiene todas las claves
for key, value in DEFAULT_CONFIG.items():
if key not in config:
config[key] = value
self.signals.config_loaded.emit(config)
return config
def save_config(self, config: dict):
"""
Guarda la configuración.
Args:
config: Configuración a guardar
"""
ConfigRepository.set("app_config", config)
self._config = config
app_logger.info("Configuración guardada")
# Reconfigurar file watcher si está corriendo
if self._running:
self._restart_file_watcher()
def get_config(self) -> dict:
"""Obtiene la configuración actual."""
return self._config.copy()
def start(self):
"""Inicia el motor."""
if self._running:
app_logger.warning("Motor ya está corriendo")
return
# Validar configuración
if not self._validate_config():
app_logger.error("Configuración inválida, no se puede iniciar el motor")
return
# Configurar thread pool
extract_workers = self._config["concurrency"]["extract_workers"]
restore_workers = self._config["concurrency"]["restore_workers"]
max_threads = extract_workers + restore_workers
self._thread_pool.setMaxThreadCount(max_threads)
app_logger.info(
f"Thread pool configurado: {max_threads} threads "
f"(extract: {extract_workers}, restore: {restore_workers})"
)
# Iniciar file watcher
if self._config["features"]["auto_scan_enabled"]:
self._start_file_watcher()
self._running = True
self._paused = False
EventRepository.create("INFO", "Motor iniciado")
app_logger.info("Motor iniciado")
def stop(self):
"""Detiene el motor."""
if not self._running:
return
# Detener file watcher
if self._file_watcher:
self._file_watcher.stop()
self._file_watcher = None
# Esperar a que terminen los workers
self._thread_pool.waitForDone(msecs=30000) # 30s timeout
self._running = False
EventRepository.create("INFO", "Motor detenido")
app_logger.info("Motor detenido")
def pause(self):
"""Pausa el procesamiento (no toma nuevos jobs)."""
self._paused = True
if self._file_watcher:
self._file_watcher.stop()
EventRepository.create("INFO", "Motor pausado")
app_logger.info("Motor pausado")
def resume(self):
"""Reanuda el procesamiento."""
if not self._running:
return
self._paused = False
if self._config["features"]["auto_scan_enabled"]:
self._start_file_watcher()
EventRepository.create("INFO", "Motor reanudado")
app_logger.info("Motor reanudado")
def is_running(self) -> bool:
"""Verifica si el motor está corriendo."""
return self._running
def is_paused(self) -> bool:
"""Verifica si el motor está pausado."""
return self._paused
def scan_now(self):
"""Fuerza un escaneo manual de la carpeta de entrada."""
if self._file_watcher:
self._file_watcher.scan_now()
else:
# Crear watcher temporal para un escaneo
self._start_file_watcher()
if self._file_watcher:
self._file_watcher.scan_now()
def process_file(self, file_path: str):
"""
Procesa un archivo manualmente.
Args:
file_path: Ruta al archivo ZIP
"""
try:
self._on_file_ready(file_path)
except Exception as e:
app_logger.error(f"Error procesando archivo {file_path}: {e}")
def get_stats(self) -> dict:
"""Obtiene estadísticas del motor."""
stats = JobRepository.get_stats()
stats["active_threads"] = self._thread_pool.activeThreadCount()
stats["max_threads"] = self._thread_pool.maxThreadCount()
stats["running"] = self._running
stats["paused"] = self._paused
# Tiempos promedio
avg_times = JobRepository.get_average_times()
stats.update(avg_times)
return stats
def _validate_config(self) -> bool:
"""Valida que la configuración sea correcta."""
paths = self._config["paths"]
# Validar carpetas requeridas
required_paths = ["input_folder", "extract_folder", "data_sql_folder"]
for key in required_paths:
if not paths.get(key):
app_logger.error(f"Falta configurar: {key}")
return False
path = Path(paths[key])
if not path.exists():
try:
path.mkdir(parents=True, exist_ok=True)
except Exception as e:
app_logger.error(f"No se pudo crear carpeta {key}: {e}")
return False
# Validar 7-Zip
if not paths.get("seven_zip_exe"):
app_logger.error("Falta configurar ruta a 7-Zip")
return False
if not Path(paths["seven_zip_exe"]).exists():
app_logger.error(f"7-Zip no encontrado en: {paths['seven_zip_exe']}")
return False
return True
def _start_file_watcher(self):
"""Inicia el file watcher."""
if self._file_watcher:
self._file_watcher.stop()
# Configurar stability checker
stability_config = self._config["stability"]
stability_checker = FileStabilityChecker(
check_interval=stability_config["check_interval_seconds"],
stable_duration=stability_config["stable_duration_seconds"],
use_ready_marker=stability_config["use_ready_marker"]
)
# Crear file watcher
self._file_watcher = FileWatcher(
watch_folder=self._config["paths"]["input_folder"],
on_file_ready=self._on_file_ready,
scan_interval=self._config["features"]["scan_interval_seconds"],
stability_checker=stability_checker
)
self._file_watcher.start()
def _restart_file_watcher(self):
"""Reinicia el file watcher con nueva configuración."""
if self._running and not self._paused:
self._start_file_watcher()
def _on_file_ready(self, file_path: str):
"""
Callback cuando un archivo está listo para procesar.
Args:
file_path: Ruta al archivo
"""
if self._paused:
app_logger.info(f"Motor pausado, ignorando archivo: {file_path}")
return
try:
# Calcular hash para evitar duplicados
file_hash = calculate_file_hash(file_path)
if JobRepository.exists_by_hash(file_hash):
app_logger.warning(f"Archivo ya procesado (hash duplicado): {file_path}")
return
# Crear job
file_name = Path(file_path).name
job_id = JobRepository.create(file_path, file_name, file_hash)
app_logger.info(f"Job creado: {job_id} para archivo {file_name}")
EventRepository.create("INFO", f"Nuevo job creado: {file_name}", job_id)
self.signals.job_created.emit(job_id)
# Crear worker y encolar
worker = RestoreWorker(
job_id=job_id,
config=self._config,
dry_run=self._config["features"]["dry_run_mode"]
)
# Conectar señales
worker.signals.job_completed.connect(self._on_job_completed)
# Ejecutar en thread pool
self._thread_pool.start(worker)
app_logger.info(f"Worker encolado para job {job_id}")
except Exception as e:
app_logger.error(f"Error creando job para {file_path}: {e}", exc_info=True)
EventRepository.create("ERROR", f"Error creando job: {e}")
def _on_job_completed(self, job_id: str, success: bool):
"""
Callback cuando un job se completa.
Args:
job_id: ID del job
success: Si fue exitoso
"""
app_logger.info(f"Job {job_id} completado - Éxito: {success}")
# Actualizar estadísticas
stats = self.get_stats()
self.signals.stats_updated.emit(stats)

211
app/engine/file_watcher.py Normal file
View File

@@ -0,0 +1,211 @@
"""File watcher para detectar nuevos archivos ZIP."""
import os
import time
import hashlib
from pathlib import Path
from typing import Optional, Dict, Callable
from threading import Thread, Event
from ..utils.logger import app_logger
class FileStabilityChecker:
"""Verifica que un archivo esté estable (no siendo copiado)."""
def __init__(
self,
check_interval: int = 5,
stable_duration: int = 10,
use_ready_marker: bool = False
):
"""
Inicializa el verificador de estabilidad.
Args:
check_interval: Intervalo entre verificaciones (segundos)
stable_duration: Tiempo que debe estar estable (segundos)
use_ready_marker: Si True, requiere archivo .ready
"""
self.check_interval = check_interval
self.stable_duration = stable_duration
self.use_ready_marker = use_ready_marker
def is_file_ready(self, file_path: str) -> bool:
"""
Verifica si un archivo está listo para procesarse.
Args:
file_path: Ruta al archivo
Returns:
True si está listo
"""
path = Path(file_path)
if not path.exists():
return False
# Verificar marcador .ready si está habilitado
if self.use_ready_marker:
ready_marker = path.parent / f"{path.name}.ready"
if not ready_marker.exists():
app_logger.debug(f"Esperando marcador .ready para {path.name}")
return False
# Verificar estabilidad de tamaño
try:
initial_size = path.stat().st_size
initial_mtime = path.stat().st_mtime
time.sleep(self.stable_duration)
if not path.exists():
return False
final_size = path.stat().st_size
final_mtime = path.stat().st_mtime
is_stable = (initial_size == final_size and initial_mtime == final_mtime)
if not is_stable:
app_logger.debug(
f"Archivo {path.name} aún está cambiando "
f"(size: {initial_size} -> {final_size})"
)
return is_stable
except Exception as e:
app_logger.error(f"Error verificando estabilidad de {file_path}: {e}")
return False
class FileWatcher:
"""Vigila una carpeta por nuevos archivos ZIP."""
def __init__(
self,
watch_folder: str,
on_file_ready: Callable[[str], None],
scan_interval: int = 30,
stability_checker: Optional[FileStabilityChecker] = None
):
"""
Inicializa el file watcher.
Args:
watch_folder: Carpeta a vigilar
on_file_ready: Callback cuando un archivo está listo
scan_interval: Intervalo de escaneo (segundos)
stability_checker: Verificador de estabilidad (opcional)
"""
self.watch_folder = watch_folder
self.on_file_ready = on_file_ready
self.scan_interval = scan_interval
self.stability_checker = stability_checker or FileStabilityChecker()
self._stop_event = Event()
self._thread: Optional[Thread] = None
self._known_files: Dict[str, float] = {} # path -> mtime
def start(self):
"""Inicia el watcher en un thread separado."""
if self._thread and self._thread.is_alive():
app_logger.warning("FileWatcher ya está corriendo")
return
self._stop_event.clear()
self._thread = Thread(target=self._run, daemon=True)
self._thread.start()
app_logger.info(f"FileWatcher iniciado en: {self.watch_folder}")
def stop(self):
"""Detiene el watcher."""
if self._thread:
self._stop_event.set()
self._thread.join(timeout=5)
app_logger.info("FileWatcher detenido")
def scan_now(self):
"""Fuerza un escaneo inmediato."""
app_logger.info("Escaneo manual solicitado")
self._scan_folder()
def _run(self):
"""Loop principal del watcher."""
while not self._stop_event.is_set():
try:
self._scan_folder()
except Exception as e:
app_logger.error(f"Error en FileWatcher: {e}", exc_info=True)
# Esperar con posibilidad de interrumpir
self._stop_event.wait(self.scan_interval)
def _scan_folder(self):
"""Escanea la carpeta por archivos nuevos."""
folder = Path(self.watch_folder)
if not folder.exists():
app_logger.warning(f"Carpeta de entrada no existe: {self.watch_folder}")
return
# Buscar archivos .zip y .zip.001
patterns = ["*.zip", "*.zip.001"]
found_files = []
for pattern in patterns:
found_files.extend(folder.glob(pattern))
for file_path in found_files:
file_str = str(file_path)
# Ignorar archivos ya procesados recientemente
if file_str in self._known_files:
continue
# Verificar si está listo
if not self.stability_checker.is_file_ready(file_str):
continue
# Marcar como conocido
self._known_files[file_str] = file_path.stat().st_mtime
# Notificar
app_logger.info(f"Archivo nuevo detectado y listo: {file_path.name}")
try:
self.on_file_ready(file_str)
except Exception as e:
app_logger.error(f"Error procesando archivo {file_path.name}: {e}")
# Limpiar archivos conocidos que ya no existen
self._clean_known_files()
def _clean_known_files(self):
"""Limpia archivos conocidos que ya no existen."""
to_remove = []
for file_path in list(self._known_files.keys()):
if not Path(file_path).exists():
to_remove.append(file_path)
for file_path in to_remove:
del self._known_files[file_path]
def calculate_file_hash(file_path: str) -> str:
"""
Calcula el SHA256 hash de un archivo.
Args:
file_path: Ruta al archivo
Returns:
Hash hexadecimal
"""
sha256 = hashlib.sha256()
with open(file_path, 'rb') as f:
while chunk := f.read(8192):
sha256.update(chunk)
return sha256.hexdigest()

View File

@@ -0,0 +1,324 @@
"""Worker para procesar jobs de restauración."""
import time
import shutil
from pathlib import Path
from typing import Optional
from datetime import datetime
from PySide6.QtCore import QObject, Signal, QRunnable
from ..constants import JobStatus, StepType
from ..db.job_repository import JobRepository
from ..db.job_step_repository import JobStepRepository
from ..db.event_repository import EventRepository
from ..db.node_repository import NodeRepository
from ..extract.seven_zip import SevenZipExtractor
from ..sql.sql_manager import SQLServerManager
from ..utils.logger import app_logger
class RestoreWorkerSignals(QObject):
"""Señales para comunicación con la UI."""
job_started = Signal(str) # job_id
job_progress = Signal(str, str) # job_id, status
job_completed = Signal(str, bool) # job_id, success
error_occurred = Signal(str, str) # job_id, error
class RestoreWorker(QRunnable):
"""Worker que procesa un job de restauración."""
def __init__(
self,
job_id: str,
config: dict,
dry_run: bool = False
):
"""
Inicializa el worker.
Args:
job_id: ID del job a procesar
config: Configuración de la aplicación
dry_run: Modo dry run (no ejecuta RESTORE)
"""
super().__init__()
self.job_id = job_id
self.config = config
self.dry_run = dry_run
self.signals = RestoreWorkerSignals()
self._extract_dir: Optional[str] = None
self._bak_path: Optional[str] = None
def run(self):
"""Ejecuta el procesamiento del job."""
start_time = time.time()
try:
app_logger.info(f"Iniciando procesamiento de job {self.job_id}")
self.signals.job_started.emit(self.job_id)
# Obtener job
job = JobRepository.get(self.job_id)
if not job:
raise ValueError(f"Job {self.job_id} no encontrado")
# Pipeline de procesamiento
self._process_node_mapping(job)
self._extract_backup(job)
self._restore_database(job)
self._cleanup(job)
# Actualizar tiempos
total_ms = int((time.time() - start_time) * 1000)
JobRepository.update_timing(self.job_id, total_ms=total_ms)
# Marcar como completado
JobRepository.update_status(self.job_id, JobStatus.COMPLETED)
app_logger.info(
f"Job {self.job_id} completado exitosamente en {total_ms}ms"
)
self.signals.job_completed.emit(self.job_id, True)
except Exception as e:
error_msg = str(e)
app_logger.error(f"Error procesando job {self.job_id}: {error_msg}", exc_info=True)
JobRepository.update_status(
self.job_id,
JobStatus.FAILED,
error=error_msg,
increment_attempts=True
)
EventRepository.create("ERROR", f"Job falló: {error_msg}", self.job_id)
self.signals.error_occurred.emit(self.job_id, error_msg)
self.signals.job_completed.emit(self.job_id, False)
def _process_node_mapping(self, job):
"""Procesa el mapeo de nodo a base de datos."""
step_id = JobStepRepository.create(self.job_id, StepType.NODE_MAPPING)
try:
# Obtener node name del archivo (con extensión, uppercase)
node_name = Path(job.source_name).name.upper()
# Buscar mapeo en la tabla nodes
db_name = NodeRepository.get_db_for_node(node_name)
if not db_name:
raise ValueError(
f"NODE_NOT_MAPPED: No existe mapeo activo para nodo '{node_name}'"
)
# Actualizar job con node_name y db_name
JobRepository.update_node_and_db(self.job_id, node_name, db_name)
app_logger.info(f"Node '{node_name}' mapeado a DB '{db_name}'")
JobStepRepository.complete(step_id, exit_code=0, stdout=f"DB: {db_name}")
except Exception as e:
JobStepRepository.complete(step_id, exit_code=1, error=str(e))
raise
def _extract_backup(self, job):
"""Extrae el backup del archivo ZIP."""
JobRepository.update_status(self.job_id, JobStatus.EXTRACTING)
self.signals.job_progress.emit(self.job_id, JobStatus.EXTRACTING)
step_id = JobStepRepository.create(self.job_id, StepType.EXTRACT)
extract_start = time.time()
try:
# Configuración
seven_zip_path = self.config["paths"]["seven_zip_exe"]
extract_base = self.config["paths"]["extract_folder"]
timeout_minutes = self.config["timeouts"]["extract_minutes"]
# Crear carpeta de extracción para este job
self._extract_dir = str(Path(extract_base) / self.job_id)
Path(self._extract_dir).mkdir(parents=True, exist_ok=True)
# Extraer
extractor = SevenZipExtractor(seven_zip_path)
# Si es multipart, asegurar que usamos el primer archivo
source_path = job.source_path
if extractor.is_multipart(source_path):
source_path = extractor.get_first_part(source_path)
app_logger.info(f"Archivo multipart detectado, usando: {source_path}")
exit_code, stdout, stderr = extractor.extract(
source_path,
self._extract_dir,
timeout_minutes
)
extract_ms = int((time.time() - extract_start) * 1000)
JobRepository.update_timing(self.job_id, extract_ms=extract_ms)
if exit_code != 0:
raise RuntimeError(f"7-Zip falló con código {exit_code}: {stderr}")
JobStepRepository.complete(
step_id,
exit_code=exit_code,
stdout=stdout[:1000] if stdout else None,
stderr=stderr[:1000] if stderr else None
)
# Buscar archivo .bak
locate_step_id = JobStepRepository.create(self.job_id, StepType.LOCATE_BAK)
self._bak_path = SevenZipExtractor.find_bak_file(self._extract_dir)
if not self._bak_path:
raise FileNotFoundError("No se encontró archivo .bak en el archivo extraído")
JobStepRepository.complete(
locate_step_id,
exit_code=0,
stdout=f"BAK: {Path(self._bak_path).name}"
)
except Exception as e:
JobStepRepository.complete(step_id, exit_code=1, error=str(e))
raise
def _restore_database(self, job):
"""Restaura la base de datos desde el backup."""
JobRepository.update_status(self.job_id, JobStatus.RESTORING)
self.signals.job_progress.emit(self.job_id, JobStatus.RESTORING)
# Recargar job para obtener db_name actualizado
job = JobRepository.get(self.job_id)
if not job.db_name:
raise ValueError("DB name no está configurado en el job")
# Conectar a SQL
connect_step_id = JobStepRepository.create(self.job_id, StepType.SQL_CONNECT)
try:
sql_config = self.config["sql"]
sql_manager = SQLServerManager(
server=sql_config["server"],
use_windows_auth=sql_config["use_windows_auth"],
username=sql_config.get("username"),
password=sql_config.get("password")
)
# Test connection
success, error = sql_manager.test_connection()
if not success:
raise RuntimeError(f"Conexión SQL falló: {error}")
JobStepRepository.complete(connect_step_id, exit_code=0)
except Exception as e:
JobStepRepository.complete(connect_step_id, exit_code=1, error=str(e))
raise
# Obtener FILELISTONLY
filelist_step_id = JobStepRepository.create(self.job_id, StepType.FILELIST)
filelist_start = time.time()
try:
logical_files, error = sql_manager.get_filelist_from_backup(
self._bak_path,
timeout_minutes=self.config["timeouts"]["restore_minutes"]
)
filelist_ms = int((time.time() - filelist_start) * 1000)
JobRepository.update_timing(self.job_id, filelist_ms=filelist_ms)
if error:
raise RuntimeError(error)
files_info = ", ".join([f"{lf.logical_name}({lf.type})" for lf in logical_files])
JobStepRepository.complete(
filelist_step_id,
exit_code=0,
stdout=files_info[:1000]
)
except Exception as e:
JobStepRepository.complete(filelist_step_id, exit_code=1, error=str(e))
raise
# RESTORE DATABASE
restore_step_id = JobStepRepository.create(self.job_id, StepType.RESTORE)
restore_start = time.time()
try:
data_folder = self.config["paths"]["data_sql_folder"]
success, stdout, error = sql_manager.restore_database(
db_name=job.db_name,
backup_path=self._bak_path,
data_folder=data_folder,
logical_files=logical_files,
timeout_minutes=self.config["timeouts"]["restore_minutes"],
dry_run=self.dry_run
)
restore_ms = int((time.time() - restore_start) * 1000)
JobRepository.update_timing(self.job_id, restore_ms=restore_ms)
if not success:
raise RuntimeError(error or "RESTORE falló sin mensaje de error")
JobStepRepository.complete(
restore_step_id,
exit_code=0,
stdout=stdout[:1000] if stdout else None
)
except Exception as e:
JobStepRepository.complete(restore_step_id, exit_code=1, error=str(e))
raise
def _cleanup(self, job):
"""Limpia archivos temporales y mueve el ZIP."""
JobRepository.update_status(self.job_id, JobStatus.CLEANING)
self.signals.job_progress.emit(self.job_id, JobStatus.CLEANING)
step_id = JobStepRepository.create(self.job_id, StepType.CLEANUP)
try:
# Eliminar carpeta de extracción
if self._extract_dir and Path(self._extract_dir).exists():
shutil.rmtree(self._extract_dir)
app_logger.info(f"Carpeta de extracción eliminada: {self._extract_dir}")
# Mover ZIP a Processed
processed_folder = Path(self.config["paths"]["processed_folder"])
date_folder = processed_folder / datetime.now().strftime("%Y-%m-%d")
date_folder.mkdir(parents=True, exist_ok=True)
source_path = Path(job.source_path)
dest_path = date_folder / source_path.name
# Si es multipart, mover todas las partes
if SevenZipExtractor.is_multipart(str(source_path)):
# Buscar todas las partes
base_name = source_path.stem.split('.zip')[0]
parts = list(source_path.parent.glob(f"{base_name}.zip.*"))
for part in parts:
part_dest = date_folder / part.name
shutil.move(str(part), str(part_dest))
app_logger.info(f"Movido: {part.name} -> {part_dest}")
else:
shutil.move(str(source_path), str(dest_path))
app_logger.info(f"Movido: {source_path.name} -> {dest_path}")
JobStepRepository.complete(step_id, exit_code=0)
except Exception as e:
# No fallar el job por errores de limpieza
app_logger.warning(f"Error en limpieza (no crítico): {e}")
JobStepRepository.complete(step_id, exit_code=1, error=str(e))