initial
This commit is contained in:
mike
50
app/shared/__init__.py
Normal file
50
app/shared/__init__.py
Normal file
@@ -0,0 +1,50 @@
|
||||
"""Shared package exports"""
|
||||
from .models import (
|
||||
FileRecord,
|
||||
OperationRecord,
|
||||
DiskInfo,
|
||||
MigrationPlan,
|
||||
ProcessingStats
|
||||
)
|
||||
from .config import (
|
||||
Config,
|
||||
DatabaseConfig,
|
||||
ProcessingConfig,
|
||||
LoggingConfig,
|
||||
load_config
|
||||
)
|
||||
from .logger import (
|
||||
ProgressLogger,
|
||||
create_logger,
|
||||
format_size,
|
||||
format_rate,
|
||||
format_time
|
||||
)
|
||||
from ._protocols import IDatabase, ILogger
|
||||
|
||||
__all__ = [
|
||||
# Models
|
||||
'FileRecord',
|
||||
'OperationRecord',
|
||||
'DiskInfo',
|
||||
'MigrationPlan',
|
||||
'ProcessingStats',
|
||||
|
||||
# Config
|
||||
'Config',
|
||||
'DatabaseConfig',
|
||||
'ProcessingConfig',
|
||||
'LoggingConfig',
|
||||
'load_config',
|
||||
|
||||
# Logger
|
||||
'ProgressLogger',
|
||||
'create_logger',
|
||||
'format_size',
|
||||
'format_rate',
|
||||
'format_time',
|
||||
|
||||
# Protocols
|
||||
'IDatabase',
|
||||
'ILogger',
|
||||
]
|
||||
67
app/shared/_protocols.py
Normal file
67
app/shared/_protocols.py
Normal file
@@ -0,0 +1,67 @@
|
||||
"""Protocol definitions for the shared package"""
|
||||
from typing import Protocol, Any
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileRecord:
|
||||
"""Core file record with all metadata"""
|
||||
path: Path
|
||||
size: int
|
||||
modified_time: float
|
||||
created_time: float
|
||||
disk_label: str
|
||||
checksum: str | None = None
|
||||
status: str = 'indexed' # indexed, planned, moved, verified
|
||||
category: str | None = None
|
||||
duplicate_of: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OperationRecord:
|
||||
"""Record of a migration operation"""
|
||||
source_path: Path
|
||||
target_path: Path
|
||||
operation_type: str # move, copy, hardlink, symlink
|
||||
status: str = 'pending' # pending, in_progress, completed, failed
|
||||
error: str | None = None
|
||||
executed_at: datetime | None = None
|
||||
verified: bool = False
|
||||
|
||||
|
||||
class IDatabase(Protocol):
|
||||
"""Protocol for database operations"""
|
||||
|
||||
def store_file(self, file_record: FileRecord) -> None:
|
||||
"""Store a file record"""
|
||||
...
|
||||
|
||||
def get_files_by_disk(self, disk: str) -> list[FileRecord]:
|
||||
"""Get all files on a specific disk"""
|
||||
...
|
||||
|
||||
def store_operation(self, operation: OperationRecord) -> None:
|
||||
"""Store an operation record"""
|
||||
...
|
||||
|
||||
def get_pending_operations(self) -> list[OperationRecord]:
|
||||
"""Get all pending operations"""
|
||||
...
|
||||
|
||||
|
||||
class ILogger(Protocol):
|
||||
"""Protocol for logging operations"""
|
||||
|
||||
def info(self, message: str) -> None:
|
||||
...
|
||||
|
||||
def warning(self, message: str) -> None:
|
||||
...
|
||||
|
||||
def error(self, message: str) -> None:
|
||||
...
|
||||
|
||||
def debug(self, message: str) -> None:
|
||||
...
|
||||
110
app/shared/config.py
Normal file
110
app/shared/config.py
Normal file
@@ -0,0 +1,110 @@
|
||||
"""Configuration management for disk reorganizer"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, asdict
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatabaseConfig:
|
||||
"""Database connection configuration"""
|
||||
host: str = '192.168.1.159'
|
||||
port: int = 5432
|
||||
database: str = 'disk_reorganizer_db'
|
||||
user: str = 'disk_reorg_user'
|
||||
password: str = 'heel-goed-wachtwoord'
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary"""
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessingConfig:
|
||||
"""Processing behavior configuration"""
|
||||
batch_size: int = 1000
|
||||
commit_interval: int = 100
|
||||
parallel_workers: int = 4
|
||||
chunk_size: int = 8192
|
||||
hash_algorithm: str = 'sha256'
|
||||
verify_operations: bool = True
|
||||
preserve_timestamps: bool = True
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary"""
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoggingConfig:
|
||||
"""Logging configuration"""
|
||||
level: str = 'INFO'
|
||||
log_file: str = 'disk_reorganizer.log'
|
||||
console_output: bool = True
|
||||
file_output: bool = True
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary"""
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
"""Main configuration container"""
|
||||
database: DatabaseConfig = None
|
||||
processing: ProcessingConfig = None
|
||||
logging: LoggingConfig = None
|
||||
|
||||
def __post_init__(self):
|
||||
"""Initialize nested configs with defaults if not provided"""
|
||||
if self.database is None:
|
||||
self.database = DatabaseConfig()
|
||||
if self.processing is None:
|
||||
self.processing = ProcessingConfig()
|
||||
if self.logging is None:
|
||||
self.logging = LoggingConfig()
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, config_path: Path) -> 'Config':
|
||||
"""Load configuration from JSON file"""
|
||||
if not config_path.exists():
|
||||
return cls()
|
||||
|
||||
with open(config_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
return cls(
|
||||
database=DatabaseConfig(**data.get('database', {})),
|
||||
processing=ProcessingConfig(**data.get('processing', {})),
|
||||
logging=LoggingConfig(**data.get('logging', {}))
|
||||
)
|
||||
|
||||
def to_file(self, config_path: Path) -> None:
|
||||
"""Save configuration to JSON file"""
|
||||
data = {
|
||||
'database': self.database.to_dict(),
|
||||
'processing': self.processing.to_dict(),
|
||||
'logging': self.logging.to_dict()
|
||||
}
|
||||
|
||||
with open(config_path, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary"""
|
||||
return {
|
||||
'database': self.database.to_dict(),
|
||||
'processing': self.processing.to_dict(),
|
||||
'logging': self.logging.to_dict()
|
||||
}
|
||||
|
||||
|
||||
def load_config(config_path: Optional[Path] = None) -> Config:
|
||||
"""Load configuration from file or return default"""
|
||||
if config_path is None:
|
||||
config_path = Path('config.json')
|
||||
|
||||
if config_path.exists():
|
||||
return Config.from_file(config_path)
|
||||
|
||||
return Config()
|
||||
217
app/shared/logger.py
Normal file
217
app/shared/logger.py
Normal file
@@ -0,0 +1,217 @@
|
||||
"""Dynamic progress logger with formatting utilities"""
|
||||
import sys
|
||||
import logging
|
||||
from typing import Optional
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def format_size(bytes_size: int) -> str:
|
||||
"""Format bytes to human-readable size string
|
||||
|
||||
Args:
|
||||
bytes_size: Size in bytes
|
||||
|
||||
Returns:
|
||||
Human-readable size string (e.g., "1.5 GB", "234.5 MB")
|
||||
"""
|
||||
for unit in ['B', 'KB', 'MB', 'GB', 'TB', 'PB']:
|
||||
if bytes_size < 1024.0:
|
||||
return f"{bytes_size:.1f} {unit}"
|
||||
bytes_size /= 1024.0
|
||||
return f"{bytes_size:.1f} EB"
|
||||
|
||||
|
||||
def format_rate(bytes_per_second: float) -> str:
|
||||
"""Format transfer rate to human-readable string
|
||||
|
||||
Args:
|
||||
bytes_per_second: Transfer rate in bytes per second
|
||||
|
||||
Returns:
|
||||
Human-readable rate string (e.g., "125.3 MB/s")
|
||||
"""
|
||||
return f"{format_size(int(bytes_per_second))}/s"
|
||||
|
||||
|
||||
def format_time(seconds: float) -> str:
|
||||
"""Format seconds to human-readable time string
|
||||
|
||||
Args:
|
||||
seconds: Time in seconds
|
||||
|
||||
Returns:
|
||||
Human-readable time string (e.g., "2h 34m 12s", "45m 23s", "12s")
|
||||
"""
|
||||
if seconds < 60:
|
||||
return f"{int(seconds)}s"
|
||||
elif seconds < 3600:
|
||||
minutes = int(seconds // 60)
|
||||
secs = int(seconds % 60)
|
||||
return f"{minutes}m {secs}s"
|
||||
else:
|
||||
hours = int(seconds // 3600)
|
||||
minutes = int((seconds % 3600) // 60)
|
||||
secs = int(seconds % 60)
|
||||
return f"{hours}h {minutes}m {secs}s"
|
||||
|
||||
|
||||
class ProgressLogger:
|
||||
"""Dynamic progress logger with real-time statistics"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str = "defrag",
|
||||
level: int = logging.INFO,
|
||||
log_file: Optional[Path] = None,
|
||||
console_output: bool = True
|
||||
):
|
||||
"""Initialize progress logger
|
||||
|
||||
Args:
|
||||
name: Logger name
|
||||
level: Logging level
|
||||
log_file: Optional log file path
|
||||
console_output: Whether to output to console
|
||||
"""
|
||||
self.logger = logging.getLogger(name)
|
||||
self.logger.setLevel(level)
|
||||
self.logger.handlers.clear()
|
||||
|
||||
# Create formatter
|
||||
formatter = logging.Formatter(
|
||||
'%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
|
||||
# Add console handler
|
||||
if console_output:
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
console_handler.setLevel(level)
|
||||
console_handler.setFormatter(formatter)
|
||||
self.logger.addHandler(console_handler)
|
||||
|
||||
# Add file handler
|
||||
if log_file:
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_handler = logging.FileHandler(log_file)
|
||||
file_handler.setLevel(level)
|
||||
file_handler.setFormatter(formatter)
|
||||
self.logger.addHandler(file_handler)
|
||||
|
||||
self._last_progress_line = ""
|
||||
|
||||
def info(self, message: str) -> None:
|
||||
"""Log info message"""
|
||||
self.logger.info(message)
|
||||
|
||||
def warning(self, message: str) -> None:
|
||||
"""Log warning message"""
|
||||
self.logger.warning(message)
|
||||
|
||||
def error(self, message: str) -> None:
|
||||
"""Log error message"""
|
||||
self.logger.error(message)
|
||||
|
||||
def debug(self, message: str) -> None:
|
||||
"""Log debug message"""
|
||||
self.logger.debug(message)
|
||||
|
||||
def critical(self, message: str) -> None:
|
||||
"""Log critical message"""
|
||||
self.logger.critical(message)
|
||||
|
||||
def progress(
|
||||
self,
|
||||
current: int,
|
||||
total: int,
|
||||
prefix: str = "",
|
||||
suffix: str = "",
|
||||
bytes_processed: Optional[int] = None,
|
||||
elapsed_seconds: Optional[float] = None
|
||||
) -> None:
|
||||
"""Log progress with dynamic statistics
|
||||
|
||||
Args:
|
||||
current: Current progress count
|
||||
total: Total count
|
||||
prefix: Prefix message
|
||||
suffix: Suffix message
|
||||
bytes_processed: Optional bytes processed for rate calculation
|
||||
elapsed_seconds: Optional elapsed time for rate calculation
|
||||
"""
|
||||
if total == 0:
|
||||
percent = 0.0
|
||||
else:
|
||||
percent = (current / total) * 100
|
||||
|
||||
progress_msg = f"{prefix} [{current}/{total}] {percent:.1f}%"
|
||||
|
||||
if bytes_processed is not None and elapsed_seconds is not None and elapsed_seconds > 0:
|
||||
rate = bytes_per_second = bytes_processed / elapsed_seconds
|
||||
progress_msg += f" | {format_size(bytes_processed)} @ {format_rate(rate)}"
|
||||
|
||||
# Estimate time remaining
|
||||
if current > 0:
|
||||
estimated_total_seconds = (elapsed_seconds / current) * total
|
||||
remaining_seconds = estimated_total_seconds - elapsed_seconds
|
||||
progress_msg += f" | ETA: {format_time(remaining_seconds)}"
|
||||
|
||||
if suffix:
|
||||
progress_msg += f" | {suffix}"
|
||||
|
||||
self.info(progress_msg)
|
||||
|
||||
def section(self, title: str) -> None:
|
||||
"""Log section header
|
||||
|
||||
Args:
|
||||
title: Section title
|
||||
"""
|
||||
separator = "=" * 60
|
||||
self.info(separator)
|
||||
self.info(f" {title}")
|
||||
self.info(separator)
|
||||
|
||||
def subsection(self, title: str) -> None:
|
||||
"""Log subsection header
|
||||
|
||||
Args:
|
||||
title: Subsection title
|
||||
"""
|
||||
self.info(f"\n--- {title} ---")
|
||||
|
||||
|
||||
def create_logger(
|
||||
name: str = "defrag",
|
||||
level: str = "INFO",
|
||||
log_file: Optional[Path] = None,
|
||||
console_output: bool = True
|
||||
) -> ProgressLogger:
|
||||
"""Create and configure a progress logger
|
||||
|
||||
Args:
|
||||
name: Logger name
|
||||
level: Logging level as string
|
||||
log_file: Optional log file path
|
||||
console_output: Whether to output to console
|
||||
|
||||
Returns:
|
||||
Configured ProgressLogger instance
|
||||
"""
|
||||
level_map = {
|
||||
'DEBUG': logging.DEBUG,
|
||||
'INFO': logging.INFO,
|
||||
'WARNING': logging.WARNING,
|
||||
'ERROR': logging.ERROR,
|
||||
'CRITICAL': logging.CRITICAL
|
||||
}
|
||||
|
||||
log_level = level_map.get(level.upper(), logging.INFO)
|
||||
|
||||
return ProgressLogger(
|
||||
name=name,
|
||||
level=log_level,
|
||||
log_file=log_file,
|
||||
console_output=console_output
|
||||
)
|
||||
127
app/shared/models.py
Normal file
127
app/shared/models.py
Normal file
@@ -0,0 +1,127 @@
|
||||
"""Data models for the disk reorganizer"""
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileRecord:
|
||||
"""Core file record with all metadata"""
|
||||
path: Path
|
||||
size: int
|
||||
modified_time: float
|
||||
created_time: float
|
||||
disk_label: str
|
||||
checksum: Optional[str] = None
|
||||
status: str = 'indexed' # indexed, planned, moved, verified
|
||||
category: Optional[str] = None
|
||||
duplicate_of: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary for serialization"""
|
||||
return {
|
||||
'path': str(self.path),
|
||||
'size': self.size,
|
||||
'modified_time': self.modified_time,
|
||||
'created_time': self.created_time,
|
||||
'disk_label': self.disk_label,
|
||||
'checksum': self.checksum,
|
||||
'status': self.status,
|
||||
'category': self.category,
|
||||
'duplicate_of': self.duplicate_of
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class OperationRecord:
|
||||
"""Record of a migration operation"""
|
||||
source_path: Path
|
||||
target_path: Path
|
||||
operation_type: str # move, copy, hardlink, symlink
|
||||
size: int = 0
|
||||
status: str = 'pending' # pending, in_progress, completed, failed
|
||||
error: Optional[str] = None
|
||||
executed_at: Optional[datetime] = None
|
||||
verified: bool = False
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary for serialization"""
|
||||
return {
|
||||
'source_path': str(self.source_path),
|
||||
'target_path': str(self.target_path),
|
||||
'operation_type': self.operation_type,
|
||||
'size': self.size,
|
||||
'status': self.status,
|
||||
'error': self.error,
|
||||
'executed_at': self.executed_at.isoformat() if self.executed_at else None,
|
||||
'verified': self.verified
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class DiskInfo:
|
||||
"""Information about a disk/volume"""
|
||||
name: str
|
||||
device: str
|
||||
mount_point: Path
|
||||
total_size: int
|
||||
used_size: int
|
||||
free_size: int
|
||||
fs_type: str
|
||||
|
||||
@property
|
||||
def usage_percent(self) -> float:
|
||||
"""Calculate usage percentage"""
|
||||
if self.total_size == 0:
|
||||
return 0.0
|
||||
return (self.used_size / self.total_size) * 100
|
||||
|
||||
|
||||
@dataclass
|
||||
class MigrationPlan:
|
||||
"""Complete migration plan"""
|
||||
target_disk: str
|
||||
destination_disks: list[str]
|
||||
operations: list[OperationRecord]
|
||||
total_size: int
|
||||
file_count: int
|
||||
created_at: datetime = field(default_factory=datetime.now)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dictionary for serialization"""
|
||||
return {
|
||||
'target_disk': self.target_disk,
|
||||
'destination_disks': self.destination_disks,
|
||||
'operations': [op.to_dict() for op in self.operations],
|
||||
'total_size': self.total_size,
|
||||
'file_count': self.file_count,
|
||||
'created_at': self.created_at.isoformat()
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessingStats:
|
||||
"""Statistics for processing operations"""
|
||||
files_processed: int = 0
|
||||
bytes_processed: int = 0
|
||||
files_succeeded: int = 0
|
||||
files_failed: int = 0
|
||||
start_time: datetime = field(default_factory=datetime.now)
|
||||
|
||||
@property
|
||||
def elapsed_seconds(self) -> float:
|
||||
"""Calculate elapsed time in seconds"""
|
||||
return (datetime.now() - self.start_time).total_seconds()
|
||||
|
||||
@property
|
||||
def files_per_second(self) -> float:
|
||||
"""Calculate processing rate"""
|
||||
elapsed = self.elapsed_seconds
|
||||
return self.files_processed / elapsed if elapsed > 0 else 0.0
|
||||
|
||||
@property
|
||||
def bytes_per_second(self) -> float:
|
||||
"""Calculate throughput"""
|
||||
elapsed = self.elapsed_seconds
|
||||
return self.bytes_processed / elapsed if elapsed > 0 else 0.0
|
||||
Reference in New Issue
Block a user