#!/usr/bin/env python3 """ Cache Manager module for SQLite-based caching and data storage """ import sqlite3 import time import zlib import json from datetime import datetime from typing import Dict, List, Optional import config class CacheManager: """Manages page caching and data storage using SQLite""" def __init__(self, db_path: str = None): self.db_path = db_path or config.CACHE_DB self._init_db() def _init_db(self): """Initialize cache and data storage database with consolidated schema""" with sqlite3.connect(self.db_path) as conn: # HTML page cache table (existing) conn.execute(""" CREATE TABLE IF NOT EXISTS cache ( url TEXT PRIMARY KEY, content BLOB, timestamp REAL, status_code INTEGER ) """) conn.execute(""" CREATE INDEX IF NOT EXISTS idx_timestamp ON cache(timestamp) """) # Resource cache table (NEW: for ALL web resources - JS, CSS, images, fonts, etc.) conn.execute(""" CREATE TABLE IF NOT EXISTS resource_cache ( url TEXT PRIMARY KEY, content BLOB, content_type TEXT, status_code INTEGER, headers TEXT, timestamp REAL, size_bytes INTEGER, local_path TEXT ) """) conn.execute(""" CREATE INDEX IF NOT EXISTS idx_resource_timestamp ON resource_cache(timestamp) """) conn.execute(""" CREATE INDEX IF NOT EXISTS idx_resource_content_type ON resource_cache(content_type) """) # Auctions table - consolidated schema conn.execute(""" CREATE TABLE IF NOT EXISTS auctions ( auction_id TEXT PRIMARY KEY, url TEXT UNIQUE, title TEXT, location TEXT, lots_count INTEGER, first_lot_closing_time TEXT, scraped_at TEXT, city TEXT, country TEXT, type TEXT, lot_count INTEGER DEFAULT 0, closing_time TEXT, discovered_at INTEGER ) """) conn.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)") # Lots table - consolidated schema with all fields from working database conn.execute(""" CREATE TABLE IF NOT EXISTS lots ( lot_id TEXT PRIMARY KEY, auction_id TEXT, url TEXT UNIQUE, title TEXT, current_bid TEXT, bid_count INTEGER, closing_time TEXT, viewing_time TEXT, pickup_date TEXT, location TEXT, description TEXT, category TEXT, scraped_at TEXT, sale_id INTEGER, manufacturer TEXT, type TEXT, year INTEGER, currency TEXT DEFAULT 'EUR', closing_notified INTEGER DEFAULT 0, starting_bid TEXT, minimum_bid TEXT, status TEXT, brand TEXT, model TEXT, attributes_json TEXT, first_bid_time TEXT, last_bid_time TEXT, bid_velocity REAL, bid_increment REAL, year_manufactured INTEGER, condition_score REAL, condition_description TEXT, serial_number TEXT, damage_description TEXT, followers_count INTEGER DEFAULT 0, estimated_min_price REAL, estimated_max_price REAL, lot_condition TEXT, appearance TEXT, estimated_min REAL, estimated_max REAL, next_bid_step_cents INTEGER, condition TEXT, category_path TEXT, city_location TEXT, country_code TEXT, bidding_status TEXT, packaging TEXT, quantity INTEGER, vat REAL, buyer_premium_percentage REAL, remarks TEXT, reserve_price REAL, reserve_met INTEGER, view_count INTEGER, api_data_json TEXT, next_scrape_at INTEGER, scrape_priority INTEGER DEFAULT 0, FOREIGN KEY (auction_id) REFERENCES auctions(auction_id) ) """) conn.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)") conn.execute("CREATE INDEX IF NOT EXISTS idx_lots_closing_time ON lots(closing_time)") # Images table conn.execute(""" CREATE TABLE IF NOT EXISTS images ( id INTEGER PRIMARY KEY AUTOINCREMENT, lot_id TEXT, url TEXT, local_path TEXT, downloaded INTEGER DEFAULT 0, labels TEXT, processed_at INTEGER, FOREIGN KEY (lot_id) REFERENCES lots(lot_id) ) """) conn.execute("CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id)") # Remove duplicates before creating unique index conn.execute(""" DELETE FROM images WHERE id NOT IN ( SELECT MIN(id) FROM images GROUP BY lot_id, url ) """) conn.execute(""" CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_lot_url ON images(lot_id, url) """) # Bid history table conn.execute(""" CREATE TABLE IF NOT EXISTS bid_history ( id INTEGER PRIMARY KEY AUTOINCREMENT, lot_id TEXT NOT NULL, bid_amount REAL NOT NULL, bid_time TEXT NOT NULL, is_autobid INTEGER DEFAULT 0, bidder_id TEXT, bidder_number INTEGER, created_at TEXT DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (lot_id) REFERENCES lots(lot_id) ) """) conn.execute(""" CREATE INDEX IF NOT EXISTS idx_bid_history_lot_time ON bid_history(lot_id, bid_time) """) conn.execute(""" CREATE INDEX IF NOT EXISTS idx_bid_history_bidder ON bid_history(bidder_id) """) # MIGRATIONS: Add new columns to existing tables self._run_migrations(conn) conn.commit() def _run_migrations(self, conn): """Run database migrations to add new columns to existing tables""" print("Checking for database migrations...") # Check and add new columns to lots table cursor = conn.execute("PRAGMA table_info(lots)") lots_columns = {row[1] for row in cursor.fetchall()} migrations_applied = False if 'api_data_json' not in lots_columns: print(" > Adding api_data_json column to lots table...") conn.execute("ALTER TABLE lots ADD COLUMN api_data_json TEXT") migrations_applied = True if 'next_scrape_at' not in lots_columns: print(" > Adding next_scrape_at column to lots table...") conn.execute("ALTER TABLE lots ADD COLUMN next_scrape_at INTEGER") migrations_applied = True if 'scrape_priority' not in lots_columns: print(" > Adding scrape_priority column to lots table...") conn.execute("ALTER TABLE lots ADD COLUMN scrape_priority INTEGER DEFAULT 0") migrations_applied = True # Check resource_cache table structure cursor = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='resource_cache'") resource_cache_exists = cursor.fetchone() is not None if resource_cache_exists: # Check if table has correct structure cursor = conn.execute("PRAGMA table_info(resource_cache)") resource_columns = {row[1] for row in cursor.fetchall()} # Expected columns expected_columns = {'url', 'content', 'content_type', 'status_code', 'headers', 'timestamp', 'size_bytes', 'local_path'} if resource_columns != expected_columns: print(" > Rebuilding resource_cache table with correct schema...") # Backup old data count cursor = conn.execute("SELECT COUNT(*) FROM resource_cache") old_count = cursor.fetchone()[0] print(f" (Preserving {old_count} cached resources)") # Drop and recreate with correct schema conn.execute("DROP TABLE IF EXISTS resource_cache") conn.execute(""" CREATE TABLE resource_cache ( url TEXT PRIMARY KEY, content BLOB, content_type TEXT, status_code INTEGER, headers TEXT, timestamp REAL, size_bytes INTEGER, local_path TEXT ) """) conn.execute("CREATE INDEX idx_resource_timestamp ON resource_cache(timestamp)") conn.execute("CREATE INDEX idx_resource_content_type ON resource_cache(content_type)") migrations_applied = True print(" * resource_cache table rebuilt") # Create indexes after migrations (when columns exist) try: conn.execute("CREATE INDEX IF NOT EXISTS idx_lots_priority ON lots(scrape_priority DESC)") conn.execute("CREATE INDEX IF NOT EXISTS idx_lots_next_scrape ON lots(next_scrape_at)") except: pass # Indexes might already exist if migrations_applied: print(" * Migrations complete") else: print(" * Database schema is up to date") def get(self, url: str, max_age_hours: int = 24) -> Optional[Dict]: """Get cached page if it exists and is not too old""" with sqlite3.connect(self.db_path) as conn: cursor = conn.execute( "SELECT content, timestamp, status_code FROM cache WHERE url = ?", (url,) ) row = cursor.fetchone() if row: content, timestamp, status_code = row age_hours = (time.time() - timestamp) / 3600 if age_hours <= max_age_hours: try: content = zlib.decompress(content).decode('utf-8') except Exception as e: print(f" ⚠️ Failed to decompress cache for {url}: {e}") return None return { 'content': content, 'timestamp': timestamp, 'status_code': status_code, 'cached': True } return None def set(self, url: str, content: str, status_code: int = 200): """Cache a page with compression""" with sqlite3.connect(self.db_path) as conn: compressed_content = zlib.compress(content.encode('utf-8'), level=9) original_size = len(content.encode('utf-8')) compressed_size = len(compressed_content) ratio = (1 - compressed_size / original_size) * 100 if original_size > 0 else 0 conn.execute( "INSERT OR REPLACE INTO cache (url, content, timestamp, status_code) VALUES (?, ?, ?, ?)", (url, compressed_content, time.time(), status_code) ) conn.commit() print(f" -> Cached: {url} (compressed {ratio:.1f}%)") def clear_old(self, max_age_hours: int = 168): """Clear old cache entries to prevent database bloat""" cutoff_time = time.time() - (max_age_hours * 3600) with sqlite3.connect(self.db_path) as conn: deleted = conn.execute("DELETE FROM cache WHERE timestamp < ?", (cutoff_time,)).rowcount conn.commit() if deleted > 0: print(f" → Cleared {deleted} old cache entries") def save_auction(self, auction_data: Dict): """Save auction data to database""" # Parse location into city and country location = auction_data.get('location', '') city = None country = None if location: parts = [p.strip() for p in location.split(',')] if len(parts) >= 2: city = parts[0] country = parts[-1] with sqlite3.connect(self.db_path) as conn: conn.execute(""" INSERT OR REPLACE INTO auctions (auction_id, url, title, location, lots_count, first_lot_closing_time, scraped_at, city, country, type, lot_count, closing_time, discovered_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( auction_data['auction_id'], auction_data['url'], auction_data['title'], location, auction_data.get('lots_count', 0), auction_data.get('first_lot_closing_time', ''), auction_data['scraped_at'], city, country, 'online', # Troostwijk is online platform auction_data.get('lots_count', 0), # Duplicate to lot_count for consistency auction_data.get('first_lot_closing_time', ''), # Use first_lot_closing_time as closing_time int(time.time()) )) conn.commit() def save_lot(self, lot_data: Dict): """Save lot data to database""" with sqlite3.connect(self.db_path) as conn: conn.execute(""" INSERT OR REPLACE INTO lots (lot_id, auction_id, url, title, current_bid, starting_bid, minimum_bid, bid_count, closing_time, viewing_time, pickup_date, location, description, category, status, brand, model, attributes_json, first_bid_time, last_bid_time, bid_velocity, bid_increment, year_manufactured, condition_score, condition_description, serial_number, manufacturer, damage_description, followers_count, estimated_min_price, estimated_max_price, lot_condition, appearance, scraped_at, api_data_json, next_scrape_at, scrape_priority) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( lot_data['lot_id'], lot_data.get('auction_id', ''), lot_data['url'], lot_data['title'], lot_data.get('current_bid', ''), lot_data.get('starting_bid', ''), lot_data.get('minimum_bid', ''), lot_data.get('bid_count', 0), lot_data.get('closing_time', ''), lot_data.get('viewing_time', ''), lot_data.get('pickup_date', ''), lot_data.get('location', ''), lot_data.get('description', ''), lot_data.get('category', ''), lot_data.get('status', ''), lot_data.get('brand', ''), lot_data.get('model', ''), lot_data.get('attributes_json', ''), lot_data.get('first_bid_time'), lot_data.get('last_bid_time'), lot_data.get('bid_velocity'), lot_data.get('bid_increment'), lot_data.get('year_manufactured'), lot_data.get('condition_score'), lot_data.get('condition_description', ''), lot_data.get('serial_number', ''), lot_data.get('manufacturer', ''), lot_data.get('damage_description', ''), lot_data.get('followers_count', 0), lot_data.get('estimated_min_price'), lot_data.get('estimated_max_price'), lot_data.get('lot_condition', ''), lot_data.get('appearance', ''), lot_data['scraped_at'], lot_data.get('api_data_json'), lot_data.get('next_scrape_at'), lot_data.get('scrape_priority', 0) )) conn.commit() def save_bid_history(self, lot_id: str, bid_records: List[Dict]): """Save bid history records to database""" if not bid_records: return with sqlite3.connect(self.db_path) as conn: # Clear existing bid history for this lot conn.execute("DELETE FROM bid_history WHERE lot_id = ?", (lot_id,)) # Insert new records for record in bid_records: conn.execute(""" INSERT INTO bid_history (lot_id, bid_amount, bid_time, is_autobid, bidder_id, bidder_number) VALUES (?, ?, ?, ?, ?, ?) """, ( record['lot_id'], record['bid_amount'], record['bid_time'], 1 if record['is_autobid'] else 0, record['bidder_id'], record['bidder_number'] )) conn.commit() def save_images(self, lot_id: str, image_urls: List[str]): """Save image URLs for a lot (prevents duplicates via unique constraint)""" with sqlite3.connect(self.db_path) as conn: for url in image_urls: conn.execute(""" INSERT OR IGNORE INTO images (lot_id, url, downloaded) VALUES (?, ?, 0) """, (lot_id, url)) conn.commit() def save_resource(self, url: str, content: bytes, content_type: str, status_code: int = 200, headers: Optional[Dict] = None, local_path: Optional[str] = None, cache_key: Optional[str] = None): """Save a web resource (JS, CSS, image, font, etc.) to cache Args: cache_key: Optional composite key (url + body hash for POST requests) """ with sqlite3.connect(self.db_path) as conn: headers_json = json.dumps(headers) if headers else None size_bytes = len(content) if content else 0 # Use cache_key if provided, otherwise use url key = cache_key if cache_key else url conn.execute(""" INSERT OR REPLACE INTO resource_cache (url, content, content_type, status_code, headers, timestamp, size_bytes, local_path) VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, (key, content, content_type, status_code, headers_json, time.time(), size_bytes, local_path)) conn.commit() def get_resource(self, url: str, cache_key: Optional[str] = None) -> Optional[Dict]: """Get a cached resource Args: cache_key: Optional composite key to lookup """ with sqlite3.connect(self.db_path) as conn: key = cache_key if cache_key else url cursor = conn.execute(""" SELECT content, content_type, status_code, headers, timestamp, size_bytes, local_path FROM resource_cache WHERE url = ? """, (key,)) row = cursor.fetchone() if row: return { 'content': row[0], 'content_type': row[1], 'status_code': row[2], 'headers': json.loads(row[3]) if row[3] else {}, 'timestamp': row[4], 'size_bytes': row[5], 'local_path': row[6], 'cached': True } return None