Fix mock tests

scripts/cleanup-database.sh

@@ -0,0 +1,160 @@
+#!/bin/bash
+#
+# Database Cleanup Utility
+#
+# Removes invalid/old data from the local database
+#
+# Usage:
+#   ./scripts/cleanup-database.sh [--dry-run]
+#
+# Options:
+#   --dry-run   Show what would be deleted without actually deleting
+#
+
+set -e
+
+# Configuration
+LOCAL_DB_PATH="${1:-c:/mnt/okcomputer/cache.db}"
+DRY_RUN=false
+
+# Colors
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+# Parse arguments
+if [ "$1" = "--dry-run" ] || [ "$2" = "--dry-run" ]; then
+    DRY_RUN=true
+fi
+
+if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then
+    grep '^#' "$0" | sed 's/^# \?//'
+    exit 0
+fi
+
+echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
+echo -e "${BLUE}║        Database Cleanup - Auctiora Monitor             ║${NC}"
+echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
+echo ""
+
+if [ ! -f "${LOCAL_DB_PATH}" ]; then
+    echo -e "${RED}Error: Database not found at ${LOCAL_DB_PATH}${NC}"
+    exit 1
+fi
+
+# Backup database before cleanup
+if [ "$DRY_RUN" = false ]; then
+    BACKUP_PATH="${LOCAL_DB_PATH}.backup-before-cleanup-$(date +%Y%m%d-%H%M%S)"
+    echo -e "${YELLOW}Creating backup: ${BACKUP_PATH}${NC}"
+    cp "${LOCAL_DB_PATH}" "${BACKUP_PATH}"
+    echo ""
+fi
+
+# Show current state
+echo -e "${BLUE}Current database state:${NC}"
+sqlite3 "${LOCAL_DB_PATH}" <<EOF
+.mode box
+SELECT
+    'Total lots' as metric,
+    COUNT(*) as count
+FROM lots
+UNION ALL
+SELECT
+    'Valid lots (with auction_id)',
+    COUNT(*)
+FROM lots
+WHERE auction_id IS NOT NULL AND auction_id != ''
+UNION ALL
+SELECT
+    'Invalid lots (missing auction_id)',
+    COUNT(*)
+FROM lots
+WHERE auction_id IS NULL OR auction_id = '';
+EOF
+echo ""
+
+# Count items to be deleted
+echo -e "${YELLOW}Analyzing data to clean up...${NC}"
+
+INVALID_LOTS=$(sqlite3 "${LOCAL_DB_PATH}" "SELECT COUNT(*) FROM lots WHERE auction_id IS NULL OR auction_id = '';")
+ORPHANED_IMAGES=$(sqlite3 "${LOCAL_DB_PATH}" "SELECT COUNT(*) FROM images WHERE lot_id NOT IN (SELECT lot_id FROM lots);")
+
+echo -e "  ${RED}→ Invalid lots to delete: ${INVALID_LOTS}${NC}"
+echo -e "  ${YELLOW}→ Orphaned images to delete: ${ORPHANED_IMAGES}${NC}"
+echo ""
+
+if [ "$INVALID_LOTS" -eq 0 ] && [ "$ORPHANED_IMAGES" -eq 0 ]; then
+    echo -e "${GREEN}✓ Database is clean! No cleanup needed.${NC}"
+    exit 0
+fi
+
+if [ "$DRY_RUN" = true ]; then
+    echo -e "${BLUE}DRY RUN MODE - No changes will be made${NC}"
+    echo ""
+    echo "Would delete:"
+    echo "  - $INVALID_LOTS invalid lots"
+    echo "  - $ORPHANED_IMAGES orphaned images"
+    echo ""
+    echo "Run without --dry-run to perform cleanup"
+    exit 0
+fi
+
+# Confirm cleanup
+echo -e "${YELLOW}This will permanently delete the above records.${NC}"
+read -p "Continue? (y/N) " -n 1 -r
+echo
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "Cleanup cancelled"
+    exit 0
+fi
+
+# Perform cleanup
+echo ""
+echo -e "${YELLOW}Cleaning up database...${NC}"
+
+# Delete invalid lots
+if [ "$INVALID_LOTS" -gt 0 ]; then
+    echo -e "  ${BLUE}[1/2] Deleting invalid lots...${NC}"
+    sqlite3 "${LOCAL_DB_PATH}" "DELETE FROM lots WHERE auction_id IS NULL OR auction_id = '';"
+    echo -e "  ${GREEN}✓ Deleted ${INVALID_LOTS} invalid lots${NC}"
+fi
+
+# Delete orphaned images
+if [ "$ORPHANED_IMAGES" -gt 0 ]; then
+    echo -e "  ${BLUE}[2/2] Deleting orphaned images...${NC}"
+    sqlite3 "${LOCAL_DB_PATH}" "DELETE FROM images WHERE lot_id NOT IN (SELECT lot_id FROM lots);"
+    echo -e "  ${GREEN}✓ Deleted ${ORPHANED_IMAGES} orphaned images${NC}"
+fi
+
+# Vacuum database to reclaim space
+echo -e "  ${BLUE}[3/3] Compacting database...${NC}"
+sqlite3 "${LOCAL_DB_PATH}" "VACUUM;"
+echo -e "  ${GREEN}✓ Database compacted${NC}"
+
+echo ""
+echo -e "${GREEN}╔════════════════════════════════════════════════════════╗${NC}"
+echo -e "${GREEN}║          Cleanup completed successfully                ║${NC}"
+echo -e "${GREEN}╚════════════════════════════════════════════════════════╝${NC}"
+echo ""
+
+# Show final state
+echo -e "${BLUE}Final database state:${NC}"
+sqlite3 "${LOCAL_DB_PATH}" <<EOF
+.mode box
+SELECT
+    'Total lots' as metric,
+    COUNT(*) as count
+FROM lots
+UNION ALL
+SELECT
+    'Total images',
+    COUNT(*)
+FROM images;
+EOF
+
+echo ""
+DB_SIZE=$(du -h "${LOCAL_DB_PATH}" | cut -f1)
+echo -e "${BLUE}Database size: ${DB_SIZE}${NC}"
+echo ""

scripts/sync-production-data.sh

@@ -69,11 +69,14 @@ sync_database() {
     ssh ${REMOTE_HOST} "docker run --rm -v ${REMOTE_VOLUME}:/data -v ${REMOTE_TMP}:${REMOTE_TMP} alpine cp /data/cache.db ${REMOTE_TMP}/cache.db"
 
     echo -e "${YELLOW}[2/3] Downloading database from remote server...${NC}"
-    # Create backup of existing local database
+    # Create backup and remove old local database
     if [ -f "${LOCAL_DB_PATH}" ]; then
         BACKUP_PATH="${LOCAL_DB_PATH}.backup-$(date +%Y%m%d-%H%M%S)"
         echo -e "${BLUE}   Backing up existing database to: ${BACKUP_PATH}${NC}"
         cp "${LOCAL_DB_PATH}" "${BACKUP_PATH}"
+
+        echo -e "${BLUE}   Removing old local database...${NC}"
+        rm -f "${LOCAL_DB_PATH}"
     fi
 
     # Download new database
@@ -98,6 +101,32 @@ UNION ALL
 SELECT 'images', COUNT(*) FROM images
 UNION ALL
 SELECT 'cache', COUNT(*) FROM cache;
+EOF
+
+    # Show data quality report
+    echo -e "${BLUE}   Data quality:${NC}"
+    sqlite3 "${LOCAL_DB_PATH}" <<EOF
+.mode box
+SELECT
+    'Valid lots' as metric,
+    COUNT(*) as count,
+    ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%' as percentage
+FROM lots
+WHERE auction_id IS NOT NULL AND auction_id != ''
+UNION ALL
+SELECT
+    'Invalid lots (missing auction_id)',
+    COUNT(*),
+    ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
+FROM lots
+WHERE auction_id IS NULL OR auction_id = ''
+UNION ALL
+SELECT
+    'Lots with intelligence fields',
+    COUNT(*),
+    ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
+FROM lots
+WHERE followers_count IS NOT NULL OR estimated_min IS NOT NULL;
 EOF
 }