auctiora/scripts/sync-production-data.sh

#!/bin/bash
#
# Sync Production Data to Local
#
# This script copies the production SQLite database and images from the remote
# server (athena.lan) to your local development environment.
#
# Usage:
#   ./scripts/sync-production-data.sh [--db-only|--images-only|--all]
#
# Options:
#   --db-only      Only sync the database (default)
#   --images-only  Only sync the images
#   --all          Sync both database and images
#   --help         Show this help message
#

set -e  # Exit on error

# Configuration
REMOTE_HOST="tour@athena.lan"
REMOTE_VOLUME="shared-auction-data"
LOCAL_DB_PATH="c:/mnt/okcomputer/output/cache.db"
LOCAL_IMAGES_PATH="c:/mnt/okcomputer/images"
REMOTE_TMP="/tmp"

# Colors for output
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color

# Parse arguments
SYNC_MODE="db"  # Default: database only

case "${1:-}" in
    --db-only)
        SYNC_MODE="db"
        ;;
    --images-only)
        SYNC_MODE="images"
        ;;
    --all)
        SYNC_MODE="all"
        ;;
    --help|-h)
        grep '^#' "$0" | sed 's/^# \?//'
        exit 0
        ;;
    "")
        SYNC_MODE="db"
        ;;
    *)
        echo -e "${RED}Error: Unknown option '$1'${NC}"
        echo "Use --help for usage information"
        exit 1
        ;;
esac

echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║     Production Data Sync - Auctiora Monitor           ║${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
echo ""

# Function to sync database
sync_database() {
    echo -e "${YELLOW}[1/3] Copying database from Docker volume to /tmp...${NC}"
    ssh ${REMOTE_HOST} "docker run --rm -v ${REMOTE_VOLUME}:/data -v ${REMOTE_TMP}:${REMOTE_TMP} alpine cp /data/cache.db ${REMOTE_TMP}/cache.db"

    echo -e "${YELLOW}[2/3] Downloading database from remote server...${NC}"
    # Create backup and remove old local database
    if [ -f "${LOCAL_DB_PATH}" ]; then
        BACKUP_PATH="${LOCAL_DB_PATH}.backup-$(date +%Y%m%d-%H%M%S)"
        echo -e "${BLUE}   Backing up existing database to: ${BACKUP_PATH}${NC}"
        cp "${LOCAL_DB_PATH}" "${BACKUP_PATH}"

        echo -e "${BLUE}   Removing old local database...${NC}"
        rm -f "${LOCAL_DB_PATH}"
    fi

    # Download new database
    scp ${REMOTE_HOST}:${REMOTE_TMP}/cache.db "${LOCAL_DB_PATH}"

    echo -e "${YELLOW}[3/3] Cleaning up remote /tmp...${NC}"
    ssh ${REMOTE_HOST} "rm -f ${REMOTE_TMP}/cache.db"

    # Show database info
    DB_SIZE=$(du -h "${LOCAL_DB_PATH}" | cut -f1)
    echo -e "${GREEN}✓ Database synced successfully (${DB_SIZE})${NC}"

    # Show table counts
    echo -e "${BLUE}   Database statistics:${NC}"
    sqlite3 "${LOCAL_DB_PATH}" <<EOF
.mode box
SELECT
    'auctions' as table_name, COUNT(*) as count FROM auctions
UNION ALL
SELECT 'lots', COUNT(*) FROM lots
UNION ALL
SELECT 'images', COUNT(*) FROM images
UNION ALL
SELECT 'cache', COUNT(*) FROM cache;
EOF

    # Show data quality report
    echo -e "${BLUE}   Data quality:${NC}"
    sqlite3 "${LOCAL_DB_PATH}" <<EOF
.mode box
SELECT
    'Valid lots' as metric,
    COUNT(*) as count,
    ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%' as percentage
FROM lots
WHERE auction_id IS NOT NULL AND auction_id != ''
UNION ALL
SELECT
    'Invalid lots (missing auction_id)',
    COUNT(*),
    ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
FROM lots
WHERE auction_id IS NULL OR auction_id = ''
UNION ALL
SELECT
    'Lots with intelligence fields',
    COUNT(*),
    ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
FROM lots
WHERE followers_count IS NOT NULL OR estimated_min IS NOT NULL;
EOF
}

# Function to sync images
sync_images() {
    echo -e "${YELLOW}[1/4] Getting image directory structure from Docker volume...${NC}"

    # Create local images directory if it doesn't exist
    mkdir -p "${LOCAL_IMAGES_PATH}"

    echo -e "${YELLOW}[2/4] Copying images from Docker volume to /tmp...${NC}"
    # Copy entire images directory from volume to /tmp
    ssh ${REMOTE_HOST} "docker run --rm -v ${REMOTE_VOLUME}:/data -v ${REMOTE_TMP}:${REMOTE_TMP} alpine sh -c 'mkdir -p ${REMOTE_TMP}/auction-images && cp -r /data/images/* ${REMOTE_TMP}/auction-images/ 2>/dev/null || true'"

    echo -e "${YELLOW}[3/4] Syncing images to local directory (this may take a while)...${NC}"
    # Use rsync for efficient incremental sync
    if command -v rsync &> /dev/null; then
        echo -e "${BLUE}   Using rsync for efficient transfer...${NC}"
        rsync -avz --progress ${REMOTE_HOST}:${REMOTE_TMP}/auction-images/ "${LOCAL_IMAGES_PATH}/"
    else
        echo -e "${BLUE}   Using scp for transfer (install rsync for faster incremental sync)...${NC}"
        scp -r ${REMOTE_HOST}:${REMOTE_TMP}/auction-images/* "${LOCAL_IMAGES_PATH}/"
    fi

    echo -e "${YELLOW}[4/4] Cleaning up remote /tmp...${NC}"
    ssh ${REMOTE_HOST} "rm -rf ${REMOTE_TMP}/auction-images"

    # Show image stats
    IMAGE_COUNT=$(find "${LOCAL_IMAGES_PATH}" -type f 2>/dev/null | wc -l)
    IMAGE_SIZE=$(du -sh "${LOCAL_IMAGES_PATH}" 2>/dev/null | cut -f1)
    echo -e "${GREEN}✓ Images synced successfully${NC}"
    echo -e "${BLUE}   Total images: ${IMAGE_COUNT}${NC}"
    echo -e "${BLUE}   Total size: ${IMAGE_SIZE}${NC}"
}

# Execute sync based on mode
START_TIME=$(date +%s)

case "$SYNC_MODE" in
    db)
        echo -e "${BLUE}Mode: Database only${NC}"
        echo ""
        sync_database
        ;;
    images)
        echo -e "${BLUE}Mode: Images only${NC}"
        echo ""
        sync_images
        ;;
    all)
        echo -e "${BLUE}Mode: Database + Images${NC}"
        echo ""
        sync_database
        echo ""
        sync_images
        ;;
esac

END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))

echo ""
echo -e "${GREEN}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║  Sync completed successfully in ${DURATION} seconds               ║${NC}"
echo -e "${GREEN}╚════════════════════════════════════════════════════════╝${NC}"
echo ""
echo -e "${BLUE}Next steps:${NC}"
echo -e "  1. Verify data: sqlite3 ${LOCAL_DB_PATH} 'SELECT COUNT(*) FROM lots;'"
echo -e "  2. Start monitor: mvn quarkus:dev"
echo -e "  3. Open dashboard: http://localhost:8080"
echo ""