Initial clean commit
This commit is contained in:
Tour
200
scripts/sync-production-data.sh
Normal file
200
scripts/sync-production-data.sh
Normal file
@@ -0,0 +1,200 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Sync Production Data to Local
|
||||
#
|
||||
# This script copies the production SQLite database and images from the remote
|
||||
# server (athena.lan) to your local development environment.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/sync-production-data.sh [--db-only|--images-only|--all]
|
||||
#
|
||||
# Options:
|
||||
# --db-only Only sync the database (default)
|
||||
# --images-only Only sync the images
|
||||
# --all Sync both database and images
|
||||
# --help Show this help message
|
||||
#
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
# Configuration
|
||||
REMOTE_HOST="tour@athena.lan"
|
||||
REMOTE_VOLUME="shared-auction-data"
|
||||
LOCAL_DB_PATH="c:/mnt/okcomputer/output/cache.db"
|
||||
LOCAL_IMAGES_PATH="c:/mnt/okcomputer/images"
|
||||
REMOTE_TMP="/tmp"
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Parse arguments
|
||||
SYNC_MODE="db" # Default: database only
|
||||
|
||||
case "${1:-}" in
|
||||
--db-only)
|
||||
SYNC_MODE="db"
|
||||
;;
|
||||
--images-only)
|
||||
SYNC_MODE="images"
|
||||
;;
|
||||
--all)
|
||||
SYNC_MODE="all"
|
||||
;;
|
||||
--help|-h)
|
||||
grep '^#' "$0" | sed 's/^# \?//'
|
||||
exit 0
|
||||
;;
|
||||
"")
|
||||
SYNC_MODE="db"
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Error: Unknown option '$1'${NC}"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${BLUE}║ Production Data Sync - Auctiora Monitor ║${NC}"
|
||||
echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
|
||||
# Function to sync database
|
||||
sync_database() {
|
||||
echo -e "${YELLOW}[1/3] Copying database from Docker volume to /tmp...${NC}"
|
||||
ssh ${REMOTE_HOST} "docker run --rm -v ${REMOTE_VOLUME}:/data -v ${REMOTE_TMP}:${REMOTE_TMP} alpine cp /data/cache.db ${REMOTE_TMP}/cache.db"
|
||||
|
||||
echo -e "${YELLOW}[2/3] Downloading database from remote server...${NC}"
|
||||
# Create backup and remove old local database
|
||||
if [ -f "${LOCAL_DB_PATH}" ]; then
|
||||
BACKUP_PATH="${LOCAL_DB_PATH}.backup-$(date +%Y%m%d-%H%M%S)"
|
||||
echo -e "${BLUE} Backing up existing database to: ${BACKUP_PATH}${NC}"
|
||||
cp "${LOCAL_DB_PATH}" "${BACKUP_PATH}"
|
||||
|
||||
echo -e "${BLUE} Removing old local database...${NC}"
|
||||
rm -f "${LOCAL_DB_PATH}"
|
||||
fi
|
||||
|
||||
# Download new database
|
||||
scp ${REMOTE_HOST}:${REMOTE_TMP}/cache.db "${LOCAL_DB_PATH}"
|
||||
|
||||
echo -e "${YELLOW}[3/3] Cleaning up remote /tmp...${NC}"
|
||||
ssh ${REMOTE_HOST} "rm -f ${REMOTE_TMP}/cache.db"
|
||||
|
||||
# Show database info
|
||||
DB_SIZE=$(du -h "${LOCAL_DB_PATH}" | cut -f1)
|
||||
echo -e "${GREEN}✓ Database synced successfully (${DB_SIZE})${NC}"
|
||||
|
||||
# Show table counts
|
||||
echo -e "${BLUE} Database statistics:${NC}"
|
||||
sqlite3 "${LOCAL_DB_PATH}" <<EOF
|
||||
.mode box
|
||||
SELECT
|
||||
'auctions' as table_name, COUNT(*) as count FROM auctions
|
||||
UNION ALL
|
||||
SELECT 'lots', COUNT(*) FROM lots
|
||||
UNION ALL
|
||||
SELECT 'images', COUNT(*) FROM images
|
||||
UNION ALL
|
||||
SELECT 'cache', COUNT(*) FROM cache;
|
||||
EOF
|
||||
|
||||
# Show data quality report
|
||||
echo -e "${BLUE} Data quality:${NC}"
|
||||
sqlite3 "${LOCAL_DB_PATH}" <<EOF
|
||||
.mode box
|
||||
SELECT
|
||||
'Valid lots' as metric,
|
||||
COUNT(*) as count,
|
||||
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%' as percentage
|
||||
FROM lots
|
||||
WHERE auction_id IS NOT NULL AND auction_id != ''
|
||||
UNION ALL
|
||||
SELECT
|
||||
'Invalid lots (missing auction_id)',
|
||||
COUNT(*),
|
||||
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
|
||||
FROM lots
|
||||
WHERE auction_id IS NULL OR auction_id = ''
|
||||
UNION ALL
|
||||
SELECT
|
||||
'Lots with intelligence fields',
|
||||
COUNT(*),
|
||||
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
|
||||
FROM lots
|
||||
WHERE followers_count IS NOT NULL OR estimated_min IS NOT NULL;
|
||||
EOF
|
||||
}
|
||||
|
||||
# Function to sync images
|
||||
sync_images() {
|
||||
echo -e "${YELLOW}[1/4] Getting image directory structure from Docker volume...${NC}"
|
||||
|
||||
# Create local images directory if it doesn't exist
|
||||
mkdir -p "${LOCAL_IMAGES_PATH}"
|
||||
|
||||
echo -e "${YELLOW}[2/4] Copying images from Docker volume to /tmp...${NC}"
|
||||
# Copy entire images directory from volume to /tmp
|
||||
ssh ${REMOTE_HOST} "docker run --rm -v ${REMOTE_VOLUME}:/data -v ${REMOTE_TMP}:${REMOTE_TMP} alpine sh -c 'mkdir -p ${REMOTE_TMP}/auction-images && cp -r /data/images/* ${REMOTE_TMP}/auction-images/ 2>/dev/null || true'"
|
||||
|
||||
echo -e "${YELLOW}[3/4] Syncing images to local directory (this may take a while)...${NC}"
|
||||
# Use rsync for efficient incremental sync
|
||||
if command -v rsync &> /dev/null; then
|
||||
echo -e "${BLUE} Using rsync for efficient transfer...${NC}"
|
||||
rsync -avz --progress ${REMOTE_HOST}:${REMOTE_TMP}/auction-images/ "${LOCAL_IMAGES_PATH}/"
|
||||
else
|
||||
echo -e "${BLUE} Using scp for transfer (install rsync for faster incremental sync)...${NC}"
|
||||
scp -r ${REMOTE_HOST}:${REMOTE_TMP}/auction-images/* "${LOCAL_IMAGES_PATH}/"
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}[4/4] Cleaning up remote /tmp...${NC}"
|
||||
ssh ${REMOTE_HOST} "rm -rf ${REMOTE_TMP}/auction-images"
|
||||
|
||||
# Show image stats
|
||||
IMAGE_COUNT=$(find "${LOCAL_IMAGES_PATH}" -type f 2>/dev/null | wc -l)
|
||||
IMAGE_SIZE=$(du -sh "${LOCAL_IMAGES_PATH}" 2>/dev/null | cut -f1)
|
||||
echo -e "${GREEN}✓ Images synced successfully${NC}"
|
||||
echo -e "${BLUE} Total images: ${IMAGE_COUNT}${NC}"
|
||||
echo -e "${BLUE} Total size: ${IMAGE_SIZE}${NC}"
|
||||
}
|
||||
|
||||
# Execute sync based on mode
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
case "$SYNC_MODE" in
|
||||
db)
|
||||
echo -e "${BLUE}Mode: Database only${NC}"
|
||||
echo ""
|
||||
sync_database
|
||||
;;
|
||||
images)
|
||||
echo -e "${BLUE}Mode: Images only${NC}"
|
||||
echo ""
|
||||
sync_images
|
||||
;;
|
||||
all)
|
||||
echo -e "${BLUE}Mode: Database + Images${NC}"
|
||||
echo ""
|
||||
sync_database
|
||||
echo ""
|
||||
sync_images
|
||||
;;
|
||||
esac
|
||||
|
||||
END_TIME=$(date +%s)
|
||||
DURATION=$((END_TIME - START_TIME))
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}╔════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${GREEN}║ Sync completed successfully in ${DURATION} seconds ║${NC}"
|
||||
echo -e "${GREEN}╚════════════════════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
echo -e "${BLUE}Next steps:${NC}"
|
||||
echo -e " 1. Verify data: sqlite3 ${LOCAL_DB_PATH} 'SELECT COUNT(*) FROM lots;'"
|
||||
echo -e " 2. Start monitor: mvn quarkus:dev"
|
||||
echo -e " 3. Open dashboard: http://localhost:8080"
|
||||
echo ""
|
||||
Reference in New Issue
Block a user