From 1b336c49ba93b6d1120b05089937c5e384fb0eae Mon Sep 17 00:00:00 2001 From: Tour Date: Sun, 7 Dec 2025 12:32:39 +0100 Subject: [PATCH] gogo --- AUTOSTART_SETUP.md | 120 +++++++++++++++++++++++++++++++++++++++++ install_service.sh | 33 ++++++++++++ scaev-monitor.service | 19 +++++++ setup_windows_task.ps1 | 47 ++++++++++++++++ src/cache.py | 54 ++++++++++++++++--- 5 files changed, 266 insertions(+), 7 deletions(-) create mode 100644 AUTOSTART_SETUP.md create mode 100644 install_service.sh create mode 100644 scaev-monitor.service create mode 100644 setup_windows_task.ps1 diff --git a/AUTOSTART_SETUP.md b/AUTOSTART_SETUP.md new file mode 100644 index 0000000..d68fe5c --- /dev/null +++ b/AUTOSTART_SETUP.md @@ -0,0 +1,120 @@ +# Auto-Start Setup Guide + +The monitor doesn't run automatically yet. Choose your setup based on your server OS: + +--- + +## Linux Server (Systemd Service) ⭐ RECOMMENDED + +**Install:** +```bash +cd /home/tour/scaev +chmod +x install_service.sh +./install_service.sh +``` + +**The service will:** +- ✅ Start automatically on server boot +- ✅ Restart automatically if it crashes +- ✅ Log to `~/scaev/logs/monitor.log` +- ✅ Poll every 30 minutes + +**Management commands:** +```bash +sudo systemctl status scaev-monitor # Check if running +sudo systemctl stop scaev-monitor # Stop +sudo systemctl start scaev-monitor # Start +sudo systemctl restart scaev-monitor # Restart +journalctl -u scaev-monitor -f # Live logs +tail -f ~/scaev/logs/monitor.log # Monitor log file +``` + +--- + +## Windows (Task Scheduler) + +**Install (Run as Administrator):** +```powershell +cd C:\vibe\scaev +.\setup_windows_task.ps1 +``` + +**The task will:** +- ✅ Start automatically on Windows boot +- ✅ Restart automatically if it crashes (up to 3 times) +- ✅ Run as SYSTEM user +- ✅ Poll every 30 minutes + +**Management:** +1. Open Task Scheduler (`taskschd.msc`) +2. Find `ScaevAuctionMonitor` in Task Scheduler Library +3. Right-click to Run/Stop/Disable + +**Or via PowerShell:** +```powershell +Start-ScheduledTask -TaskName "ScaevAuctionMonitor" +Stop-ScheduledTask -TaskName "ScaevAuctionMonitor" +Get-ScheduledTask -TaskName "ScaevAuctionMonitor" | Get-ScheduledTaskInfo +``` + +--- + +## Alternative: Cron Job (Linux) + +**For simpler setup without systemd:** + +```bash +# Edit crontab +crontab -e + +# Add this line (runs on boot and restarts every hour if not running) +@reboot cd /home/tour/scaev && python3 src/monitor.py 30 >> logs/monitor.log 2>&1 +0 * * * * pgrep -f "monitor.py" || (cd /home/tour/scaev && python3 src/monitor.py 30 >> logs/monitor.log 2>&1 &) +``` + +--- + +## Verify It's Working + +**Check process is running:** +```bash +# Linux +ps aux | grep monitor.py + +# Windows +tasklist | findstr python +``` + +**Check logs:** +```bash +# Linux +tail -f ~/scaev/logs/monitor.log + +# Windows +# Check Task Scheduler history +``` + +**Check database is updating:** +```bash +# Last modified time should update every 30 minutes +ls -lh C:/mnt/okcomputer/output/cache.db +``` + +--- + +## Troubleshooting + +**Service won't start:** +1. Check Python path is correct in service file +2. Check working directory exists +3. Check user permissions +4. View error logs: `journalctl -u scaev-monitor -n 50` + +**Monitor stops after a while:** +- Check disk space for logs +- Check rate limiting isn't blocking requests +- Increase RestartSec in service file + +**Database locked errors:** +- Ensure only one monitor instance is running +- Add timeout to SQLite connections in config diff --git a/install_service.sh b/install_service.sh new file mode 100644 index 0000000..b615cef --- /dev/null +++ b/install_service.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Install scaev-monitor as a systemd service (Linux server) + +echo "Installing Scaev Monitor Service..." + +# Create logs directory +mkdir -p ~/scaev/logs + +# Copy service file to systemd +sudo cp scaev-monitor.service /etc/systemd/system/ + +# Reload systemd +sudo systemctl daemon-reload + +# Enable service (start on boot) +sudo systemctl enable scaev-monitor.service + +# Start service now +sudo systemctl start scaev-monitor.service + +# Show status +sudo systemctl status scaev-monitor.service + +echo "" +echo "Service installed successfully!" +echo "" +echo "Useful commands:" +echo " sudo systemctl status scaev-monitor # Check status" +echo " sudo systemctl stop scaev-monitor # Stop service" +echo " sudo systemctl start scaev-monitor # Start service" +echo " sudo systemctl restart scaev-monitor # Restart service" +echo " journalctl -u scaev-monitor -f # View live logs" +echo " tail -f ~/scaev/logs/monitor.log # View monitor log" diff --git a/scaev-monitor.service b/scaev-monitor.service new file mode 100644 index 0000000..5364daf --- /dev/null +++ b/scaev-monitor.service @@ -0,0 +1,19 @@ +[Unit] +Description=Scaev Auction Monitor - Continuous scraping service +After=network.target + +[Service] +Type=simple +User=tour +WorkingDirectory=/home/tour/scaev +ExecStart=/usr/bin/python3 /home/tour/scaev/src/monitor.py 30 +Restart=always +RestartSec=60 +StandardOutput=append:/home/tour/scaev/logs/monitor.log +StandardError=append:/home/tour/scaev/logs/monitor.error.log + +# Environment +Environment="PYTHONUNBUFFERED=1" + +[Install] +WantedBy=multi-user.target diff --git a/setup_windows_task.ps1 b/setup_windows_task.ps1 new file mode 100644 index 0000000..e30f4a5 --- /dev/null +++ b/setup_windows_task.ps1 @@ -0,0 +1,47 @@ +# PowerShell script to create Windows Task Scheduler job for Scaev Monitor +# Run as Administrator + +$TaskName = "ScaevAuctionMonitor" +$ScriptPath = "C:\vibe\scaev\src\monitor.py" +$PythonPath = "python3" # Adjust if needed +$WorkingDir = "C:\vibe\scaev" + +# Create the action (run Python script) +$Action = New-ScheduledTaskAction -Execute $PythonPath ` + -Argument "$ScriptPath 30" ` + -WorkingDirectory $WorkingDir + +# Trigger: On system startup +$TriggerStartup = New-ScheduledTaskTrigger -AtStartup + +# Settings +$Settings = New-ScheduledTaskSettingsSet ` + -AllowStartIfOnBatteries ` + -DontStopIfGoingOnBatteries ` + -StartWhenAvailable ` + -RestartCount 3 ` + -RestartInterval (New-TimeSpan -Minutes 5) + +# Principal: Run with highest privileges +$Principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest + +# Register the task +Register-ScheduledTask ` + -TaskName $TaskName ` + -Action $Action ` + -Trigger $TriggerStartup ` + -Settings $Settings ` + -Principal $Principal ` + -Description "Scaev auction monitor - polls for new auctions every 30 minutes" ` + -Force + +Write-Host "`nTask '$TaskName' created successfully!" -ForegroundColor Green +Write-Host "`nTo manage the task:" +Write-Host " 1. Open Task Scheduler (taskschd.msc)" +Write-Host " 2. Find 'ScaevAuctionMonitor' in Task Scheduler Library" +Write-Host " 3. Right-click to Run, Stop, or Disable" +Write-Host "`nOr use PowerShell commands:" +Write-Host " Start-ScheduledTask -TaskName '$TaskName'" +Write-Host " Stop-ScheduledTask -TaskName '$TaskName'" +Write-Host " Disable-ScheduledTask -TaskName '$TaskName'" +Write-Host " Get-ScheduledTask -TaskName '$TaskName' | Get-ScheduledTaskInfo" diff --git a/src/cache.py b/src/cache.py index b0b52c8..169fe74 100644 --- a/src/cache.py +++ b/src/cache.py @@ -74,6 +74,26 @@ class CacheManager: ) """) + # Add new columns to auctions table if they don't exist + cursor = conn.execute("PRAGMA table_info(auctions)") + auction_columns = {row[1] for row in cursor.fetchall()} + + if 'city' not in auction_columns: + conn.execute("ALTER TABLE auctions ADD COLUMN city TEXT") + if 'country' not in auction_columns: + conn.execute("ALTER TABLE auctions ADD COLUMN country TEXT") + if 'type' not in auction_columns: + conn.execute("ALTER TABLE auctions ADD COLUMN type TEXT") + if 'lot_count' not in auction_columns: + conn.execute("ALTER TABLE auctions ADD COLUMN lot_count INTEGER DEFAULT 0") + if 'closing_time' not in auction_columns: + conn.execute("ALTER TABLE auctions ADD COLUMN closing_time TEXT") + if 'discovered_at' not in auction_columns: + conn.execute("ALTER TABLE auctions ADD COLUMN discovered_at INTEGER") + + # Add index for country filtering + conn.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)") + # Add new columns to lots table if they don't exist cursor = conn.execute("PRAGMA table_info(lots)") columns = {row[1] for row in cursor.fetchall()} @@ -126,6 +146,8 @@ class CacheManager: conn.execute("ALTER TABLE lots ADD COLUMN lot_condition TEXT") if 'appearance' not in columns: conn.execute("ALTER TABLE lots ADD COLUMN appearance TEXT") + if 'scraped_at_timestamp' not in columns: + conn.execute("ALTER TABLE lots ADD COLUMN scraped_at_timestamp INTEGER") # Create bid_history table conn.execute(""" @@ -224,19 +246,36 @@ class CacheManager: def save_auction(self, auction_data: Dict): """Save auction data to database""" + # Parse location into city and country + location = auction_data.get('location', '') + city = None + country = None + if location: + parts = [p.strip() for p in location.split(',')] + if len(parts) >= 2: + city = parts[0] + country = parts[-1] + with sqlite3.connect(self.db_path) as conn: conn.execute(""" INSERT OR REPLACE INTO auctions - (auction_id, url, title, location, lots_count, first_lot_closing_time, scraped_at) - VALUES (?, ?, ?, ?, ?, ?, ?) + (auction_id, url, title, location, lots_count, first_lot_closing_time, scraped_at, + city, country, type, lot_count, closing_time, discovered_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( auction_data['auction_id'], auction_data['url'], auction_data['title'], - auction_data['location'], + location, auction_data.get('lots_count', 0), auction_data.get('first_lot_closing_time', ''), - auction_data['scraped_at'] + auction_data['scraped_at'], + city, + country, + 'online', # Troostwijk is online platform + auction_data.get('lots_count', 0), # Duplicate to lot_count for consistency + auction_data.get('first_lot_closing_time', ''), # Use first_lot_closing_time as closing_time + int(time.time()) )) conn.commit() @@ -252,8 +291,8 @@ class CacheManager: year_manufactured, condition_score, condition_description, serial_number, manufacturer, damage_description, followers_count, estimated_min_price, estimated_max_price, lot_condition, appearance, - scraped_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + scraped_at, scraped_at_timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( lot_data['lot_id'], lot_data.get('auction_id', ''), @@ -288,7 +327,8 @@ class CacheManager: lot_data.get('estimated_max_price'), lot_data.get('lot_condition', ''), lot_data.get('appearance', ''), - lot_data['scraped_at'] + lot_data['scraped_at'], + int(time.time()) )) conn.commit()