gogo

2025-12-07 12:32:39 +01:00
parent 07d58cf59c
commit 1b336c49ba
5 changed files with 266 additions and 7 deletions
--- a/AUTOSTART_SETUP.md
+++ b/AUTOSTART_SETUP.md
@@ -0,0 +1,120 @@
+# Auto-Start Setup Guide
+
+The monitor doesn't run automatically yet. Choose your setup based on your server OS:
+
+---
+
+## Linux Server (Systemd Service) ⭐ RECOMMENDED
+
+**Install:**
+```bash
+cd /home/tour/scaev
+chmod +x install_service.sh
+./install_service.sh
+```
+
+**The service will:**
+- ✅ Start automatically on server boot
+- ✅ Restart automatically if it crashes
+- ✅ Log to `~/scaev/logs/monitor.log`
+- ✅ Poll every 30 minutes
+
+**Management commands:**
+```bash
+sudo systemctl status scaev-monitor     # Check if running
+sudo systemctl stop scaev-monitor       # Stop
+sudo systemctl start scaev-monitor      # Start
+sudo systemctl restart scaev-monitor    # Restart
+journalctl -u scaev-monitor -f          # Live logs
+tail -f ~/scaev/logs/monitor.log        # Monitor log file
+```
+
+---
+
+## Windows (Task Scheduler)
+
+**Install (Run as Administrator):**
+```powershell
+cd C:\vibe\scaev
+.\setup_windows_task.ps1
+```
+
+**The task will:**
+- ✅ Start automatically on Windows boot
+- ✅ Restart automatically if it crashes (up to 3 times)
+- ✅ Run as SYSTEM user
+- ✅ Poll every 30 minutes
+
+**Management:**
+1. Open Task Scheduler (`taskschd.msc`)
+2. Find `ScaevAuctionMonitor` in Task Scheduler Library
+3. Right-click to Run/Stop/Disable
+
+**Or via PowerShell:**
+```powershell
+Start-ScheduledTask -TaskName "ScaevAuctionMonitor"
+Stop-ScheduledTask -TaskName "ScaevAuctionMonitor"
+Get-ScheduledTask -TaskName "ScaevAuctionMonitor" | Get-ScheduledTaskInfo
+```
+
+---
+
+## Alternative: Cron Job (Linux)
+
+**For simpler setup without systemd:**
+
+```bash
+# Edit crontab
+crontab -e
+
+# Add this line (runs on boot and restarts every hour if not running)
+@reboot cd /home/tour/scaev && python3 src/monitor.py 30 >> logs/monitor.log 2>&1
+0 * * * * pgrep -f "monitor.py" || (cd /home/tour/scaev && python3 src/monitor.py 30 >> logs/monitor.log 2>&1 &)
+```
+
+---
+
+## Verify It's Working
+
+**Check process is running:**
+```bash
+# Linux
+ps aux | grep monitor.py
+
+# Windows
+tasklist | findstr python
+```
+
+**Check logs:**
+```bash
+# Linux
+tail -f ~/scaev/logs/monitor.log
+
+# Windows
+# Check Task Scheduler history
+```
+
+**Check database is updating:**
+```bash
+# Last modified time should update every 30 minutes
+ls -lh C:/mnt/okcomputer/output/cache.db
+```
+
+---
+
+## Troubleshooting
+
+**Service won't start:**
+1. Check Python path is correct in service file
+2. Check working directory exists
+3. Check user permissions
+4. View error logs: `journalctl -u scaev-monitor -n 50`
+
+**Monitor stops after a while:**
+- Check disk space for logs
+- Check rate limiting isn't blocking requests
+- Increase RestartSec in service file
+
+**Database locked errors:**
+- Ensure only one monitor instance is running
+- Add timeout to SQLite connections in config
--- a/install_service.sh
+++ b/install_service.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# Install scaev-monitor as a systemd service (Linux server)
+
+echo "Installing Scaev Monitor Service..."
+
+# Create logs directory
+mkdir -p ~/scaev/logs
+
+# Copy service file to systemd
+sudo cp scaev-monitor.service /etc/systemd/system/
+
+# Reload systemd
+sudo systemctl daemon-reload
+
+# Enable service (start on boot)
+sudo systemctl enable scaev-monitor.service
+
+# Start service now
+sudo systemctl start scaev-monitor.service
+
+# Show status
+sudo systemctl status scaev-monitor.service
+
+echo ""
+echo "Service installed successfully!"
+echo ""
+echo "Useful commands:"
+echo "  sudo systemctl status scaev-monitor    # Check status"
+echo "  sudo systemctl stop scaev-monitor      # Stop service"
+echo "  sudo systemctl start scaev-monitor     # Start service"
+echo "  sudo systemctl restart scaev-monitor   # Restart service"
+echo "  journalctl -u scaev-monitor -f         # View live logs"
+echo "  tail -f ~/scaev/logs/monitor.log       # View monitor log"
--- a/scaev-monitor.service
+++ b/scaev-monitor.service
@@ -0,0 +1,19 @@
+[Unit]
+Description=Scaev Auction Monitor - Continuous scraping service
+After=network.target
+
+[Service]
+Type=simple
+User=tour
+WorkingDirectory=/home/tour/scaev
+ExecStart=/usr/bin/python3 /home/tour/scaev/src/monitor.py 30
+Restart=always
+RestartSec=60
+StandardOutput=append:/home/tour/scaev/logs/monitor.log
+StandardError=append:/home/tour/scaev/logs/monitor.error.log
+
+# Environment
+Environment="PYTHONUNBUFFERED=1"
+
+[Install]
+WantedBy=multi-user.target
--- a/setup_windows_task.ps1
+++ b/setup_windows_task.ps1
@@ -0,0 +1,47 @@
+# PowerShell script to create Windows Task Scheduler job for Scaev Monitor
+# Run as Administrator
+
+$TaskName = "ScaevAuctionMonitor"
+$ScriptPath = "C:\vibe\scaev\src\monitor.py"
+$PythonPath = "python3"  # Adjust if needed
+$WorkingDir = "C:\vibe\scaev"
+
+# Create the action (run Python script)
+$Action = New-ScheduledTaskAction -Execute $PythonPath `
+    -Argument "$ScriptPath 30" `
+    -WorkingDirectory $WorkingDir
+
+# Trigger: On system startup
+$TriggerStartup = New-ScheduledTaskTrigger -AtStartup
+
+# Settings
+$Settings = New-ScheduledTaskSettingsSet `
+    -AllowStartIfOnBatteries `
+    -DontStopIfGoingOnBatteries `
+    -StartWhenAvailable `
+    -RestartCount 3 `
+    -RestartInterval (New-TimeSpan -Minutes 5)
+
+# Principal: Run with highest privileges
+$Principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest
+
+# Register the task
+Register-ScheduledTask `
+    -TaskName $TaskName `
+    -Action $Action `
+    -Trigger $TriggerStartup `
+    -Settings $Settings `
+    -Principal $Principal `
+    -Description "Scaev auction monitor - polls for new auctions every 30 minutes" `
+    -Force
+
+Write-Host "`nTask '$TaskName' created successfully!" -ForegroundColor Green
+Write-Host "`nTo manage the task:"
+Write-Host "  1. Open Task Scheduler (taskschd.msc)"
+Write-Host "  2. Find 'ScaevAuctionMonitor' in Task Scheduler Library"
+Write-Host "  3. Right-click to Run, Stop, or Disable"
+Write-Host "`nOr use PowerShell commands:"
+Write-Host "  Start-ScheduledTask -TaskName '$TaskName'"
+Write-Host "  Stop-ScheduledTask -TaskName '$TaskName'"
+Write-Host "  Disable-ScheduledTask -TaskName '$TaskName'"
+Write-Host "  Get-ScheduledTask -TaskName '$TaskName' | Get-ScheduledTaskInfo"
--- a/src/cache.py
+++ b/src/cache.py
@@ -74,6 +74,26 @@ class CacheManager:
                )
            """)

+            # Add new columns to auctions table if they don't exist
+            cursor = conn.execute("PRAGMA table_info(auctions)")
+            auction_columns = {row[1] for row in cursor.fetchall()}
+
+            if 'city' not in auction_columns:
+                conn.execute("ALTER TABLE auctions ADD COLUMN city TEXT")
+            if 'country' not in auction_columns:
+                conn.execute("ALTER TABLE auctions ADD COLUMN country TEXT")
+            if 'type' not in auction_columns:
+                conn.execute("ALTER TABLE auctions ADD COLUMN type TEXT")
+            if 'lot_count' not in auction_columns:
+                conn.execute("ALTER TABLE auctions ADD COLUMN lot_count INTEGER DEFAULT 0")
+            if 'closing_time' not in auction_columns:
+                conn.execute("ALTER TABLE auctions ADD COLUMN closing_time TEXT")
+            if 'discovered_at' not in auction_columns:
+                conn.execute("ALTER TABLE auctions ADD COLUMN discovered_at INTEGER")
+
+            # Add index for country filtering
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)")
+
            # Add new columns to lots table if they don't exist
            cursor = conn.execute("PRAGMA table_info(lots)")
            columns = {row[1] for row in cursor.fetchall()}
@@ -126,6 +146,8 @@ class CacheManager:
                conn.execute("ALTER TABLE lots ADD COLUMN lot_condition TEXT")
            if 'appearance' not in columns:
                conn.execute("ALTER TABLE lots ADD COLUMN appearance TEXT")
+            if 'scraped_at_timestamp' not in columns:
+                conn.execute("ALTER TABLE lots ADD COLUMN scraped_at_timestamp INTEGER")

            # Create bid_history table
            conn.execute("""
@@ -224,19 +246,36 @@ class CacheManager:

    def save_auction(self, auction_data: Dict):
        """Save auction data to database"""
+        # Parse location into city and country
+        location = auction_data.get('location', '')
+        city = None
+        country = None
+        if location:
+            parts = [p.strip() for p in location.split(',')]
+            if len(parts) >= 2:
+                city = parts[0]
+                country = parts[-1]
+
        with sqlite3.connect(self.db_path) as conn:
            conn.execute("""
                INSERT OR REPLACE INTO auctions
-                (auction_id, url, title, location, lots_count, first_lot_closing_time, scraped_at)
-                VALUES (?, ?, ?, ?, ?, ?, ?)
+                (auction_id, url, title, location, lots_count, first_lot_closing_time, scraped_at,
+                 city, country, type, lot_count, closing_time, discovered_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                auction_data['auction_id'],
                auction_data['url'],
                auction_data['title'],
-                auction_data['location'],
+                location,
                auction_data.get('lots_count', 0),
                auction_data.get('first_lot_closing_time', ''),
-                auction_data['scraped_at']
+                auction_data['scraped_at'],
+                city,
+                country,
+                'online',  # Troostwijk is online platform
+                auction_data.get('lots_count', 0),  # Duplicate to lot_count for consistency
+                auction_data.get('first_lot_closing_time', ''),  # Use first_lot_closing_time as closing_time
+                int(time.time())
            ))
            conn.commit()

@@ -252,8 +291,8 @@ class CacheManager:
                 year_manufactured, condition_score, condition_description,
                 serial_number, manufacturer, damage_description,
                 followers_count, estimated_min_price, estimated_max_price, lot_condition, appearance,
-                 scraped_at)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                 scraped_at, scraped_at_timestamp)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                lot_data['lot_id'],
                lot_data.get('auction_id', ''),
@@ -288,7 +327,8 @@ class CacheManager:
                lot_data.get('estimated_max_price'),
                lot_data.get('lot_condition', ''),
                lot_data.get('appearance', ''),
-                lot_data['scraped_at']
+                lot_data['scraped_at'],
+                int(time.time())
            ))
            conn.commit()