diff --git a/SCHEMA_FIX_INSTRUCTIONS.md b/SCHEMA_FIX_INSTRUCTIONS.md deleted file mode 100644 index 995af61..0000000 --- a/SCHEMA_FIX_INSTRUCTIONS.md +++ /dev/null @@ -1,113 +0,0 @@ -# Database Schema Fix Instructions - -## Problem -The server database was created with `BIGINT` primary keys for `auction_id` and `lot_id`, but the scraper uses TEXT IDs like "A7-40063-2". This causes PRIMARY KEY constraint failures. - -## Root Cause -- Local DB: `auction_id TEXT PRIMARY KEY`, `lot_id TEXT PRIMARY KEY` -- Server DB (old): `auction_id BIGINT PRIMARY KEY`, `lot_id BIGINT PRIMARY KEY` -- Scraper data: Uses TEXT IDs like "A7-40063-2", "A1-34732-49" - -This mismatch prevents the scraper from inserting data, resulting in zero bids showing in the UI. - -## Solution - -### Step 1: Backup Server Database -```bash -ssh tour@192.168.1.149 -cd /mnt/okcomputer/output -cp cache.db cache.db.backup.$(date +%Y%m%d_%H%M%S) -``` - -### Step 2: Upload Fix Script -From your local machine: -```bash -scp C:\vibe\auctiora\fix-schema.sql tour@192.168.1.149:/tmp/ -``` - -### Step 3: Stop the Application -```bash -ssh tour@192.168.1.149 -cd /path/to/docker/compose # wherever docker-compose.yml is located -docker-compose down -``` - -### Step 4: Apply Schema Fix -```bash -ssh tour@192.168.1.149 -cd /mnt/okcomputer/output -sqlite3 cache.db < /tmp/fix-schema.sql -``` - -### Step 5: Verify Schema -```bash -sqlite3 cache.db "PRAGMA table_info(auctions);" -# Should show: auction_id TEXT PRIMARY KEY - -sqlite3 cache.db "PRAGMA table_info(lots);" -# Should show: lot_id TEXT PRIMARY KEY, sale_id TEXT, auction_id TEXT - -# Check data integrity -sqlite3 cache.db "SELECT COUNT(*) FROM auctions;" -sqlite3 cache.db "SELECT COUNT(*) FROM lots;" -``` - -### Step 6: Rebuild Application with Fixed Schema -```bash -# Build new image with fixed DatabaseService.java -cd C:\vibe\auctiora -./mvnw clean package -DskipTests - -# Copy new JAR to server -scp target/quarkus-app/quarkus-run.jar tour@192.168.1.149:/path/to/app/ -``` - -### Step 7: Restart Application -```bash -ssh tour@192.168.1.149 -cd /path/to/docker/compose -docker-compose up -d -``` - -### Step 8: Verify Fix -```bash -# Check logs for successful imports -docker-compose logs -f --tail=100 - -# Should see: -# ✓ Imported XXX auctions -# ✓ Imported XXXXX lots -# No more "PRIMARY KEY constraint failed" errors - -# Check UI at http://192.168.1.149:8081/ -# Should now show: -# - Lots with Bids: > 0 -# - Total Bid Value: > €0.00 -# - Average Bid: > €0.00 -``` - -## Alternative: Quick Fix Without Downtime -If you can't afford downtime, delete the corrupted database and let it rebuild: - -```bash -ssh tour@192.168.1.149 -cd /mnt/okcomputer/output -mv cache.db cache.db.old -docker-compose restart - -# The app will create a new database with correct schema -# Wait for scraper to re-populate data (may take 10-15 minutes) -``` - -## Files Changed -1. `DatabaseService.java` - Fixed schema definitions (auction_id, lot_id, sale_id now TEXT) -2. `fix-schema.sql` - SQL migration script to fix existing database -3. `SCHEMA_FIX_INSTRUCTIONS.md` - This file - -## Testing Locally -Before deploying to server, test locally: -```bash -cd C:\vibe\auctiora -./mvnw clean test -# All tests should pass with new schema -``` diff --git a/fix-schema.sql b/fix-schema.sql deleted file mode 100644 index ccbcf94..0000000 --- a/fix-schema.sql +++ /dev/null @@ -1,128 +0,0 @@ --- Schema Fix Script for Server Database --- This script migrates auction_id and lot_id from BIGINT to TEXT to match scraper format --- The scraper uses TEXT IDs like "A7-40063-2" but DatabaseService.java was creating BIGINT columns - --- Step 1: Backup existing data -CREATE TABLE IF NOT EXISTS auctions_backup AS SELECT * FROM auctions; -CREATE TABLE IF NOT EXISTS lots_backup AS SELECT * FROM lots; -CREATE TABLE IF NOT EXISTS images_backup AS SELECT * FROM images; - --- Step 2: Drop existing tables (CASCADE would drop foreign keys) -DROP TABLE IF EXISTS images; -DROP TABLE IF EXISTS lots; -DROP TABLE IF EXISTS auctions; - --- Step 3: Recreate auctions table with TEXT primary key (matching scraper format) -CREATE TABLE auctions ( - auction_id TEXT PRIMARY KEY, - title TEXT NOT NULL, - location TEXT, - city TEXT, - country TEXT, - url TEXT NOT NULL UNIQUE, - type TEXT, - lot_count INTEGER DEFAULT 0, - closing_time TEXT, - discovered_at INTEGER -); - --- Step 4: Recreate lots table with TEXT primary key (matching scraper format) -CREATE TABLE lots ( - lot_id TEXT PRIMARY KEY, - sale_id TEXT, - auction_id TEXT, - title TEXT, - description TEXT, - manufacturer TEXT, - type TEXT, - year INTEGER, - category TEXT, - current_bid REAL, - currency TEXT DEFAULT 'EUR', - url TEXT UNIQUE, - closing_time TEXT, - closing_notified INTEGER DEFAULT 0, - starting_bid REAL, - minimum_bid REAL, - status TEXT, - brand TEXT, - model TEXT, - attributes_json TEXT, - first_bid_time TEXT, - last_bid_time TEXT, - bid_velocity REAL, - bid_increment REAL, - year_manufactured INTEGER, - condition_score REAL, - condition_description TEXT, - serial_number TEXT, - damage_description TEXT, - followers_count INTEGER DEFAULT 0, - estimated_min_price REAL, - estimated_max_price REAL, - lot_condition TEXT, - appearance TEXT, - estimated_min REAL, - estimated_max REAL, - next_bid_step_cents INTEGER, - condition TEXT, - category_path TEXT, - city_location TEXT, - country_code TEXT, - bidding_status TEXT, - packaging TEXT, - quantity INTEGER, - vat REAL, - buyer_premium_percentage REAL, - remarks TEXT, - reserve_price REAL, - reserve_met INTEGER, - view_count INTEGER, - bid_count INTEGER, - viewing_time TEXT, - pickup_date TEXT, - location TEXT, - scraped_at TEXT, - FOREIGN KEY (auction_id) REFERENCES auctions(auction_id), - FOREIGN KEY (sale_id) REFERENCES auctions(auction_id) -); - --- Step 5: Recreate images table -CREATE TABLE images ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - lot_id TEXT, - url TEXT, - local_path TEXT, - labels TEXT, - processed_at INTEGER, - downloaded INTEGER DEFAULT 0, - FOREIGN KEY (lot_id) REFERENCES lots(lot_id) -); - --- Step 6: Create bid_history table if it doesn't exist -CREATE TABLE IF NOT EXISTS bid_history ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - lot_id TEXT, - bid_amount REAL, - bid_time TEXT, - is_autobid INTEGER DEFAULT 0, - bidder_id TEXT, - bidder_number INTEGER, - FOREIGN KEY (lot_id) REFERENCES lots(lot_id) -); - --- Step 7: Restore data from backup (converting BIGINT to TEXT if needed) -INSERT OR IGNORE INTO auctions SELECT * FROM auctions_backup; -INSERT OR IGNORE INTO lots SELECT * FROM lots_backup; -INSERT OR IGNORE INTO images SELECT * FROM images_backup; - --- Step 8: Create indexes for performance -CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country); -CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id); -CREATE INDEX IF NOT EXISTS idx_lots_auction_id ON lots(auction_id); -CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id); - --- Step 9: Clean up backup tables (optional - comment out if you want to keep backups) --- DROP TABLE auctions_backup; --- DROP TABLE lots_backup; --- DROP TABLE images_backup; diff --git a/pom.xml b/pom.xml index 8b40a4d..483fff1 100644 --- a/pom.xml +++ b/pom.xml @@ -175,6 +175,12 @@ + + io.quarkus + quarkus-junit5-mockito + 3.30.2 + test + org.mockito mockito-core diff --git a/src/main/java/auctiora/AuctionInfo.java b/src/main/java/auctiora/AuctionInfo.java index 06b7cb4..5092c6a 100644 --- a/src/main/java/auctiora/AuctionInfo.java +++ b/src/main/java/auctiora/AuctionInfo.java @@ -7,13 +7,13 @@ import java.time.LocalDateTime; * Data typically populated by the external scraper process */ public record AuctionInfo( - long auctionId, // Unique auction ID (from URL) - String title, // Auction title - String location, // Location (e.g., "Amsterdam, NL") - String city, // City name - String country, // Country code (e.g., "NL") - String url, // Full auction URL - String typePrefix, // Auction type (A1 or A7) - int lotCount, // Number of lots/kavels - LocalDateTime firstLotClosingTime // Closing time if available + long auctionId, // Unique auction ID (from URL) + String title, // Auction title + String location, // Location (e.g., "Amsterdam, NL") + String city, // City name + String country, // Country code (e.g., "NL") + String url, // Full auction URL + String typePrefix, // Auction type (A1 or A7) + int lotCount, // Number of lots/kavels + LocalDateTime firstLotClosingTime // Closing time if available ) { } diff --git a/src/main/java/auctiora/AuctionMonitorHealthCheck.java b/src/main/java/auctiora/AuctionMonitorHealthCheck.java index cd69e70..bec19c3 100644 --- a/src/main/java/auctiora/AuctionMonitorHealthCheck.java +++ b/src/main/java/auctiora/AuctionMonitorHealthCheck.java @@ -11,93 +11,72 @@ import org.eclipse.microprofile.health.Startup; import java.nio.file.Files; import java.nio.file.Paths; -/** - * Health checks for Auction Monitor. - * Provides liveness and readiness probes for Kubernetes/Docker deployment. - */ @ApplicationScoped public class AuctionMonitorHealthCheck { - - @Inject - DatabaseService db; - - /** - * Liveness probe - checks if application is alive - * GET /health/live - */ - @Liveness - public static class LivenessCheck implements HealthCheck { - @Override - public HealthCheckResponse call() { - return HealthCheckResponse.up("Auction Monitor is alive"); - } - } - - /** - * Readiness probe - checks if application is ready to serve requests - * GET /health/ready - */ - @Readiness - @ApplicationScoped - public static class ReadinessCheck implements HealthCheck { - - @Inject - DatabaseService db; - - @Override - public HealthCheckResponse call() { - try { - // Check database connection - var auctions = db.getAllAuctions(); - - // Check database path exists - var dbPath = Paths.get("C:\\mnt\\okcomputer\\output\\cache.db"); - if (!Files.exists(dbPath.getParent())) { - return HealthCheckResponse.down("Database directory does not exist"); - } - - return HealthCheckResponse.named("database") - .up() - .withData("auctions", auctions.size()) - .build(); - - } catch (Exception e) { - return HealthCheckResponse.named("database") - .down() - .withData("error", e.getMessage()) - .build(); + + @Liveness + public static class LivenessCheck + implements HealthCheck { + + @Override public HealthCheckResponse call() { + return HealthCheckResponse.up("Auction Monitor is alive"); + } + } + + @Readiness + @ApplicationScoped + public static class ReadinessCheck + implements HealthCheck { + + @Inject DatabaseService db; + + @Override + public HealthCheckResponse call() { + try { + var auctions = db.getAllAuctions(); + var dbPath = Paths.get("C:\\mnt\\okcomputer\\output\\cache.db"); + if (!Files.exists(dbPath.getParent())) { + return HealthCheckResponse.down("Database directory does not exist"); } - } - } - - /** - * Startup probe - checks if application has started correctly - * GET /health/started - */ - @Startup - @ApplicationScoped - public static class StartupCheck implements HealthCheck { - - @Inject - DatabaseService db; - - @Override - public HealthCheckResponse call() { - try { - // Verify database schema - db.ensureSchema(); - - return HealthCheckResponse.named("startup") - .up() - .withData("message", "Database schema initialized") - .build(); - - } catch (Exception e) { - return HealthCheckResponse.named("startup") - .down() - .withData("error", e.getMessage()) - .build(); - } - } - } + + return HealthCheckResponse.named("database") + .up() + .withData("auctions", auctions.size()) + .build(); + + } catch (Exception e) { + return HealthCheckResponse.named("database") + .down() + .withData("error", e.getMessage()) + .build(); + } + } + } + + @Startup + @ApplicationScoped + public static class StartupCheck + implements HealthCheck { + + @Inject DatabaseService db; + + @Override + public HealthCheckResponse call() { + try { + // Verify database schema + db.ensureSchema(); + + return HealthCheckResponse.named("startup") + .up() + .withData("message", "Database schema initialized") + .build(); + + } catch (Exception e) { + return HealthCheckResponse.named("startup") + .down() + .withData("error", e.getMessage()) + .build(); + } + } + } } diff --git a/src/main/java/auctiora/AuctionMonitorProducer.java b/src/main/java/auctiora/AuctionMonitorProducer.java index 4c06517..64e768d 100644 --- a/src/main/java/auctiora/AuctionMonitorProducer.java +++ b/src/main/java/auctiora/AuctionMonitorProducer.java @@ -19,58 +19,42 @@ import java.sql.SQLException; @Startup @ApplicationScoped public class AuctionMonitorProducer { - - private static final Logger LOG = Logger.getLogger(AuctionMonitorProducer.class); - - @PostConstruct - void init() { - // Load OpenCV native library at startup - try { - nu.pattern.OpenCV.loadLocally(); - LOG.info("✓ OpenCV loaded successfully"); - } catch (Exception e) { - LOG.warn("⚠️ OpenCV not available - image detection will be disabled: " + e.getMessage()); - } - } - - @Produces - @Singleton - public DatabaseService produceDatabaseService( - @ConfigProperty(name = "auction.database.path") String dbPath) throws SQLException { - - LOG.infof("Initializing DatabaseService with path: %s", dbPath); - var db = new DatabaseService(dbPath); - db.ensureSchema(); - return db; - } - - @Produces - @Singleton - public NotificationService produceNotificationService( - @ConfigProperty(name = "auction.notification.config") String config) { - - LOG.infof("Initializing NotificationService with config: %s", config); - return new NotificationService(config); - } - - @Produces - @Singleton - public ObjectDetectionService produceObjectDetectionService( - @ConfigProperty(name = "auction.yolo.config") String cfgPath, - @ConfigProperty(name = "auction.yolo.weights") String weightsPath, - @ConfigProperty(name = "auction.yolo.classes") String classesPath) throws IOException { - - LOG.infof("Initializing ObjectDetectionService"); - return new ObjectDetectionService(cfgPath, weightsPath, classesPath); - } - - @Produces - @Singleton - public ImageProcessingService produceImageProcessingService( - DatabaseService db, - ObjectDetectionService detector) { - - LOG.infof("Initializing ImageProcessingService"); - return new ImageProcessingService(db, detector); - } + + private static final Logger LOG = Logger.getLogger(AuctionMonitorProducer.class); + + @PostConstruct void init() { + try { + nu.pattern.OpenCV.loadLocally(); + LOG.info("✓ OpenCV loaded successfully"); + } catch (Exception e) { + LOG.warn("⚠️ OpenCV not available - image detection will be disabled: " + e.getMessage()); + } + } + + @Produces @Singleton public DatabaseService produceDatabaseService( + @ConfigProperty(name = "auction.database.path") String dbPath) throws SQLException { + var db = new DatabaseService(dbPath); + db.ensureSchema(); + return db; + } + + @Produces @Singleton public NotificationService produceNotificationService( + @ConfigProperty(name = "auction.notification.config") String config) { + + return new NotificationService(config); + } + + @Produces @Singleton public ObjectDetectionService produceObjectDetectionService( + @ConfigProperty(name = "auction.yolo.config") String cfgPath, + @ConfigProperty(name = "auction.yolo.weights") String weightsPath, + @ConfigProperty(name = "auction.yolo.classes") String classesPath) throws IOException { + + return new ObjectDetectionService(cfgPath, weightsPath, classesPath); + } + + @Produces @Singleton public ImageProcessingService produceImageProcessingService( + DatabaseService db, + ObjectDetectionService detector) { + return new ImageProcessingService(db, detector); + } } diff --git a/src/main/java/auctiora/AuctionMonitorResource.java b/src/main/java/auctiora/AuctionMonitorResource.java index 088528c..6cc7087 100644 --- a/src/main/java/auctiora/AuctionMonitorResource.java +++ b/src/main/java/auctiora/AuctionMonitorResource.java @@ -30,21 +30,12 @@ public class AuctionMonitorResource { private static final Logger LOG = Logger.getLogger(AuctionMonitorResource.class); - @Inject - DatabaseService db; + @Inject DatabaseService db; + @Inject QuarkusWorkflowScheduler scheduler; + @Inject NotificationService notifier; + @Inject RateLimitedHttpClient httpClient; + @Inject LotEnrichmentService enrichmentService; - @Inject - QuarkusWorkflowScheduler scheduler; - - @Inject - NotificationService notifier; - - @Inject - RateLimitedHttpClient httpClient; - - @Inject - LotEnrichmentService enrichmentService; - /** * GET /api/monitor/status * Returns current monitoring status @@ -99,33 +90,33 @@ public class AuctionMonitorResource { stats.put("totalImages", db.getImageCount()); // Lot statistics - var activeLots = 0; - var lotsWithBids = 0; - double totalBids = 0; - var hotLots = 0; - var sleeperLots = 0; - var bargainLots = 0; - var lotsClosing1h = 0; - var lotsClosing6h = 0; + var activeLots = 0; + var lotsWithBids = 0; + double totalBids = 0; + var hotLots = 0; + var sleeperLots = 0; + var bargainLots = 0; + var lotsClosing1h = 0; + var lotsClosing6h = 0; double totalBidVelocity = 0; - int velocityCount = 0; - + int velocityCount = 0; + for (var lot : lots) { long minutesLeft = lot.closingTime() != null ? lot.minutesUntilClose() : Long.MAX_VALUE; - + if (lot.closingTime() != null && minutesLeft > 0) { activeLots++; - + // Time-based counts if (minutesLeft < 60) lotsClosing1h++; if (minutesLeft < 360) lotsClosing6h++; } - + if (lot.currentBid() > 0) { lotsWithBids++; totalBids += lot.currentBid(); } - + // Intelligence metrics (require GraphQL enrichment) if (lot.followersCount() != null && lot.followersCount() > 20) { hotLots++; @@ -136,22 +127,22 @@ public class AuctionMonitorResource { if (lot.isBelowEstimate()) { bargainLots++; } - + // Bid velocity if (lot.bidVelocity() != null && lot.bidVelocity() > 0) { totalBidVelocity += lot.bidVelocity(); velocityCount++; } } - + // Calculate bids per hour (average velocity across all lots with velocity data) double bidsPerHour = velocityCount > 0 ? totalBidVelocity / velocityCount : 0; - + stats.put("activeLots", activeLots); stats.put("lotsWithBids", lotsWithBids); stats.put("totalBidValue", String.format("€%.2f", totalBids)); stats.put("averageBid", lotsWithBids > 0 ? String.format("€%.2f", totalBids / lotsWithBids) : "€0.00"); - + // Bidding intelligence stats.put("bidsPerHour", String.format("%.1f", bidsPerHour)); stats.put("hotLots", hotLots); @@ -159,11 +150,11 @@ public class AuctionMonitorResource { stats.put("bargainLots", bargainLots); stats.put("lotsClosing1h", lotsClosing1h); stats.put("lotsClosing6h", lotsClosing6h); - + // Conversion rate double conversionRate = activeLots > 0 ? (lotsWithBids * 100.0 / activeLots) : 0; stats.put("conversionRate", String.format("%.1f%%", conversionRate)); - + return Response.ok(stats).build(); } catch (Exception e) { @@ -184,12 +175,12 @@ public class AuctionMonitorResource { try { var lots = db.getAllLots(); var closingSoon = lots.stream() - .filter(lot -> lot.closingTime() != null) - .filter(lot -> lot.minutesUntilClose() > 0 && lot.minutesUntilClose() <= hours * 60) - .sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose())) - .limit(100) - .toList(); - + .filter(lot -> lot.closingTime() != null) + .filter(lot -> lot.minutesUntilClose() > 0 && lot.minutesUntilClose() <= hours * 60) + .sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose())) + .limit(100) + .toList(); + return Response.ok(closingSoon).build(); } catch (Exception e) { LOG.error("Failed to get closing soon lots", e); @@ -198,7 +189,7 @@ public class AuctionMonitorResource { .build(); } } - + /** * GET /api/monitor/lots/{lotId}/bid-history * Returns bid history for a specific lot @@ -216,7 +207,7 @@ public class AuctionMonitorResource { .build(); } } - + /** * POST /api/monitor/trigger/scraper-import * Manually trigger scraper import workflow @@ -288,7 +279,7 @@ public class AuctionMonitorResource { .build(); } } - + /** * POST /api/monitor/trigger/graphql-enrichment * Manually trigger GraphQL enrichment for all lots or lots closing soon @@ -301,15 +292,15 @@ public class AuctionMonitorResource { if (hours > 0) { enriched = enrichmentService.enrichClosingSoonLots(hours); return Response.ok(Map.of( - "message", "GraphQL enrichment triggered for lots closing within " + hours + " hours", - "enrichedCount", enriched - )).build(); + "message", "GraphQL enrichment triggered for lots closing within " + hours + " hours", + "enrichedCount", enriched + )).build(); } else { enriched = enrichmentService.enrichAllActiveLots(); return Response.ok(Map.of( - "message", "GraphQL enrichment triggered for all lots", - "enrichedCount", enriched - )).build(); + "message", "GraphQL enrichment triggered for all lots", + "enrichedCount", enriched + )).build(); } } catch (Exception e) { LOG.error("Failed to trigger GraphQL enrichment", e); @@ -318,7 +309,7 @@ public class AuctionMonitorResource { .build(); } } - + /** * GET /api/monitor/auctions * Returns list of all auctions @@ -375,7 +366,7 @@ public class AuctionMonitorResource { }) .sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose())) .toList(); - + return Response.ok(closingSoon).build(); } catch (Exception e) { LOG.error("Failed to get closing lots", e); @@ -530,7 +521,7 @@ public class AuctionMonitorResource { public Response getCategoryDistribution() { try { var lots = db.getAllLots(); - + // Category distribution Map distribution = lots.stream() .filter(l -> l.category() != null && !l.category().isEmpty()) @@ -538,34 +529,34 @@ public class AuctionMonitorResource { l -> l.category().length() > 20 ? l.category().substring(0, 20) + "..." : l.category(), Collectors.counting() )); - + // Find top category by count var topCategory = distribution.entrySet().stream() - .max(Map.Entry.comparingByValue()) - .map(Map.Entry::getKey) - .orElse("N/A"); - + .max(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .orElse("N/A"); + // Calculate average bids per category Map avgBidsByCategory = lots.stream() - .filter(l -> l.category() != null && !l.category().isEmpty() && l.currentBid() > 0) - .collect(Collectors.groupingBy( - l -> l.category().length() > 20 ? l.category().substring(0, 20) + "..." : l.category(), - Collectors.averagingDouble(Lot::currentBid) - )); - + .filter(l -> l.category() != null && !l.category().isEmpty() && l.currentBid() > 0) + .collect(Collectors.groupingBy( + l -> l.category().length() > 20 ? l.category().substring(0, 20) + "..." : l.category(), + Collectors.averagingDouble(Lot::currentBid) + )); + double overallAvgBid = lots.stream() - .filter(l -> l.currentBid() > 0) - .mapToDouble(Lot::currentBid) - .average() - .orElse(0.0); - + .filter(l -> l.currentBid() > 0) + .mapToDouble(Lot::currentBid) + .average() + .orElse(0.0); + Map response = new HashMap<>(); response.put("distribution", distribution); response.put("topCategory", topCategory); response.put("categoryCount", distribution.size()); response.put("averageBidOverall", String.format("€%.2f", overallAvgBid)); response.put("avgBidsByCategory", avgBidsByCategory); - + return Response.ok(response).build(); } catch (Exception e) { LOG.error("Failed to get category distribution", e); @@ -663,7 +654,7 @@ public class AuctionMonitorResource { .max(Map.Entry.comparingByValue()) .map(Map.Entry::getKey) .orElse("N/A"); - + if (!"N/A".equals(topCountry)) { insights.add(Map.of( "icon", "fa-globe", @@ -671,7 +662,7 @@ public class AuctionMonitorResource { "description", "Top performing country" )); } - + // Add sleeper lots insight long sleeperCount = lots.stream().filter(Lot::isSleeperLot).count(); if (sleeperCount > 0) { @@ -681,7 +672,7 @@ public class AuctionMonitorResource { "description", "High interest, low bids - opportunity?" )); } - + // Add bargain insight long bargainCount = lots.stream().filter(Lot::isBelowEstimate).count(); if (bargainCount > 5) { @@ -691,7 +682,7 @@ public class AuctionMonitorResource { "description", "Priced below auction house estimates" )); } - + // Add watch/followers insight long highWatchCount = lots.stream() .filter(l -> l.followersCount() != null && l.followersCount() > 20) @@ -703,7 +694,7 @@ public class AuctionMonitorResource { "description", "High follower count, strong competition" )); } - + return Response.ok(insights).build(); } catch (Exception e) { LOG.error("Failed to get insights", e); @@ -725,12 +716,12 @@ public class AuctionMonitorResource { var sleepers = allLots.stream() .filter(Lot::isSleeperLot) .toList(); - + Map response = Map.of( "count", sleepers.size(), "lots", sleepers - ); - + ); + return Response.ok(response).build(); } catch (Exception e) { LOG.error("Failed to get sleeper lots", e); @@ -739,7 +730,7 @@ public class AuctionMonitorResource { .build(); } } - + /** * GET /api/monitor/intelligence/bargains * Returns lots priced below auction house estimates @@ -759,12 +750,12 @@ public class AuctionMonitorResource { return ratioA.compareTo(ratioB); }) .toList(); - + Map response = Map.of( "count", bargains.size(), "lots", bargains - ); - + ); + return Response.ok(response).build(); } catch (Exception e) { LOG.error("Failed to get bargains", e); @@ -773,7 +764,7 @@ public class AuctionMonitorResource { .build(); } } - + /** * GET /api/monitor/intelligence/popular * Returns lots by popularity level @@ -791,13 +782,13 @@ public class AuctionMonitorResource { return followersB.compareTo(followersA); }) .toList(); - + Map response = Map.of( "count", popular.size(), "level", level, "lots", popular - ); - + ); + return Response.ok(response).build(); } catch (Exception e) { LOG.error("Failed to get popular lots", e); @@ -806,7 +797,7 @@ public class AuctionMonitorResource { .build(); } } - + /** * GET /api/monitor/intelligence/price-analysis * Returns price vs estimate analysis @@ -816,28 +807,28 @@ public class AuctionMonitorResource { public Response getPriceAnalysis() { try { var allLots = db.getAllLots(); - + long belowEstimate = allLots.stream().filter(Lot::isBelowEstimate).count(); long aboveEstimate = allLots.stream().filter(Lot::isAboveEstimate).count(); long withEstimates = allLots.stream() .filter(lot -> lot.estimatedMin() != null && lot.estimatedMax() != null) .count(); - + double avgPriceVsEstimate = allLots.stream() .map(Lot::getPriceVsEstimateRatio) .filter(ratio -> ratio != null) .mapToDouble(Double::doubleValue) .average() .orElse(0.0); - + Map response = Map.of( "totalLotsWithEstimates", withEstimates, "belowEstimate", belowEstimate, "aboveEstimate", aboveEstimate, "averagePriceVsEstimatePercent", Math.round(avgPriceVsEstimate), "bargainOpportunities", belowEstimate - ); - + ); + return Response.ok(response).build(); } catch (Exception e) { LOG.error("Failed to get price analysis", e); @@ -846,7 +837,7 @@ public class AuctionMonitorResource { .build(); } } - + /** * GET /api/monitor/lots/{lotId}/intelligence * Returns detailed intelligence for a specific lot @@ -859,13 +850,13 @@ public class AuctionMonitorResource { .filter(l -> l.lotId() == lotId) .findFirst() .orElse(null); - + if (lot == null) { return Response.status(Response.Status.NOT_FOUND) .entity(Map.of("error", "Lot not found")) .build(); } - + Map intelligence = new HashMap<>(); intelligence.put("lotId", lot.lotId()); intelligence.put("followersCount", lot.followersCount()); @@ -882,7 +873,7 @@ public class AuctionMonitorResource { intelligence.put("condition", lot.condition()); intelligence.put("vat", lot.vat()); intelligence.put("buyerPremium", lot.buyerPremiumPercentage()); - + return Response.ok(intelligence).build(); } catch (Exception e) { LOG.error("Failed to get lot intelligence", e); @@ -891,7 +882,7 @@ public class AuctionMonitorResource { .build(); } } - + /** * GET /api/monitor/charts/watch-distribution * Returns follower/watch count distribution @@ -901,14 +892,14 @@ public class AuctionMonitorResource { public Response getWatchDistribution() { try { var lots = db.getAllLots(); - + Map distribution = new HashMap<>(); distribution.put("0 watchers", lots.stream().filter(l -> l.followersCount() == null || l.followersCount() == 0).count()); distribution.put("1-5 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 1 && l.followersCount() <= 5).count()); distribution.put("6-20 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 6 && l.followersCount() <= 20).count()); distribution.put("21-50 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 21 && l.followersCount() <= 50).count()); distribution.put("50+ watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() > 50).count()); - + return Response.ok(distribution).build(); } catch (Exception e) { LOG.error("Failed to get watch distribution", e); @@ -917,14 +908,14 @@ public class AuctionMonitorResource { .build(); } } - + // Helper class for trend data public static class TrendHour { - + public int hour; public int lots; public int bids; - + public TrendHour(int hour, int lots, int bids) { this.hour = hour; this.lots = lots; diff --git a/src/main/java/auctiora/DatabaseService.java b/src/main/java/auctiora/DatabaseService.java index 2fa3672..9412493 100644 --- a/src/main/java/auctiora/DatabaseService.java +++ b/src/main/java/auctiora/DatabaseService.java @@ -20,9 +20,9 @@ import java.util.List; */ @Slf4j public class DatabaseService { - + private final String url; - + DatabaseService(String dbPath) { // Enable WAL mode and busy timeout for concurrent access this.url = "jdbc:sqlite:" + dbPath + "?journal_mode=WAL&busy_timeout=10000"; @@ -38,7 +38,7 @@ public class DatabaseService { stmt.execute("PRAGMA journal_mode=WAL"); stmt.execute("PRAGMA busy_timeout=10000"); stmt.execute("PRAGMA synchronous=NORMAL"); - + // Cache table (for HTTP caching) stmt.execute(""" CREATE TABLE IF NOT EXISTS cache ( @@ -47,7 +47,7 @@ public class DatabaseService { timestamp REAL, status_code INTEGER )"""); - + // Auctions table (populated by external scraper) stmt.execute(""" CREATE TABLE IF NOT EXISTS auctions ( @@ -65,7 +65,7 @@ public class DatabaseService { closing_time TEXT, discovered_at INTEGER )"""); - + // Lots table (populated by external scraper) stmt.execute(""" CREATE TABLE IF NOT EXISTS lots ( @@ -126,7 +126,7 @@ public class DatabaseService { view_count INTEGER, FOREIGN KEY (auction_id) REFERENCES auctions(auction_id) )"""); - + // Images table (populated by external scraper with URLs and local_path) stmt.execute(""" CREATE TABLE IF NOT EXISTS images ( @@ -139,7 +139,7 @@ public class DatabaseService { processed_at INTEGER, FOREIGN KEY (lot_id) REFERENCES lots(lot_id) )"""); - + // Bid history table stmt.execute(""" CREATE TABLE IF NOT EXISTS bid_history ( @@ -153,7 +153,7 @@ public class DatabaseService { created_at TEXT DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (lot_id) REFERENCES lots(lot_id) )"""); - + // Indexes for performance stmt.execute("CREATE INDEX IF NOT EXISTS idx_timestamp ON cache(timestamp)"); stmt.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)"); @@ -164,9 +164,8 @@ public class DatabaseService { stmt.execute("CREATE INDEX IF NOT EXISTS idx_bid_history_bidder ON bid_history(bidder_id)"); } } - - - /** + + /** * Inserts or updates an auction record (typically called by external scraper) * Handles both auction_id conflicts and url uniqueness constraints */ @@ -185,7 +184,7 @@ public class DatabaseService { lot_count = excluded.lot_count, closing_time = excluded.closing_time """; - + try (var conn = DriverManager.getConnection(url)) { try (var ps = conn.prepareStatement(insertSql)) { ps.setLong(1, auction.auctionId()); @@ -205,7 +204,7 @@ public class DatabaseService { if (errMsg.contains("UNIQUE constraint failed: auctions.auction_id") || errMsg.contains("UNIQUE constraint failed: auctions.url") || errMsg.contains("PRIMARY KEY constraint failed")) { - + // Try updating by URL as fallback (most reliable unique identifier) var updateByUrlSql = """ UPDATE auctions SET @@ -229,12 +228,12 @@ public class DatabaseService { ps.setInt(7, auction.lotCount()); ps.setString(8, auction.firstLotClosingTime() != null ? auction.firstLotClosingTime().toString() : null); ps.setString(9, auction.url()); - + int updated = ps.executeUpdate(); if (updated == 0) { // Auction doesn't exist by URL either - this is unexpected log.warn("Could not insert or update auction with url={}, auction_id={} - constraint violation but no existing record found", - auction.url(), auction.auctionId()); + auction.url(), auction.auctionId()); } else { log.debug("Updated existing auction by URL: {}", auction.url()); } @@ -255,12 +254,12 @@ public class DatabaseService { synchronized List getAllAuctions() throws SQLException { List auctions = new ArrayList<>(); var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time FROM auctions"; - + try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) { var rs = stmt.executeQuery(sql); while (rs.next()) { - var closingStr = rs.getString("closing_time"); - LocalDateTime closing = null; + var closingStr = rs.getString("closing_time"); + LocalDateTime closing = null; if (closingStr != null && !closingStr.isBlank()) { try { closing = LocalDateTime.parse(closingStr); @@ -268,7 +267,7 @@ public class DatabaseService { log.debug("Invalid closing_time format for auction {}: {}", rs.getLong("auction_id"), closingStr); } } - + auctions.add(new AuctionInfo( rs.getLong("auction_id"), rs.getString("title"), @@ -292,13 +291,13 @@ public class DatabaseService { List auctions = new ArrayList<>(); var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time " + "FROM auctions WHERE country = ?"; - + try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) { ps.setString(1, countryCode); var rs = ps.executeQuery(); while (rs.next()) { - var closingStr = rs.getString("closing_time"); - LocalDateTime closing = null; + var closingStr = rs.getString("closing_time"); + LocalDateTime closing = null; if (closingStr != null && !closingStr.isBlank()) { try { closing = LocalDateTime.parse(closingStr); @@ -306,7 +305,7 @@ public class DatabaseService { log.debug("Invalid closing_time format for auction {}: {}", rs.getLong("auction_id"), closingStr); } } - + auctions.add(new AuctionInfo( rs.getLong("auction_id"), rs.getString("title"), @@ -343,16 +342,16 @@ public class DatabaseService { closing_time = ? WHERE lot_id = ? """; - + var insertSql = """ INSERT OR IGNORE INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """; - + try (var conn = DriverManager.getConnection(url)) { // Try UPDATE first try (var ps = conn.prepareStatement(updateSql)) { - ps.setLong(1, lot.saleId()); + ps.setString(1, String.valueOf(lot.saleId())); ps.setString(2, lot.title()); ps.setString(3, lot.description()); ps.setString(4, lot.manufacturer()); @@ -363,18 +362,18 @@ public class DatabaseService { ps.setString(9, lot.currency()); ps.setString(10, lot.url()); ps.setString(11, lot.closingTime() != null ? lot.closingTime().toString() : null); - ps.setLong(12, lot.lotId()); - + ps.setString(12, String.valueOf(lot.lotId())); + int updated = ps.executeUpdate(); if (updated > 0) { return; // Successfully updated existing record } } - + // If no rows updated, try INSERT (ignore if conflicts with UNIQUE constraints) try (var ps = conn.prepareStatement(insertSql)) { - ps.setLong(1, lot.lotId()); - ps.setLong(2, lot.saleId()); + ps.setString(1, String.valueOf(lot.lotId())); + ps.setString(2, String.valueOf(lot.saleId())); ps.setString(3, lot.title()); ps.setString(4, lot.description()); ps.setString(5, lot.manufacturer()); @@ -390,7 +389,7 @@ public class DatabaseService { } } } - + /** * Updates a lot with full intelligence data from GraphQL enrichment. * This is a comprehensive update that includes all 24 intelligence fields. @@ -434,7 +433,7 @@ public class DatabaseService { bid_velocity = ? WHERE lot_id = ? """; - + try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) { ps.setLong(1, lot.saleId()); ps.setString(2, lot.title()); @@ -447,12 +446,16 @@ public class DatabaseService { ps.setString(9, lot.currency()); ps.setString(10, lot.url()); ps.setString(11, lot.closingTime() != null ? lot.closingTime().toString() : null); - + // Intelligence fields - if (lot.followersCount() != null) ps.setInt(12, lot.followersCount()); else ps.setNull(12, java.sql.Types.INTEGER); - if (lot.estimatedMin() != null) ps.setDouble(13, lot.estimatedMin()); else ps.setNull(13, java.sql.Types.REAL); - if (lot.estimatedMax() != null) ps.setDouble(14, lot.estimatedMax()); else ps.setNull(14, java.sql.Types.REAL); - if (lot.nextBidStepInCents() != null) ps.setLong(15, lot.nextBidStepInCents()); else ps.setNull(15, java.sql.Types.BIGINT); + if (lot.followersCount() != null) ps.setInt(12, lot.followersCount()); + else ps.setNull(12, java.sql.Types.INTEGER); + if (lot.estimatedMin() != null) ps.setDouble(13, lot.estimatedMin()); + else ps.setNull(13, java.sql.Types.REAL); + if (lot.estimatedMax() != null) ps.setDouble(14, lot.estimatedMax()); + else ps.setNull(14, java.sql.Types.REAL); + if (lot.nextBidStepInCents() != null) ps.setLong(15, lot.nextBidStepInCents()); + else ps.setNull(15, java.sql.Types.BIGINT); ps.setString(16, lot.condition()); ps.setString(17, lot.categoryPath()); ps.setString(18, lot.cityLocation()); @@ -460,28 +463,37 @@ public class DatabaseService { ps.setString(20, lot.biddingStatus()); ps.setString(21, lot.appearance()); ps.setString(22, lot.packaging()); - if (lot.quantity() != null) ps.setLong(23, lot.quantity()); else ps.setNull(23, java.sql.Types.BIGINT); - if (lot.vat() != null) ps.setDouble(24, lot.vat()); else ps.setNull(24, java.sql.Types.REAL); - if (lot.buyerPremiumPercentage() != null) ps.setDouble(25, lot.buyerPremiumPercentage()); else ps.setNull(25, java.sql.Types.REAL); + if (lot.quantity() != null) ps.setLong(23, lot.quantity()); + else ps.setNull(23, java.sql.Types.BIGINT); + if (lot.vat() != null) ps.setDouble(24, lot.vat()); + else ps.setNull(24, java.sql.Types.REAL); + if (lot.buyerPremiumPercentage() != null) ps.setDouble(25, lot.buyerPremiumPercentage()); + else ps.setNull(25, java.sql.Types.REAL); ps.setString(26, lot.remarks()); - if (lot.startingBid() != null) ps.setDouble(27, lot.startingBid()); else ps.setNull(27, java.sql.Types.REAL); - if (lot.reservePrice() != null) ps.setDouble(28, lot.reservePrice()); else ps.setNull(28, java.sql.Types.REAL); - if (lot.reserveMet() != null) ps.setInt(29, lot.reserveMet() ? 1 : 0); else ps.setNull(29, java.sql.Types.INTEGER); - if (lot.bidIncrement() != null) ps.setDouble(30, lot.bidIncrement()); else ps.setNull(30, java.sql.Types.REAL); - if (lot.viewCount() != null) ps.setInt(31, lot.viewCount()); else ps.setNull(31, java.sql.Types.INTEGER); + if (lot.startingBid() != null) ps.setDouble(27, lot.startingBid()); + else ps.setNull(27, java.sql.Types.REAL); + if (lot.reservePrice() != null) ps.setDouble(28, lot.reservePrice()); + else ps.setNull(28, java.sql.Types.REAL); + if (lot.reserveMet() != null) ps.setInt(29, lot.reserveMet() ? 1 : 0); + else ps.setNull(29, java.sql.Types.INTEGER); + if (lot.bidIncrement() != null) ps.setDouble(30, lot.bidIncrement()); + else ps.setNull(30, java.sql.Types.REAL); + if (lot.viewCount() != null) ps.setInt(31, lot.viewCount()); + else ps.setNull(31, java.sql.Types.INTEGER); ps.setString(32, lot.firstBidTime() != null ? lot.firstBidTime().toString() : null); ps.setString(33, lot.lastBidTime() != null ? lot.lastBidTime().toString() : null); - if (lot.bidVelocity() != null) ps.setDouble(34, lot.bidVelocity()); else ps.setNull(34, java.sql.Types.REAL); - + if (lot.bidVelocity() != null) ps.setDouble(34, lot.bidVelocity()); + else ps.setNull(34, java.sql.Types.REAL); + ps.setLong(35, lot.lotId()); - + int updated = ps.executeUpdate(); if (updated == 0) { log.warn("Failed to update lot {} - lot not found in database", lot.lotId()); } } } - + /** * Inserts a complete image record (for testing/legacy compatibility). * In production, scraper inserts with local_path, monitor updates labels via updateImageLabels. @@ -497,7 +509,7 @@ public class DatabaseService { ps.executeUpdate(); } } - + /** * Updates the labels field for an image after object detection */ @@ -510,7 +522,7 @@ public class DatabaseService { ps.executeUpdate(); } } - + /** * Gets the labels for a specific image */ @@ -559,7 +571,7 @@ public class DatabaseService { List list = new ArrayList<>(); var sql = "SELECT lot_id, sale_id as auction_id, title, description, manufacturer, type, year, category, " + "current_bid, currency, url, closing_time, closing_notified FROM lots"; - + try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) { var rs = stmt.executeQuery(sql); while (rs.next()) { @@ -603,7 +615,7 @@ public class DatabaseService { try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement("UPDATE lots SET current_bid = ? WHERE lot_id = ?")) { ps.setDouble(1, lot.currentBid()); - ps.setLong(2, lot.lotId()); + ps.setString(2, String.valueOf(lot.lotId())); ps.executeUpdate(); } } @@ -615,11 +627,11 @@ public class DatabaseService { try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement("UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) { ps.setInt(1, lot.closingNotified() ? 1 : 0); - ps.setLong(2, lot.lotId()); + ps.setString(2, String.valueOf(lot.lotId())); ps.executeUpdate(); } } - + /** * Retrieves bid history for a specific lot */ @@ -627,15 +639,15 @@ public class DatabaseService { List history = new ArrayList<>(); var sql = "SELECT id, lot_id, bid_amount, bid_time, is_autobid, bidder_id, bidder_number " + "FROM bid_history WHERE lot_id = ? ORDER BY bid_time DESC LIMIT 100"; - + try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) { ps.setString(1, lotId); var rs = ps.executeQuery(); - + while (rs.next()) { - LocalDateTime bidTime = null; - var bidTimeStr = rs.getString("bid_time"); + LocalDateTime bidTime = null; + var bidTimeStr = rs.getString("bid_time"); if (bidTimeStr != null && !bidTimeStr.isBlank()) { try { bidTime = LocalDateTime.parse(bidTimeStr); @@ -643,7 +655,7 @@ public class DatabaseService { log.debug("Invalid bid_time format: {}", bidTimeStr); } } - + history.add(new BidHistory( rs.getInt("id"), rs.getString("lot_id"), @@ -667,7 +679,7 @@ public class DatabaseService { */ synchronized List importAuctionsFromScraper() throws SQLException { List imported = new ArrayList<>(); - + // Derive auctions from lots table (scraper doesn't populate auctions table) var sql = """ SELECT @@ -682,7 +694,7 @@ public class DatabaseService { WHERE l.auction_id IS NOT NULL GROUP BY l.auction_id """; - + try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) { var rs = stmt.executeQuery(sql); while (rs.next()) { @@ -707,7 +719,7 @@ public class DatabaseService { // Table might not exist in scraper format - that's ok log.info("ℹ️ Scraper lots table not found or incompatible schema: {}", e.getMessage()); } - + return imported; } @@ -722,7 +734,7 @@ public class DatabaseService { var sql = "SELECT lot_id, auction_id, title, description, category, " + "current_bid, closing_time, url " + "FROM lots"; - + try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) { var rs = stmt.executeQuery(sql); while (rs.next()) { @@ -743,7 +755,7 @@ public class DatabaseService { // Table might not exist in scraper format - that's ok log.info("ℹ️ Scraper lots table not found or incompatible schema"); } - + return imported; } @@ -762,14 +774,14 @@ public class DatabaseService { AND i.local_path != '' AND (i.labels IS NULL OR i.labels = '') """; - + try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) { var rs = stmt.executeQuery(sql); while (rs.next()) { // Extract numeric lot ID from TEXT field (e.g., "A1-34732-49" -> 3473249) String lotIdStr = rs.getString("lot_id"); - long lotId = ScraperDataAdapter.extractNumericId(lotIdStr); - + long lotId = ScraperDataAdapter.extractNumericId(lotIdStr); + images.add(new ImageDetectionRecord( rs.getInt("id"), lotId, @@ -779,17 +791,11 @@ public class DatabaseService { } catch (SQLException e) { log.info("ℹ️ No images needing detection found"); } - + return images; } - /** - * Simple record for image data from database - */ record ImageRecord(int id, long lotId, String url, String filePath, String labels) { } - - /** - * Record for images that need object detection processing - */ + record ImageDetectionRecord(int id, long lotId, String filePath) { } } diff --git a/src/main/java/auctiora/ImageProcessingService.java b/src/main/java/auctiora/ImageProcessingService.java index 8f87f8d..5ffe76a 100644 --- a/src/main/java/auctiora/ImageProcessingService.java +++ b/src/main/java/auctiora/ImageProcessingService.java @@ -1,106 +1,55 @@ package auctiora; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; - import lombok.extern.slf4j.Slf4j; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.sql.SQLException; -import java.util.List; -/** - * Service responsible for processing images from the IMAGES table. - * Performs object detection on already-downloaded images and updates the database. - * - * NOTE: Image downloading is handled by the external scraper process. - * This service only performs object detection on images that already have local_path set. - */ @Slf4j -class ImageProcessingService { - - private final DatabaseService db; - private final ObjectDetectionService detector; - - ImageProcessingService(DatabaseService db, ObjectDetectionService detector) { - this.db = db; - this.detector = detector; - } - - /** - * Processes a single image: runs object detection and updates labels in database. - * - * @param imageId database ID of the image record - * @param localPath local file path to the downloaded image - * @param lotId lot identifier (for logging) - * @return true if processing succeeded - */ - boolean processImage(int imageId, String localPath, long lotId) { +public record ImageProcessingService(DatabaseService db, ObjectDetectionService detector) { + + boolean processImage(int id, String path, long lot) { try { - // Normalize path separators (convert Windows backslashes to forward slashes) - localPath = localPath.replace('\\', '/'); - - // Check if file exists before processing - var file = new java.io.File(localPath); - if (!file.exists() || !file.canRead()) { - log.warn(" Image file not accessible: {}", localPath); + path = path.replace('\\', '/'); + var f = new java.io.File(path); + if (!f.exists() || !f.canRead()) { + log.warn("Image not accessible: {}", path); return false; } - - // Check file size (skip very large files that might cause issues) - long fileSizeBytes = file.length(); - if (fileSizeBytes > 50 * 1024 * 1024) { // 50 MB limit - log.warn(" Image file too large ({}MB): {}", fileSizeBytes / (1024 * 1024), localPath); + if (f.length() > 50L * 1024 * 1024) { + log.warn("Image too large: {}", path); return false; } - - // Run object detection on the local file - var labels = detector.detectObjects(localPath); - - // Update the database with detected labels - db.updateImageLabels(imageId, labels); - - if (!labels.isEmpty()) { - log.info(" Lot {}: Detected {}", lotId, String.join(", ", labels)); - } - + + var labels = detector.detectObjects(path); + db.updateImageLabels(id, labels); + + if (!labels.isEmpty()) + log.info("Lot {}: {}", lot, String.join(", ", labels)); + return true; } catch (Exception e) { - log.error(" Failed to process image {}: {}", imageId, e.getMessage()); + log.error("Process fail {}: {}", id, e.getMessage()); return false; } } - /** - * Batch processes all pending images in the database. - * Only processes images that have been downloaded by the scraper but haven't had object detection run yet. - */ void processPendingImages() { - log.info("Processing pending images..."); - try { - var pendingImages = db.getImagesNeedingDetection(); - log.info("Found {} images needing object detection", pendingImages.size()); - - var processed = 0; - var detected = 0; - - for (var image : pendingImages) { - if (processImage(image.id(), image.filePath(), image.lotId())) { + var images = db.getImagesNeedingDetection(); + log.info("Pending {}", images.size()); + + int processed = 0, detected = 0; + + for (var i : images) { + if (processImage(i.id(), i.filePath(), i.lotId())) { processed++; - // Re-fetch to check if labels were found - var labels = db.getImageLabels(image.id()); - if (labels != null && !labels.isEmpty()) { - detected++; - } + var lbl = db.getImageLabels(i.id()); + if (lbl != null && !lbl.isEmpty()) detected++; } } - - log.info("Processed {} images, detected objects in {}", processed, detected); - - } catch (SQLException e) { - log.error("Error processing pending images: {}", e.getMessage()); + + log.info("Processed {}, detected {}", processed, detected); + + } catch (Exception e) { + log.error("Batch fail: {}", e.getMessage()); } } } diff --git a/src/main/java/auctiora/LotEnrichmentScheduler.java b/src/main/java/auctiora/LotEnrichmentScheduler.java index 5285471..c2532f2 100644 --- a/src/main/java/auctiora/LotEnrichmentScheduler.java +++ b/src/main/java/auctiora/LotEnrichmentScheduler.java @@ -16,73 +16,66 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @ApplicationScoped public class LotEnrichmentScheduler { - - @Inject - LotEnrichmentService enrichmentService; - - /** - * Enriches lots closing within 1 hour - HIGH PRIORITY - * Runs every 5 minutes - */ - @Scheduled(cron = "0 */5 * * * ?") - public void enrichCriticalLots() { - try { - log.debug("Enriching critical lots (closing < 1 hour)"); - int enriched = enrichmentService.enrichClosingSoonLots(1); - if (enriched > 0) { - log.info("Enriched {} critical lots", enriched); - } - } catch (Exception e) { - log.error("Failed to enrich critical lots", e); - } - } - - /** - * Enriches lots closing within 6 hours - MEDIUM PRIORITY - * Runs every 30 minutes - */ - @Scheduled(cron = "0 */30 * * * ?") - public void enrichUrgentLots() { - try { - log.debug("Enriching urgent lots (closing < 6 hours)"); - int enriched = enrichmentService.enrichClosingSoonLots(6); - if (enriched > 0) { - log.info("Enriched {} urgent lots", enriched); - } - } catch (Exception e) { - log.error("Failed to enrich urgent lots", e); - } - } - - /** - * Enriches lots closing within 24 hours - NORMAL PRIORITY - * Runs every 2 hours - */ - @Scheduled(cron = "0 0 */2 * * ?") - public void enrichDailyLots() { - try { - log.debug("Enriching daily lots (closing < 24 hours)"); - int enriched = enrichmentService.enrichClosingSoonLots(24); - if (enriched > 0) { - log.info("Enriched {} daily lots", enriched); - } - } catch (Exception e) { - log.error("Failed to enrich daily lots", e); - } - } - - /** - * Enriches all active lots - LOW PRIORITY - * Runs every 6 hours to keep all data fresh - */ - @Scheduled(cron = "0 0 */6 * * ?") - public void enrichAllLots() { - try { - log.info("Starting full enrichment of all lots"); - int enriched = enrichmentService.enrichAllActiveLots(); - log.info("Full enrichment complete: {} lots updated", enriched); - } catch (Exception e) { - log.error("Failed to enrich all lots", e); - } - } + + @Inject LotEnrichmentService enrichmentService; + + /** + * Enriches lots closing within 1 hour - HIGH PRIORITY + * Runs every 5 minutes + */ + @Scheduled(cron = "0 */5 * * * ?") + public void enrichCriticalLots() { + try { + log.debug("Enriching critical lots (closing < 1 hour)"); + int enriched = enrichmentService.enrichClosingSoonLots(1); + if (enriched > 0) log.info("Enriched {} critical lots", enriched); + } catch (Exception e) { + log.error("Failed to enrich critical lots", e); + } + } + + /** + * Enriches lots closing within 6 hours - MEDIUM PRIORITY + * Runs every 30 minutes + */ + @Scheduled(cron = "0 */30 * * * ?") + public void enrichUrgentLots() { + try { + log.debug("Enriching urgent lots (closing < 6 hours)"); + int enriched = enrichmentService.enrichClosingSoonLots(6); + if (enriched > 0) log.info("Enriched {} urgent lots", enriched); + } catch (Exception e) { + log.error("Failed to enrich urgent lots", e); + } + } + + /** + * Enriches lots closing within 24 hours - NORMAL PRIORITY + * Runs every 2 hours + */ + @Scheduled(cron = "0 0 */2 * * ?") + public void enrichDailyLots() { + try { + log.debug("Enriching daily lots (closing < 24 hours)"); + int enriched = enrichmentService.enrichClosingSoonLots(24); + if (enriched > 0) log.info("Enriched {} daily lots", enriched); + } catch (Exception e) { + log.error("Failed to enrich daily lots", e); + } + } + + /** + * Enriches all active lots - LOW PRIORITY + * Runs every 6 hours to keep all data fresh + */ + @Scheduled(cron = "0 0 */6 * * ?") + public void enrichAllLots() { + try { + log.info("Starting full enrichment of all lots"); + int enriched = enrichmentService.enrichAllActiveLots(); + log.info("Full enrichment complete: {} lots updated", enriched); + } catch (Exception e) { + log.error("Failed to enrich all lots", e); + } + } } diff --git a/src/main/java/auctiora/LotEnrichmentService.java b/src/main/java/auctiora/LotEnrichmentService.java index a813702..382a41d 100644 --- a/src/main/java/auctiora/LotEnrichmentService.java +++ b/src/main/java/auctiora/LotEnrichmentService.java @@ -16,190 +16,186 @@ import java.util.stream.Collectors; @Slf4j @ApplicationScoped public class LotEnrichmentService { - - @Inject - TroostwijkGraphQLClient graphQLClient; - - @Inject - DatabaseService db; - - /** - * Enriches a single lot with GraphQL intelligence data - */ - public boolean enrichLot(Lot lot) { - if (lot.displayId() == null || lot.displayId().isBlank()) { - log.debug("Cannot enrich lot {} - missing displayId", lot.lotId()); + + @Inject TroostwijkGraphQLClient graphQLClient; + @Inject DatabaseService db; + /** + * Enriches a single lot with GraphQL intelligence data + */ + public boolean enrichLot(Lot lot) { + if (lot.displayId() == null || lot.displayId().isBlank()) { + log.debug("Cannot enrich lot {} - missing displayId", lot.lotId()); + return false; + } + + try { + var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId()); + if (intelligence == null) { + log.debug("No intelligence data for lot {}", lot.displayId()); return false; - } - - try { + } + + // Merge intelligence with existing lot data + var enrichedLot = mergeLotWithIntelligence(lot, intelligence); + db.upsertLotWithIntelligence(enrichedLot); + + log.debug("Enriched lot {} with GraphQL data", lot.lotId()); + return true; + + } catch (Exception e) { + log.warn("Failed to enrich lot {}: {}", lot.lotId(), e.getMessage()); + return false; + } + } + + /** + * Enriches multiple lots sequentially + * @param lots List of lots to enrich + * @return Number of successfully enriched lots + */ + public int enrichLotsBatch(List lots) { + if (lots.isEmpty()) { + return 0; + } + + log.info("Enriching {} lots via GraphQL", lots.size()); + int enrichedCount = 0; + + for (var lot : lots) { + if (lot.displayId() == null || lot.displayId().isBlank()) { + log.debug("Skipping lot {} - missing displayId", lot.lotId()); + continue; + } + + try { var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId()); - if (intelligence == null) { - log.debug("No intelligence data for lot {}", lot.displayId()); - return false; + if (intelligence != null) { + var enrichedLot = mergeLotWithIntelligence(lot, intelligence); + db.upsertLotWithIntelligence(enrichedLot); + enrichedCount++; + } else { + log.debug("No intelligence data for lot {}", lot.displayId()); } - - // Merge intelligence with existing lot data - var enrichedLot = mergeLotWithIntelligence(lot, intelligence); - db.upsertLotWithIntelligence(enrichedLot); - - log.debug("Enriched lot {} with GraphQL data", lot.lotId()); - return true; - - } catch (Exception e) { - log.warn("Failed to enrich lot {}: {}", lot.lotId(), e.getMessage()); - return false; - } - } - - /** - * Enriches multiple lots sequentially - * @param lots List of lots to enrich - * @return Number of successfully enriched lots - */ - public int enrichLotsBatch(List lots) { - if (lots.isEmpty()) { + } catch (Exception e) { + log.warn("Failed to enrich lot {}: {}", lot.displayId(), e.getMessage()); + } + + // Small delay to respect rate limits (handled by RateLimitedHttpClient) + } + + log.info("Successfully enriched {}/{} lots", enrichedCount, lots.size()); + return enrichedCount; + } + + /** + * Enriches lots closing soon (within specified hours) with higher priority + */ + public int enrichClosingSoonLots(int hoursUntilClose) { + try { + var allLots = db.getAllLots(); + var closingSoon = allLots.stream() + .filter(lot -> lot.closingTime() != null) + .filter(lot -> { + long minutes = lot.minutesUntilClose(); + return minutes > 0 && minutes <= hoursUntilClose * 60; + }) + .toList(); + + if (closingSoon.isEmpty()) { + log.debug("No lots closing within {} hours", hoursUntilClose); return 0; - } - - log.info("Enriching {} lots via GraphQL", lots.size()); - int enrichedCount = 0; - - for (var lot : lots) { - if (lot.displayId() == null || lot.displayId().isBlank()) { - log.debug("Skipping lot {} - missing displayId", lot.lotId()); - continue; + } + + log.info("Enriching {} lots closing within {} hours", closingSoon.size(), hoursUntilClose); + return enrichLotsBatch(closingSoon); + + } catch (Exception e) { + log.error("Failed to enrich closing soon lots: {}", e.getMessage()); + return 0; + } + } + + /** + * Enriches all active lots (can be slow for large datasets) + */ + public int enrichAllActiveLots() { + try { + var allLots = db.getAllLots(); + log.info("Enriching all {} active lots", allLots.size()); + + // Process in batches to avoid overwhelming the API + int batchSize = 50; + int totalEnriched = 0; + + for (int i = 0; i < allLots.size(); i += batchSize) { + int end = Math.min(i + batchSize, allLots.size()); + List batch = allLots.subList(i, end); + + int enriched = enrichLotsBatch(batch); + totalEnriched += enriched; + + // Small delay between batches to respect rate limits + if (end < allLots.size()) { + Thread.sleep(1000); } - - try { - var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId()); - if (intelligence != null) { - var enrichedLot = mergeLotWithIntelligence(lot, intelligence); - db.upsertLotWithIntelligence(enrichedLot); - enrichedCount++; - } else { - log.debug("No intelligence data for lot {}", lot.displayId()); - } - } catch (Exception e) { - log.warn("Failed to enrich lot {}: {}", lot.displayId(), e.getMessage()); - } - - // Small delay to respect rate limits (handled by RateLimitedHttpClient) - } - - log.info("Successfully enriched {}/{} lots", enrichedCount, lots.size()); - return enrichedCount; - } - - /** - * Enriches lots closing soon (within specified hours) with higher priority - */ - public int enrichClosingSoonLots(int hoursUntilClose) { - try { - var allLots = db.getAllLots(); - var closingSoon = allLots.stream() - .filter(lot -> lot.closingTime() != null) - .filter(lot -> { - long minutes = lot.minutesUntilClose(); - return minutes > 0 && minutes <= hoursUntilClose * 60; - }) - .toList(); - - if (closingSoon.isEmpty()) { - log.debug("No lots closing within {} hours", hoursUntilClose); - return 0; - } - - log.info("Enriching {} lots closing within {} hours", closingSoon.size(), hoursUntilClose); - return enrichLotsBatch(closingSoon); - - } catch (Exception e) { - log.error("Failed to enrich closing soon lots: {}", e.getMessage()); - return 0; - } - } - - /** - * Enriches all active lots (can be slow for large datasets) - */ - public int enrichAllActiveLots() { - try { - var allLots = db.getAllLots(); - log.info("Enriching all {} active lots", allLots.size()); - - // Process in batches to avoid overwhelming the API - int batchSize = 50; - int totalEnriched = 0; - - for (int i = 0; i < allLots.size(); i += batchSize) { - int end = Math.min(i + batchSize, allLots.size()); - List batch = allLots.subList(i, end); - - int enriched = enrichLotsBatch(batch); - totalEnriched += enriched; - - // Small delay between batches to respect rate limits - if (end < allLots.size()) { - Thread.sleep(1000); - } - } - - log.info("Finished enriching all lots. Total enriched: {}/{}", totalEnriched, allLots.size()); - return totalEnriched; - - } catch (Exception e) { - log.error("Failed to enrich all lots: {}", e.getMessage()); - return 0; - } - } - - /** - * Merges existing lot data with GraphQL intelligence - */ - private Lot mergeLotWithIntelligence(Lot lot, LotIntelligence intel) { - return new Lot( - lot.saleId(), - lot.lotId(), - lot.displayId(), // Preserve displayId - lot.title(), - lot.description(), - lot.manufacturer(), - lot.type(), - lot.year(), - lot.category(), - lot.currentBid(), - lot.currency(), - lot.url(), - lot.closingTime(), - lot.closingNotified(), - // HIGH PRIORITY FIELDS from GraphQL - intel.followersCount(), - intel.estimatedMin(), - intel.estimatedMax(), - intel.nextBidStepInCents(), - intel.condition(), - intel.categoryPath(), - intel.cityLocation(), - intel.countryCode(), - // MEDIUM PRIORITY FIELDS - intel.biddingStatus(), - intel.appearance(), - intel.packaging(), - intel.quantity(), - intel.vat(), - intel.buyerPremiumPercentage(), - intel.remarks(), - // BID INTELLIGENCE FIELDS - intel.startingBid(), - intel.reservePrice(), - intel.reserveMet(), - intel.bidIncrement(), - intel.viewCount(), - intel.firstBidTime(), - intel.lastBidTime(), - intel.bidVelocity(), - null, // condition_score (computed separately) - null // provenance_docs (computed separately) - ); - } + } + + log.info("Finished enriching all lots. Total enriched: {}/{}", totalEnriched, allLots.size()); + return totalEnriched; + + } catch (Exception e) { + log.error("Failed to enrich all lots: {}", e.getMessage()); + return 0; + } + } + + /** + * Merges existing lot data with GraphQL intelligence + */ + private Lot mergeLotWithIntelligence(Lot lot, LotIntelligence intel) { + return new Lot( + lot.saleId(), + lot.lotId(), + lot.displayId(), // Preserve displayId + lot.title(), + lot.description(), + lot.manufacturer(), + lot.type(), + lot.year(), + lot.category(), + lot.currentBid(), + lot.currency(), + lot.url(), + lot.closingTime(), + lot.closingNotified(), + // HIGH PRIORITY FIELDS from GraphQL + intel.followersCount(), + intel.estimatedMin(), + intel.estimatedMax(), + intel.nextBidStepInCents(), + intel.condition(), + intel.categoryPath(), + intel.cityLocation(), + intel.countryCode(), + // MEDIUM PRIORITY FIELDS + intel.biddingStatus(), + intel.appearance(), + intel.packaging(), + intel.quantity(), + intel.vat(), + intel.buyerPremiumPercentage(), + intel.remarks(), + // BID INTELLIGENCE FIELDS + intel.startingBid(), + intel.reservePrice(), + intel.reserveMet(), + intel.bidIncrement(), + intel.viewCount(), + intel.firstBidTime(), + intel.lastBidTime(), + intel.bidVelocity(), + null, // condition_score (computed separately) + null // provenance_docs (computed separately) + ); + } } diff --git a/src/main/java/auctiora/Main.java b/src/main/java/auctiora/Main.java deleted file mode 100644 index a8bfdce..0000000 --- a/src/main/java/auctiora/Main.java +++ /dev/null @@ -1,231 +0,0 @@ -package auctiora; - -import lombok.extern.slf4j.Slf4j; -import org.opencv.core.Core; - -/** - * Main entry point for Troostwijk Auction Monitor. - * - * ARCHITECTURE: - * This project focuses on: - * 1. Image processing and object detection - * 2. Bid monitoring and notifications - * 3. Data enrichment - * - * Auction/Lot scraping is handled by the external ARCHITECTURE-TROOSTWIJK-SCRAPER process. - * That process populates the auctions and lots tables in the shared database. - * This process reads from those tables and enriches them with: - * - Downloaded images - * - Object detection labels - * - Bid monitoring - * - Notifications - */ -@Slf4j -public class Main { - - @SuppressWarnings("restricted") - private static Object loadOpenCV() { - System.loadLibrary(Core.NATIVE_LIBRARY_NAME); - return null; - } - - public static void main(String[] args) throws Exception { - log.info("=== Troostwijk Auction Monitor ===\n"); - - // Parse command line arguments - var mode = args.length > 0 ? args[0] : "workflow"; - - // Configuration - Windows paths - var databaseFile = System.getenv().getOrDefault("DATABASE_FILE", "C:\\mnt\\okcomputer\\output\\cache.db"); - var notificationConfig = System.getenv().getOrDefault("NOTIFICATION_CONFIG", "desktop"); - - // YOLO model paths (optional - monitor works without object detection) - var yoloCfg = "models/yolov4.cfg"; - var yoloWeights = "models/yolov4.weights"; - var yoloClasses = "models/coco.names"; - - // Load native OpenCV library (only if models exist) - try { - loadOpenCV(); - log.info("✓ OpenCV loaded"); - } catch (UnsatisfiedLinkError e) { - log.info("⚠️ OpenCV not available - image detection disabled"); - } - - switch (mode.toLowerCase()) { - case "workflow": - runWorkflowMode(databaseFile, notificationConfig, yoloCfg, yoloWeights, yoloClasses); - break; - - case "once": - runOnceMode(databaseFile, notificationConfig, yoloCfg, yoloWeights, yoloClasses); - break; - - case "legacy": - runLegacyMode(databaseFile, notificationConfig, yoloCfg, yoloWeights, yoloClasses); - break; - - case "status": - showStatus(databaseFile, notificationConfig, yoloCfg, yoloWeights, yoloClasses); - break; - - default: - showUsage(); - break; - } - } - - /** - * WORKFLOW MODE: Run orchestrated scheduled workflows (default) - * This is the recommended mode for production use. - */ - private static void runWorkflowMode(String dbPath, String notifConfig, - String yoloCfg, String yoloWeights, String yoloClasses) - throws Exception { - - log.info("🚀 Starting in WORKFLOW MODE (Orchestrated Scheduling)\n"); - - var orchestrator = new WorkflowOrchestrator( - dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses - ); - - // Show initial status - orchestrator.printStatus(); - - // Start all scheduled workflows - orchestrator.startScheduledWorkflows(); - - log.info("✓ All workflows are running"); - log.info(" - Scraper import: every 30 min"); - log.info(" - Image processing: every 1 hour"); - log.info(" - Bid monitoring: every 15 min"); - log.info(" - Closing alerts: every 5 min"); - log.info("\nPress Ctrl+C to stop.\n"); - - // Add shutdown hook - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - log.info("\n🛑 Shutdown signal received..."); - orchestrator.shutdown(); - })); - - // Keep application alive - try { - Thread.sleep(Long.MAX_VALUE); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - orchestrator.shutdown(); - } - } - - /** - * ONCE MODE: Run complete workflow once and exit - * Useful for cron jobs or scheduled tasks. - */ - private static void runOnceMode(String dbPath, String notifConfig, - String yoloCfg, String yoloWeights, String yoloClasses) - throws Exception { - - log.info("🔄 Starting in ONCE MODE (Single Execution)\n"); - - var orchestrator = new WorkflowOrchestrator( - dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses - ); - - orchestrator.runCompleteWorkflowOnce(); - - log.info("✓ Workflow execution completed. Exiting.\n"); - } - - /** - * LEGACY MODE: Original monitoring approach - * Kept for backward compatibility. - */ - private static void runLegacyMode(String dbPath, String notifConfig, - String yoloCfg, String yoloWeights, String yoloClasses) - throws Exception { - - log.info("⚙️ Starting in LEGACY MODE\n"); - - var monitor = new TroostwijkMonitor(dbPath, notifConfig, - yoloCfg, yoloWeights, yoloClasses); - - log.info("\n📊 Current Database State:"); - monitor.printDatabaseStats(); - - log.info("\n[1/2] Processing images..."); - monitor.processPendingImages(); - - log.info("\n[2/2] Starting bid monitoring..."); - monitor.scheduleMonitoring(); - - log.info("\n✓ Monitor is running. Press Ctrl+C to stop.\n"); - log.info("NOTE: This process expects auction/lot data from the external scraper."); - log.info(" Make sure ARCHITECTURE-TROOSTWIJK-SCRAPER is running and populating the database.\n"); - - try { - Thread.sleep(Long.MAX_VALUE); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.info("Monitor interrupted, exiting."); - } - } - - /** - * STATUS MODE: Show current status and exit - */ - private static void showStatus(String dbPath, String notifConfig, - String yoloCfg, String yoloWeights, String yoloClasses) - throws Exception { - - log.info("📊 Checking Status...\n"); - - var orchestrator = new WorkflowOrchestrator( - dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses - ); - - orchestrator.printStatus(); - } - - /** - * Show usage information - */ - private static void showUsage() { - log.info("Usage: java -jar troostwijk-monitor.jar [mode]\n"); - log.info("Modes:"); - log.info(" workflow - Run orchestrated scheduled workflows (default)"); - log.info(" once - Run complete workflow once and exit (for cron)"); - log.info(" legacy - Run original monitoring approach"); - log.info(" status - Show current status and exit"); - log.info("\nEnvironment Variables:"); - log.info(" DATABASE_FILE - Path to SQLite database"); - log.info(" (default: C:\\mnt\\okcomputer\\output\\cache.db)"); - log.info(" NOTIFICATION_CONFIG - 'desktop' or 'smtp:user:pass:email'"); - log.info(" (default: desktop)"); - log.info("\nExamples:"); - log.info(" java -jar troostwijk-monitor.jar workflow"); - log.info(" java -jar troostwijk-monitor.jar once"); - log.info(" java -jar troostwijk-monitor.jar status"); - IO.println(); - } - - /** - * Alternative entry point for container environments. - * Simply keeps the container alive for manual commands. - */ - public static void main2(String[] args) { - if (args.length > 0) { - log.info("Command mode - exiting to allow shell commands"); - return; - } - - log.info("Troostwijk Monitor container is running and healthy."); - log.info("Use 'docker exec' or 'dokku run' to execute commands."); - - try { - Thread.sleep(Long.MAX_VALUE); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.info("Container interrupted, exiting."); - } - } -} diff --git a/src/main/java/auctiora/NotificationService.java b/src/main/java/auctiora/NotificationService.java index e4f486d..d7a1a20 100644 --- a/src/main/java/auctiora/NotificationService.java +++ b/src/main/java/auctiora/NotificationService.java @@ -1,48 +1,49 @@ package auctiora; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import org.eclipse.microprofile.config.inject.ConfigProperty; +import lombok.extern.slf4j.Slf4j; import javax.mail.*; import javax.mail.internet.*; -import lombok.extern.slf4j.Slf4j; import java.awt.*; import java.util.Date; import java.util.Properties; @Slf4j -public class NotificationService { - - private final Config config; - - public NotificationService(String cfg) { - this.config = Config.parse(cfg); +public record NotificationService(Config cfg) { + + // Extra convenience constructor: raw string → Config + public NotificationService(String raw) { + this(Config.parse(raw)); } - public void sendNotification(String message, String title, int priority) { - if (config.useDesktop()) sendDesktop(title, message, priority); - if (config.useEmail()) sendEmail(title, message, priority); + public void sendNotification(String msg, String title, int prio) { + if (cfg.useDesktop()) sendDesktop(title, msg, prio); + if (cfg.useEmail()) sendEmail(title, msg, prio); } private void sendDesktop(String title, String msg, int prio) { try { if (!SystemTray.isSupported()) { - log.info("Desktop notifications not supported — " + title + " / " + msg); + log.info("Desktop not supported: {}", title); return; } - var tray = SystemTray.getSystemTray(); - var image = Toolkit.getDefaultToolkit().createImage(new byte[0]); - var trayIcon = new TrayIcon(image, "NotificationService"); - trayIcon.setImageAutoSize(true); + + var tray = SystemTray.getSystemTray(); + var icon = new TrayIcon( + Toolkit.getDefaultToolkit().createImage(new byte[0]), + "notify" + ); + icon.setImageAutoSize(true); + tray.add(icon); + var type = prio > 0 ? TrayIcon.MessageType.WARNING : TrayIcon.MessageType.INFO; - tray.add(trayIcon); - trayIcon.displayMessage(title, msg, type); + icon.displayMessage(title, msg, type); + Thread.sleep(2000); - tray.remove(trayIcon); - log.info("Desktop notification sent: " + title); + tray.remove(icon); + log.info("Desktop notification: {}", title); } catch (Exception e) { - System.err.println("Desktop notification failed: " + e); + log.warn("Desktop failed: {}", e.getMessage()); } } @@ -57,29 +58,32 @@ public class NotificationService { var session = Session.getInstance(props, new Authenticator() { + @Override protected PasswordAuthentication getPasswordAuthentication() { - return new PasswordAuthentication(config.smtpUsername(), config.smtpPassword()); + return new PasswordAuthentication(cfg.smtpUsername(), cfg.smtpPassword()); } }); var m = new MimeMessage(session); - m.setFrom(new InternetAddress(config.smtpUsername())); - m.setRecipients(Message.RecipientType.TO, InternetAddress.parse(config.toEmail())); + m.setFrom(new InternetAddress(cfg.smtpUsername())); + m.setRecipients(Message.RecipientType.TO, InternetAddress.parse(cfg.toEmail())); m.setSubject("[Troostwijk] " + title); m.setText(msg); m.setSentDate(new Date()); + if (prio > 0) { m.setHeader("X-Priority", "1"); m.setHeader("Importance", "High"); } + Transport.send(m); - log.info("Email notification sent: " + title); + log.info("Email notification: {}", title); } catch (Exception e) { - log.info("Email notification failed: " + e); + log.warn("Email failed: {}", e.getMessage()); } } - private record Config( + public record Config( boolean useDesktop, boolean useEmail, String smtpUsername, @@ -87,16 +91,20 @@ public class NotificationService { String toEmail ) { - static Config parse(String cfg) { - if ("desktop".equalsIgnoreCase(cfg)) { + public static Config parse(String raw) { + if ("desktop".equalsIgnoreCase(raw)) { return new Config(true, false, null, null, null); - } else if (cfg.startsWith("smtp:")) { - var parts = cfg.split(":", -1); // Use -1 to include trailing empty strings - if (parts.length < 4) - throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'"); - return new Config(true, true, parts[1], parts[2], parts[3]); } - throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'"); + + if (raw != null && raw.startsWith("smtp:")) { + var p = raw.split(":", -1); + if (p.length < 4) { + throw new IllegalArgumentException("Format: smtp:username:password:toEmail"); + } + return new Config(true, true, p[1], p[2], p[3]); + } + + throw new IllegalArgumentException("Use 'desktop' or 'smtp:username:password:toEmail'"); } } } diff --git a/src/main/java/auctiora/ObjectDetectionService.java b/src/main/java/auctiora/ObjectDetectionService.java index 093a0ce..8749721 100644 --- a/src/main/java/auctiora/ObjectDetectionService.java +++ b/src/main/java/auctiora/ObjectDetectionService.java @@ -33,13 +33,13 @@ import static org.opencv.dnn.Dnn.DNN_TARGET_CPU; */ @Slf4j public class ObjectDetectionService { - - private final Net net; - private final List classNames; - private final boolean enabled; - private int warnCount = 0; - private static final int MAX_WARNINGS = 5; - + + private final Net net; + private final List classNames; + private final boolean enabled; + private int warnCount = 0; + private static final int MAX_WARNINGS = 5; + ObjectDetectionService(String cfgPath, String weightsPath, String classNamesPath) throws IOException { // Check if model files exist var cfgFile = Paths.get(cfgPath); @@ -53,38 +53,38 @@ public class ObjectDetectionService { log.info(" - {}", weightsPath); log.info(" - {}", classNamesPath); log.info(" Scraper will continue without image analysis."); - this.enabled = false; - this.net = null; - this.classNames = new ArrayList<>(); + enabled = false; + net = null; + classNames = new ArrayList<>(); return; } try { // Load network - this.net = Dnn.readNetFromDarknet(cfgPath, weightsPath); - + net = Dnn.readNetFromDarknet(cfgPath, weightsPath); + // Try to use GPU/CUDA if available, fallback to CPU try { - this.net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA); - this.net.setPreferableTarget(Dnn.DNN_TARGET_CUDA); + net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA); + net.setPreferableTarget(Dnn.DNN_TARGET_CUDA); log.info("✓ Object detection enabled with YOLO (CUDA/GPU acceleration)"); } catch (Exception e) { // CUDA not available, try Vulkan for AMD GPUs try { - this.net.setPreferableBackend(Dnn.DNN_BACKEND_VKCOM); - this.net.setPreferableTarget(Dnn.DNN_TARGET_VULKAN); + net.setPreferableBackend(Dnn.DNN_BACKEND_VKCOM); + net.setPreferableTarget(Dnn.DNN_TARGET_VULKAN); log.info("✓ Object detection enabled with YOLO (Vulkan/GPU acceleration)"); } catch (Exception e2) { // GPU not available, fallback to CPU - this.net.setPreferableBackend(DNN_BACKEND_OPENCV); - this.net.setPreferableTarget(DNN_TARGET_CPU); + net.setPreferableBackend(DNN_BACKEND_OPENCV); + net.setPreferableTarget(DNN_TARGET_CPU); log.info("✓ Object detection enabled with YOLO (CPU only)"); } } - + // Load class names (one per line) - this.classNames = Files.readAllLines(classNamesFile); - this.enabled = true; + classNames = Files.readAllLines(classNamesFile); + enabled = true; } catch (UnsatisfiedLinkError e) { System.err.println("⚠️ Object detection disabled: OpenCV native libraries not loaded"); throw new IOException("Failed to initialize object detection: OpenCV native libraries not loaded", e); @@ -121,15 +121,15 @@ public class ObjectDetectionService { var confThreshold = 0.5f; for (var out : outs) { // YOLO output shape: [num_detections, 85] where 85 = 4 (bbox) + 1 (objectness) + 80 (classes) - int numDetections = out.rows(); - int numElements = out.cols(); + int numDetections = out.rows(); + int numElements = out.cols(); int expectedLength = 5 + classNames.size(); - + if (numElements < expectedLength) { // Rate-limit warnings to prevent thread blocking from excessive logging if (warnCount < MAX_WARNINGS) { log.warn("Output matrix has wrong dimensions: expected {} columns, got {}. Output shape: [{}, {}]", - expectedLength, numElements, numDetections, numElements); + expectedLength, numElements, numDetections, numElements); warnCount++; if (warnCount == MAX_WARNINGS) { log.warn("Suppressing further dimension warnings (reached {} warnings)", MAX_WARNINGS); @@ -137,27 +137,27 @@ public class ObjectDetectionService { } continue; } - + for (var i = 0; i < numDetections; i++) { // Get entire row (all 85 elements) var data = new double[numElements]; for (int j = 0; j < numElements; j++) { data[j] = out.get(i, j)[0]; } - + // Extract objectness score (index 4) and class scores (index 5+) double objectness = data[4]; if (objectness < confThreshold) { continue; // Skip low-confidence detections } - + // Extract class scores var scores = new double[classNames.size()]; System.arraycopy(data, 5, scores, 0, Math.min(scores.length, data.length - 5)); - + var classId = argMax(scores); var confidence = scores[classId] * objectness; // Combine objectness with class confidence - + if (confidence > confThreshold) { var label = classNames.get(classId); if (!labels.contains(label)) { @@ -166,7 +166,7 @@ public class ObjectDetectionService { } } } - + // Release resources image.release(); blob.release(); diff --git a/src/main/java/auctiora/QuarkusWorkflowScheduler.java b/src/main/java/auctiora/QuarkusWorkflowScheduler.java index 976e7dd..7d2c08d 100644 --- a/src/main/java/auctiora/QuarkusWorkflowScheduler.java +++ b/src/main/java/auctiora/QuarkusWorkflowScheduler.java @@ -5,6 +5,7 @@ import io.quarkus.scheduler.Scheduled; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.event.Observes; import jakarta.inject.Inject; +import lombok.extern.slf4j.Slf4j; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.jboss.logging.Logger; @@ -22,24 +23,14 @@ public class QuarkusWorkflowScheduler { private static final Logger LOG = Logger.getLogger(QuarkusWorkflowScheduler.class); - @Inject - DatabaseService db; + @Inject DatabaseService db; + @Inject NotificationService notifier; + @Inject ObjectDetectionService detector; + @Inject ImageProcessingService imageProcessor; + @Inject LotEnrichmentService enrichmentService; - @Inject - NotificationService notifier; + @ConfigProperty(name = "auction.database.path") String databasePath; - @Inject - ObjectDetectionService detector; - - @Inject - ImageProcessingService imageProcessor; - - @Inject - LotEnrichmentService enrichmentService; - - @ConfigProperty(name = "auction.database.path") - String databasePath; - /** * Triggered on application startup to enrich existing lots with bid intelligence */ @@ -108,41 +99,41 @@ public class QuarkusWorkflowScheduler { try { LOG.info("🖼️ [WORKFLOW 2] Processing pending images..."); var start = System.currentTimeMillis(); - + // Get images that have been downloaded but need object detection var pendingImages = db.getImagesNeedingDetection(); - + if (pendingImages.isEmpty()) { LOG.info(" → No pending images to process"); return; } - + // Limit batch size to prevent thread blocking (max 100 images per run) final int MAX_BATCH_SIZE = 100; - int totalPending = pendingImages.size(); + int totalPending = pendingImages.size(); if (totalPending > MAX_BATCH_SIZE) { LOG.infof(" → Found %d pending images, processing first %d (batch limit)", - totalPending, MAX_BATCH_SIZE); + totalPending, MAX_BATCH_SIZE); pendingImages = pendingImages.subList(0, MAX_BATCH_SIZE); } else { LOG.infof(" → Processing %d images", totalPending); } - + var processed = 0; var detected = 0; var failed = 0; - + for (var image : pendingImages) { try { // Run object detection on already-downloaded image if (imageProcessor.processImage(image.id(), image.filePath(), image.lotId())) { processed++; - + // Check if objects were detected var labels = db.getImageLabels(image.id()); if (labels != null && !labels.isEmpty()) { detected++; - + // Send notification for interesting detections if (labels.size() >= 3) { notifier.sendNotification( @@ -151,16 +142,16 @@ public class QuarkusWorkflowScheduler { String.join(", ", labels)), "Objects Detected", 0 - ); + ); } } } else { failed++; } - + // Rate limiting (lighter since no network I/O) Thread.sleep(100); - + } catch (Exception e) { failed++; LOG.warnf(" ⚠️ Failed to process image: %s", e.getMessage()); @@ -170,7 +161,7 @@ public class QuarkusWorkflowScheduler { var duration = System.currentTimeMillis() - start; LOG.infof(" ✓ Processed %d/%d images, detected objects in %d, failed %d (%.1fs)", processed, totalPending, detected, failed, duration / 1000.0); - + if (totalPending > MAX_BATCH_SIZE) { LOG.infof(" → %d images remaining for next run", totalPending - MAX_BATCH_SIZE); } @@ -238,7 +229,7 @@ public class QuarkusWorkflowScheduler { lot.manufacturer(), lot.type(), lot.year(), lot.category(), lot.currentBid(), lot.currency(), lot.url(), lot.closingTime(), true - ); + ); db.updateLotNotificationFlags(updated); alertsSent++; diff --git a/src/main/java/auctiora/RateLimitedHttpClient.java b/src/main/java/auctiora/RateLimitedHttpClient.java index 3f4d914..abb155c 100644 --- a/src/main/java/auctiora/RateLimitedHttpClient.java +++ b/src/main/java/auctiora/RateLimitedHttpClient.java @@ -142,29 +142,15 @@ public class RateLimitedHttpClient { * Determines max requests per second for a given host. */ private int getMaxRequestsPerSecond(String host) { - if (host.contains("troostwijk")) { - return troostwijkMaxRequestsPerSecond; - } - return defaultMaxRequestsPerSecond; + return host.contains("troostwijk") ? troostwijkMaxRequestsPerSecond : defaultMaxRequestsPerSecond; } - /** - * Extracts host from URI (e.g., "api.troostwijkauctions.com"). - */ private String extractHost(URI uri) { return uri.getHost() != null ? uri.getHost() : uri.toString(); } - - /** - * Gets statistics for all hosts. - */ public Map getAllStats() { return Map.copyOf(requestStats); } - - /** - * Gets statistics for a specific host. - */ public RequestStats getStats(String host) { return requestStats.get(host); } @@ -218,10 +204,7 @@ public class RateLimitedHttpClient { } } - /** - * Statistics tracker for HTTP requests per host. - */ - public static class RequestStats { + public static final class RequestStats { private final String host; private final AtomicLong totalRequests = new AtomicLong(0); @@ -234,25 +217,16 @@ public class RateLimitedHttpClient { this.host = host; } - void incrementTotal() { - totalRequests.incrementAndGet(); - } + void incrementTotal() { totalRequests.incrementAndGet(); } void recordSuccess(long durationMs) { successfulRequests.incrementAndGet(); totalDurationMs.addAndGet(durationMs); } - void incrementFailed() { - failedRequests.incrementAndGet(); - } - - void incrementRateLimited() { - rateLimitedRequests.incrementAndGet(); - } - - // Getters - public String getHost() { return host; } + void incrementFailed() { failedRequests.incrementAndGet(); } + void incrementRateLimited() { rateLimitedRequests.incrementAndGet(); } + public String getHost() { return host; } public long getTotalRequests() { return totalRequests.get(); } public long getSuccessfulRequests() { return successfulRequests.get(); } public long getFailedRequests() { return failedRequests.get(); } diff --git a/src/main/java/auctiora/ScraperDataAdapter.java b/src/main/java/auctiora/ScraperDataAdapter.java index 930a436..1ccf3fe 100644 --- a/src/main/java/auctiora/ScraperDataAdapter.java +++ b/src/main/java/auctiora/ScraperDataAdapter.java @@ -63,13 +63,15 @@ public class ScraperDataAdapter { lotIdStr, // Store full displayId for GraphQL queries rs.getString("title"), getStringOrDefault(rs, "description", ""), - "", "", 0, + getStringOrDefault(rs, "manufacturer", ""), + getStringOrDefault(rs, "type", ""), + getIntOrDefault(rs, "year", 0), getStringOrDefault(rs, "category", ""), bid, currency, rs.getString("url"), closing, - false, + getBooleanOrDefault(rs, "closing_notified", false), // New intelligence fields - set to null for now null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, @@ -166,4 +168,9 @@ public class ScraperDataAdapter { var v = rs.getInt(col); return rs.wasNull() ? def : v; } + + private static boolean getBooleanOrDefault(ResultSet rs, String col, boolean def) throws SQLException { + var v = rs.getInt(col); + return rs.wasNull() ? def : v != 0; + } } diff --git a/src/main/java/auctiora/StatusResource.java b/src/main/java/auctiora/StatusResource.java index 723dcf6..b4f0b73 100644 --- a/src/main/java/auctiora/StatusResource.java +++ b/src/main/java/auctiora/StatusResource.java @@ -5,7 +5,9 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; import lombok.extern.slf4j.Slf4j; +import nu.pattern.OpenCV; import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.opencv.core.Core; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -19,18 +21,11 @@ public class StatusResource { DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z") .withZone(ZoneId.systemDefault()); - @ConfigProperty(name = "application.version", defaultValue = "1.0-SNAPSHOT") - String appVersion; - @ConfigProperty(name = "application.groupId") - String groupId; + @ConfigProperty(name = "application.version", defaultValue = "1.0-SNAPSHOT") String appVersion; + @ConfigProperty(name = "application.groupId") String groupId; + @ConfigProperty(name = "application.artifactId") String artifactId; + @ConfigProperty(name = "application.version") String version; - @ConfigProperty(name = "application.artifactId") - String artifactId; - - @ConfigProperty(name = "application.version") - String version; - - // Java 16+ Record for structured response public record StatusResponse( String groupId, String artifactId, @@ -47,8 +42,6 @@ public class StatusResource { @Path("/status") @Produces(MediaType.APPLICATION_JSON) public StatusResponse getStatus() { - log.info("Status endpoint called"); - return new StatusResponse(groupId, artifactId, version, "running", FORMATTER.format(Instant.now()), @@ -63,8 +56,6 @@ public class StatusResource { @Path("/hello") @Produces(MediaType.APPLICATION_JSON) public Map sayHello() { - log.info("hello endpoint called"); - return Map.of( "message", "Hello from Scrape-UI!", "timestamp", FORMATTER.format(Instant.now()), @@ -74,9 +65,8 @@ public class StatusResource { private String getOpenCvVersion() { try { - // Load OpenCV if not already loaded (safe to call multiple times) - nu.pattern.OpenCV.loadLocally(); - return org.opencv.core.Core.VERSION; + OpenCV.loadLocally(); + return Core.VERSION; } catch (Exception e) { return "4.9.0 (default)"; } diff --git a/src/main/java/auctiora/TroostwijkGraphQLClient.java b/src/main/java/auctiora/TroostwijkGraphQLClient.java index fe53bcd..bbf5ca8 100644 --- a/src/main/java/auctiora/TroostwijkGraphQLClient.java +++ b/src/main/java/auctiora/TroostwijkGraphQLClient.java @@ -44,15 +44,15 @@ public class TroostwijkGraphQLClient { } try { - String query = buildLotQuery(); - String variables = buildVariables(displayId); + var query = buildLotQuery(); + var variables = buildVariables(displayId); // Proper GraphQL request format with query and variables - String requestBody = String.format( + var requestBody = String.format( "{\"query\":\"%s\",\"variables\":%s}", escapeJson(query), variables - ); + ); var request = java.net.http.HttpRequest.newBuilder() .uri(java.net.URI.create(GRAPHQL_ENDPOINT)) @@ -87,15 +87,15 @@ public class TroostwijkGraphQLClient { List results = new ArrayList<>(); // Split into batches of 50 to avoid query size limits - int batchSize = 50; - for (int i = 0; i < lotIds.size(); i += batchSize) { - int end = Math.min(i + batchSize, lotIds.size()); - List batch = lotIds.subList(i, end); + var batchSize = 50; + for (var i = 0; i < lotIds.size(); i += batchSize) { + var end = Math.min(i + batchSize, lotIds.size()); + var batch = lotIds.subList(i, end); try { - String query = buildBatchLotQuery(batch); - String requestBody = String.format("{\"query\":\"%s\"}", - escapeJson(query)); + var query = buildBatchLotQuery(batch); + var requestBody = String.format("{\"query\":\"%s\"}", + escapeJson(query)); var request = java.net.http.HttpRequest.newBuilder() .uri(java.net.URI.create(GRAPHQL_ENDPOINT)) @@ -162,9 +162,9 @@ public class TroostwijkGraphQLClient { } private String buildBatchLotQuery(List lotIds) { - StringBuilder query = new StringBuilder("query {"); + var query = new StringBuilder("query {"); - for (int i = 0; i < lotIds.size(); i++) { + for (var i = 0; i < lotIds.size(); i++) { query.append(String.format(""" lot%d: lot(id: %d) { id @@ -196,9 +196,9 @@ public class TroostwijkGraphQLClient { log.debug("GraphQL API returned HTML instead of JSON - likely auth required or wrong endpoint"); return null; } - - JsonNode root = objectMapper.readTree(json); - JsonNode lotNode = root.path("data").path("lotDetails"); + + var root = objectMapper.readTree(json); + var lotNode = root.path("data").path("lotDetails"); if (lotNode.isMissingNode()) { log.debug("No lotDetails in GraphQL response"); @@ -206,19 +206,19 @@ public class TroostwijkGraphQLClient { } // Extract location from nested object - JsonNode locationNode = lotNode.path("location"); - String city = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "city"); - String countryCode = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "country"); + var locationNode = lotNode.path("location"); + var city = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "city"); + var countryCode = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "country"); // Extract bids count from nested biddingStatistics - JsonNode statsNode = lotNode.path("biddingStatistics"); - Integer bidsCount = statsNode.isMissingNode() ? null : getIntOrNull(statsNode, "numberOfBids"); + var statsNode = lotNode.path("biddingStatistics"); + var bidsCount = statsNode.isMissingNode() ? null : getIntOrNull(statsNode, "numberOfBids"); // Convert cents to euros for estimates - Long estimatedMinCents = getLongOrNull(lotNode, "estimatedValueInCentsMin"); - Long estimatedMaxCents = getLongOrNull(lotNode, "estimatedValueInCentsMax"); - Double estimatedMin = estimatedMinCents != null ? estimatedMinCents.doubleValue() : null; - Double estimatedMax = estimatedMaxCents != null ? estimatedMaxCents.doubleValue() : null; + var estimatedMinCents = getLongOrNull(lotNode, "estimatedValueInCentsMin"); + var estimatedMaxCents = getLongOrNull(lotNode, "estimatedValueInCentsMax"); + var estimatedMin = estimatedMinCents != null ? estimatedMinCents.doubleValue() : null; + var estimatedMax = estimatedMaxCents != null ? estimatedMaxCents.doubleValue() : null; return new LotIntelligence( lotId, @@ -257,11 +257,11 @@ public class TroostwijkGraphQLClient { List results = new ArrayList<>(); try { - JsonNode root = objectMapper.readTree(json); - JsonNode data = root.path("data"); + var root = objectMapper.readTree(json); + var data = root.path("data"); - for (int i = 0; i < lotIds.size(); i++) { - JsonNode lotNode = data.path("lot" + i); + for (var i = 0; i < lotIds.size(); i++) { + var lotNode = data.path("lot" + i); if (!lotNode.isMissingNode()) { var intelligence = parseLotIntelligenceFromNode(lotNode, lotIds.get(i)); if (intelligence != null) { @@ -313,17 +313,17 @@ public class TroostwijkGraphQLClient { private Double calculateBidVelocity(JsonNode lotNode) { try { - Integer bidsCount = getIntOrNull(lotNode, "bidsCount"); - String firstBidStr = getStringOrNull(lotNode, "firstBidTime"); + var bidsCount = getIntOrNull(lotNode, "bidsCount"); + var firstBidStr = getStringOrNull(lotNode, "firstBidTime"); if (bidsCount == null || firstBidStr == null || bidsCount == 0) { return null; } - - LocalDateTime firstBid = parseDateTime(firstBidStr); + + var firstBid = parseDateTime(firstBidStr); if (firstBid == null) return null; - - long hoursElapsed = java.time.Duration.between(firstBid, LocalDateTime.now()).toHours(); + + var hoursElapsed = java.time.Duration.between(firstBid, LocalDateTime.now()).toHours(); if (hoursElapsed == 0) return (double) bidsCount; return (double) bidsCount / hoursElapsed; @@ -352,27 +352,27 @@ public class TroostwijkGraphQLClient { } private Integer getIntOrNull(JsonNode node, String field) { - JsonNode fieldNode = node.path(field); + var fieldNode = node.path(field); return fieldNode.isNumber() ? fieldNode.asInt() : null; } private Long getLongOrNull(JsonNode node, String field) { - JsonNode fieldNode = node.path(field); + var fieldNode = node.path(field); return fieldNode.isNumber() ? fieldNode.asLong() : null; } private Double getDoubleOrNull(JsonNode node, String field) { - JsonNode fieldNode = node.path(field); + var fieldNode = node.path(field); return fieldNode.isNumber() ? fieldNode.asDouble() : null; } private String getStringOrNull(JsonNode node, String field) { - JsonNode fieldNode = node.path(field); + var fieldNode = node.path(field); return fieldNode.isTextual() ? fieldNode.asText() : null; } private Boolean getBooleanOrNull(JsonNode node, String field) { - JsonNode fieldNode = node.path(field); + var fieldNode = node.path(field); return fieldNode.isBoolean() ? fieldNode.asBoolean() : null; } } diff --git a/src/main/java/auctiora/ValuationAnalyticsResource.java b/src/main/java/auctiora/ValuationAnalyticsResource.java index 7cef83b..2a4a401 100644 --- a/src/main/java/auctiora/ValuationAnalyticsResource.java +++ b/src/main/java/auctiora/ValuationAnalyticsResource.java @@ -50,25 +50,25 @@ public class ValuationAnalyticsResource { public Response calculateValuation(ValuationRequest request) { try { LOG.infof("Valuation request for lot: %s", request.lotId); - long startTime = System.currentTimeMillis(); + var startTime = System.currentTimeMillis(); // Step 1: Fetch comparable sales from database - List comparables = fetchComparables(request); + var comparables = fetchComparables(request); // Step 2: Calculate Fair Market Value (FMV) - FairMarketValue fmv = calculateFairMarketValue(request, comparables); + var fmv = calculateFairMarketValue(request, comparables); // Step 3: Calculate undervaluation score - double undervaluationScore = calculateUndervaluationScore(request, fmv.value); + var undervaluationScore = calculateUndervaluationScore(request, fmv.value); // Step 4: Predict final price - PricePrediction prediction = calculateFinalPrice(request, fmv.value); + var prediction = calculateFinalPrice(request, fmv.value); // Step 5: Generate bidding strategy - BiddingStrategy strategy = generateBiddingStrategy(request, fmv, prediction); + var strategy = generateBiddingStrategy(request, fmv, prediction); // Step 6: Compile response - ValuationResponse response = new ValuationResponse(); + var response = new ValuationResponse(); response.lotId = request.lotId; response.timestamp = LocalDateTime.now().toString(); response.fairMarketValue = fmv; @@ -76,8 +76,8 @@ public class ValuationAnalyticsResource { response.pricePrediction = prediction; response.biddingStrategy = strategy; response.parameters = request; - - long duration = System.currentTimeMillis() - startTime; + + var duration = System.currentTimeMillis() - startTime; LOG.infof("Valuation completed in %d ms", duration); return Response.ok(response).build(); @@ -115,24 +115,24 @@ public class ValuationAnalyticsResource { * Where weights are exponential/logistic functions of similarity */ private FairMarketValue calculateFairMarketValue(ValuationRequest req, List comparables) { - double weightedSum = 0.0; - double weightSum = 0.0; + var weightedSum = 0.0; + var weightSum = 0.0; List weightedComps = new ArrayList<>(); - for (ComparableLot comp : comparables) { + for (var comp : comparables) { // Condition weight: ω_c = exp(-λ_c · |C_target - C_i|) - double omegaC = Math.exp(-0.693 * Math.abs(req.conditionScore - comp.conditionScore)); + var omegaC = Math.exp(-0.693 * Math.abs(req.conditionScore - comp.conditionScore)); // Time weight: ω_t = exp(-λ_t · |T_target - T_i|) - double omegaT = Math.exp(-0.048 * Math.abs(req.manufacturingYear - comp.manufacturingYear)); + var omegaT = Math.exp(-0.048 * Math.abs(req.manufacturingYear - comp.manufacturingYear)); // Provenance weight: ω_p = 1 + δ_p · (P_target - P_i) - double omegaP = 1 + 0.15 * ((req.provenanceDocs > 0 ? 1 : 0) - comp.hasProvenance); + var omegaP = 1 + 0.15 * ((req.provenanceDocs > 0 ? 1 : 0) - comp.hasProvenance); // Historical weight: ω_h = 1 / (1 + e^(-kh · (D_i - D_median))) - double omegaH = 1.0 / (1 + Math.exp(-0.01 * (comp.daysAgo - 40))); - - double totalWeight = omegaC * omegaT * omegaP * omegaH; + var omegaH = 1.0 / (1 + Math.exp(-0.01 * (comp.daysAgo - 40))); + + var totalWeight = omegaC * omegaT * omegaP * omegaH; weightedSum += comp.finalPrice * totalWeight; weightSum += totalWeight; @@ -140,20 +140,20 @@ public class ValuationAnalyticsResource { // Store for transparency weightedComps.add(new WeightedComparable(comp, totalWeight, omegaC, omegaT, omegaP, omegaH)); } - - double baseFMV = weightSum > 0 ? weightedSum / weightSum : (req.estimatedMin + req.estimatedMax) / 2; + + var baseFMV = weightSum > 0 ? weightedSum / weightSum : (req.estimatedMin + req.estimatedMax) / 2; // Apply condition multiplier: M_cond = exp(α_c · √C_target - β_c) - double conditionMultiplier = Math.exp(0.15 * Math.sqrt(req.conditionScore) - 0.40); + var conditionMultiplier = Math.exp(0.15 * Math.sqrt(req.conditionScore) - 0.40); baseFMV *= conditionMultiplier; // Apply provenance premium: Δ_prov = V_base · (η_0 + η_1 · ln(1 + N_docs)) if (req.provenanceDocs > 0) { - double provenancePremium = 0.08 + 0.035 * Math.log(1 + req.provenanceDocs); + var provenancePremium = 0.08 + 0.035 * Math.log(1 + req.provenanceDocs); baseFMV *= (1 + provenancePremium); } - - FairMarketValue fmv = new FairMarketValue(); + + var fmv = new FairMarketValue(); fmv.value = Math.round(baseFMV * 100.0) / 100.0; fmv.conditionMultiplier = Math.round(conditionMultiplier * 1000.0) / 1000.0; fmv.provenancePremium = req.provenanceDocs > 0 ? 0.08 + 0.035 * Math.log(1 + req.provenanceDocs) : 0.0; @@ -170,12 +170,12 @@ public class ValuationAnalyticsResource { */ private double calculateUndervaluationScore(ValuationRequest req, double fmv) { if (fmv <= 0) return 0.0; - - double priceGap = (fmv - req.currentBid) / fmv; - double velocityFactor = 1 + req.bidVelocity / 10.0; - double watchRatio = Math.log(1 + req.watchCount / Math.max(req.bidCount, 1)); - - double uScore = priceGap * req.marketVolatility * velocityFactor * watchRatio; + + var priceGap = (fmv - req.currentBid) / fmv; + var velocityFactor = 1 + req.bidVelocity / 10.0; + var watchRatio = Math.log(1 + req.watchCount / Math.max(req.bidCount, 1)); + + var uScore = priceGap * req.marketVolatility * velocityFactor * watchRatio; return Math.max(0.0, Math.round(uScore * 1000.0) / 1000.0); } @@ -186,22 +186,22 @@ public class ValuationAnalyticsResource { */ private PricePrediction calculateFinalPrice(ValuationRequest req, double fmv) { // Bid momentum error: ε_bid = tanh(φ_1 · Λ_b - φ_2 · P_current/FMV) - double epsilonBid = Math.tanh(0.15 * req.bidVelocity - 0.10 * (req.currentBid / fmv)); + var epsilonBid = Math.tanh(0.15 * req.bidVelocity - 0.10 * (req.currentBid / fmv)); // Time pressure error: ε_time = ψ · exp(-t_close/30) - double epsilonTime = 0.20 * Math.exp(-req.minutesUntilClose / 30.0); + var epsilonTime = 0.20 * Math.exp(-req.minutesUntilClose / 30.0); // Competition error: ε_comp = ρ · ln(1 + W_watch/50) - double epsilonComp = 0.08 * Math.log(1 + req.watchCount / 50.0); - - double predictedPrice = fmv * (1 + epsilonBid + epsilonTime + epsilonComp); + var epsilonComp = 0.08 * Math.log(1 + req.watchCount / 50.0); + + var predictedPrice = fmv * (1 + epsilonBid + epsilonTime + epsilonComp); // 95% confidence interval: ± 1.96 · σ_residual - double residualStdDev = fmv * 0.08; // Mock residual standard deviation - double ciLower = predictedPrice - 1.96 * residualStdDev; - double ciUpper = predictedPrice + 1.96 * residualStdDev; - - PricePrediction pred = new PricePrediction(); + var residualStdDev = fmv * 0.08; // Mock residual standard deviation + var ciLower = predictedPrice - 1.96 * residualStdDev; + var ciUpper = predictedPrice + 1.96 * residualStdDev; + + var pred = new PricePrediction(); pred.predictedPrice = Math.round(predictedPrice * 100.0) / 100.0; pred.confidenceIntervalLower = Math.round(ciLower * 100.0) / 100.0; pred.confidenceIntervalUpper = Math.round(ciUpper * 100.0) / 100.0; @@ -218,7 +218,7 @@ public class ValuationAnalyticsResource { * Generates optimal bidding strategy based on market conditions */ private BiddingStrategy generateBiddingStrategy(ValuationRequest req, FairMarketValue fmv, PricePrediction pred) { - BiddingStrategy strategy = new BiddingStrategy(); + var strategy = new BiddingStrategy(); // Determine competition level if (req.bidVelocity > 5.0) { @@ -236,7 +236,7 @@ public class ValuationAnalyticsResource { strategy.recommendedTiming = "FINAL_10_MINUTES"; // Adjust max bid based on undervaluation - double undervaluationScore = calculateUndervaluationScore(req, fmv.value); + var undervaluationScore = calculateUndervaluationScore(req, fmv.value); if (undervaluationScore > 0.25) { // Aggressive strategy for undervalued lots strategy.maxBid = fmv.value * (1 + 0.05); // Conservative overbid @@ -270,7 +270,7 @@ public class ValuationAnalyticsResource { * Calculates confidence score based on number and quality of comparables */ private double calculateFMVConfidence(int comparableCount, double totalWeight) { - double confidence = 0.5; // Base confidence + var confidence = 0.5; // Base confidence // Boost for more comparables confidence += Math.min(comparableCount * 0.05, 0.3); diff --git a/src/test/java/auctiora/DatabaseServiceTest.java b/src/test/java/auctiora/DatabaseServiceTest.java index c0759a5..b1eb482 100644 --- a/src/test/java/auctiora/DatabaseServiceTest.java +++ b/src/test/java/auctiora/DatabaseServiceTest.java @@ -22,6 +22,13 @@ class DatabaseServiceTest { @BeforeAll void setUp() throws SQLException { + // Load SQLite JDBC driver + try { + Class.forName("org.sqlite.JDBC"); + } catch (ClassNotFoundException e) { + throw new SQLException("SQLite JDBC driver not found", e); + } + testDbPath = "test_database_" + System.currentTimeMillis() + ".db"; db = new DatabaseService(testDbPath); db.ensureSchema(); diff --git a/src/test/java/auctiora/ImageProcessingServiceTest.java b/src/test/java/auctiora/ImageProcessingServiceTest.java index 271240f..39e2382 100644 --- a/src/test/java/auctiora/ImageProcessingServiceTest.java +++ b/src/test/java/auctiora/ImageProcessingServiceTest.java @@ -20,37 +20,51 @@ class ImageProcessingServiceTest { private DatabaseService mockDb; private ObjectDetectionService mockDetector; private ImageProcessingService service; + private java.io.File testImage; @BeforeEach - void setUp() { + void setUp() throws Exception { mockDb = mock(DatabaseService.class); mockDetector = mock(ObjectDetectionService.class); service = new ImageProcessingService(mockDb, mockDetector); + + // Create a temporary test image file + testImage = java.io.File.createTempFile("test_image_", ".jpg"); + testImage.deleteOnExit(); + // Write minimal JPEG header to make it a valid file + try (var out = new java.io.FileOutputStream(testImage)) { + out.write(new byte[]{(byte)0xFF, (byte)0xD8, (byte)0xFF, (byte)0xE0}); + } } @Test @DisplayName("Should process single image and update labels") void testProcessImage() throws SQLException { - // Mock object detection - when(mockDetector.detectObjects("/path/to/image.jpg")) + // Normalize path (convert backslashes to forward slashes) + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + + // Mock object detection with normalized path + when(mockDetector.detectObjects(normalizedPath)) .thenReturn(List.of("car", "vehicle")); // Process image - boolean result = service.processImage(1, "/path/to/image.jpg", 12345); + boolean result = service.processImage(1, testImage.getAbsolutePath(), 12345); // Verify success assertTrue(result); - verify(mockDetector).detectObjects("/path/to/image.jpg"); + verify(mockDetector).detectObjects(normalizedPath); verify(mockDb).updateImageLabels(1, List.of("car", "vehicle")); } @Test @DisplayName("Should handle empty detection results") void testProcessImageWithNoDetections() throws SQLException { - when(mockDetector.detectObjects(anyString())) + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + + when(mockDetector.detectObjects(normalizedPath)) .thenReturn(List.of()); - boolean result = service.processImage(2, "/path/to/image2.jpg", 12346); + boolean result = service.processImage(2, testImage.getAbsolutePath(), 12346); assertTrue(result); verify(mockDb).updateImageLabels(2, List.of()); @@ -59,14 +73,16 @@ class ImageProcessingServiceTest { @Test @DisplayName("Should handle database error gracefully") void testProcessImageDatabaseError() throws SQLException { - when(mockDetector.detectObjects(anyString())) + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + + when(mockDetector.detectObjects(normalizedPath)) .thenReturn(List.of("object")); doThrow(new SQLException("Database error")) .when(mockDb).updateImageLabels(anyInt(), anyList()); // Should return false on error - boolean result = service.processImage(3, "/path/to/image3.jpg", 12347); + boolean result = service.processImage(3, testImage.getAbsolutePath(), 12347); assertFalse(result); } @@ -84,13 +100,15 @@ class ImageProcessingServiceTest { @Test @DisplayName("Should process pending images batch") void testProcessPendingImages() throws SQLException { - // Mock pending images from database + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + + // Mock pending images from database - use real test image path when(mockDb.getImagesNeedingDetection()).thenReturn(List.of( - new DatabaseService.ImageDetectionRecord(1, 100L, "/images/100/001.jpg"), - new DatabaseService.ImageDetectionRecord(2, 101L, "/images/101/001.jpg") + new DatabaseService.ImageDetectionRecord(1, 100L, testImage.getAbsolutePath()), + new DatabaseService.ImageDetectionRecord(2, 101L, testImage.getAbsolutePath()) )); - when(mockDetector.detectObjects(anyString())) + when(mockDetector.detectObjects(normalizedPath)) .thenReturn(List.of("item1")) .thenReturn(List.of("item2")); @@ -103,7 +121,7 @@ class ImageProcessingServiceTest { // Verify all images were processed verify(mockDb).getImagesNeedingDetection(); - verify(mockDetector, times(2)).detectObjects(anyString()); + verify(mockDetector, times(2)).detectObjects(normalizedPath); verify(mockDb, times(2)).updateImageLabels(anyInt(), anyList()); } @@ -121,15 +139,16 @@ class ImageProcessingServiceTest { @Test @DisplayName("Should continue processing after single image failure") void testProcessPendingImagesWithFailure() throws SQLException { + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + when(mockDb.getImagesNeedingDetection()).thenReturn(List.of( - new DatabaseService.ImageDetectionRecord(1, 100L, "/images/100/001.jpg"), - new DatabaseService.ImageDetectionRecord(2, 101L, "/images/101/001.jpg") + new DatabaseService.ImageDetectionRecord(1, 100L, testImage.getAbsolutePath()), + new DatabaseService.ImageDetectionRecord(2, 101L, testImage.getAbsolutePath()) )); // First image fails, second succeeds - when(mockDetector.detectObjects("/images/100/001.jpg")) - .thenThrow(new RuntimeException("Detection error")); - when(mockDetector.detectObjects("/images/101/001.jpg")) + when(mockDetector.detectObjects(normalizedPath)) + .thenThrow(new RuntimeException("Detection error")) .thenReturn(List.of("item")); when(mockDb.getImageLabels(2)) @@ -138,7 +157,7 @@ class ImageProcessingServiceTest { service.processPendingImages(); // Verify second image was still processed - verify(mockDetector, times(2)).detectObjects(anyString()); + verify(mockDetector, times(2)).detectObjects(normalizedPath); } @Test @@ -154,10 +173,12 @@ class ImageProcessingServiceTest { @Test @DisplayName("Should process images with multiple detected objects") void testProcessImageMultipleDetections() throws SQLException { - when(mockDetector.detectObjects(anyString())) + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + + when(mockDetector.detectObjects(normalizedPath)) .thenReturn(List.of("car", "truck", "vehicle", "road")); - boolean result = service.processImage(5, "/path/to/image5.jpg", 12349); + boolean result = service.processImage(5, testImage.getAbsolutePath(), 12349); assertTrue(result); verify(mockDb).updateImageLabels(5, List.of("car", "truck", "vehicle", "road"));