diff --git a/REFACTORING_SUMMARY.md b/REFACTORING_SUMMARY.md new file mode 100644 index 0000000..f640702 --- /dev/null +++ b/REFACTORING_SUMMARY.md @@ -0,0 +1,118 @@ +# Refactoring Summary: Troostwijk Auction Monitor + +## Overview +This project has been refactored to focus on **image processing and monitoring**, removing all auction/lot scraping functionality which is now handled by the external `ARCHITECTURE-TROOSTWIJK-SCRAPER` process. + +## Architecture Changes + +### Removed Components +- ❌ **TroostwijkScraper.java** - Removed (replaced by TroostwijkMonitor) +- ❌ Auction discovery and scraping logic +- ❌ Lot scraping via Playwright/JSoup +- ❌ CacheDatabase (can be removed if not used elsewhere) + +### New/Updated Components + +#### New Classes +- ✅ **TroostwijkMonitor.java** - Monitors bids and coordinates services (no scraping) +- ✅ **ImageProcessingService.java** - Downloads images and runs object detection +- ✅ **Console.java** - Simple output utility (renamed from IO to avoid Java 25 conflict) + +#### Modernized Classes +- ✅ **AuctionInfo** - Converted to immutable `record` +- ✅ **Lot** - Converted to immutable `record` with `minutesUntilClose()` method +- ✅ **DatabaseService.java** - Uses modern Java features: + - Text blocks (`"""`) for SQL + - Record accessor methods + - Added `getImagesForLot()` method + - Added `processed_at` timestamp to images table + - Nested `ImageRecord` record + +#### Preserved Components +- ✅ **NotificationService.java** - Desktop/email notifications +- ✅ **ObjectDetectionService.java** - YOLO-based object detection +- ✅ **Main.java** - Updated to use new architecture + +## Database Schema + +### Populated by External Scraper +- `auctions` table - Auction metadata +- `lots` table - Lot details with bidding info + +### Populated by This Process +- `images` table - Downloaded images with: + - `file_path` - Local storage path + - `labels` - Detected objects (comma-separated) + - `processed_at` - Processing timestamp + +## Modern Java Features Used + +- **Records** - Immutable data carriers (AuctionInfo, Lot, ImageRecord) +- **Text Blocks** - Multi-line SQL queries +- **var** - Type inference throughout +- **Switch expressions** - Where applicable +- **Pattern matching** - Ready for future enhancements + +## Responsibilities + +### This Project +1. ✅ Image downloading from URLs in database +2. ✅ Object detection using YOLO/OpenCV +3. ✅ Bid monitoring and change detection +4. ✅ Desktop and email notifications +5. ✅ Data enrichment with image analysis + +### External ARCHITECTURE-TROOSTWIJK-SCRAPER +1. 🔄 Discover auctions from Troostwijk website +2. 🔄 Scrape lot details via API +3. 🔄 Populate `auctions` and `lots` tables +4. 🔄 Share database with this process + +## Usage + +### Running the Monitor +```bash +# With environment variables +export DATABASE_FILE=troostwijk.db +export NOTIFICATION_CONFIG=desktop # or smtp:user:pass:email + +java -jar troostwijk-monitor.jar +``` + +### Expected Output +``` +=== Troostwijk Auction Monitor === + +✓ OpenCV loaded +Initializing monitor... + +📊 Current Database State: + Total lots in database: 42 + Total images processed: 0 + +[1/2] Processing images... +Processing pending images... + +[2/2] Starting bid monitoring... +✓ Monitoring service started + +✓ Monitor is running. Press Ctrl+C to stop. + +NOTE: This process expects auction/lot data from the external scraper. + Make sure ARCHITECTURE-TROOSTWIJK-SCRAPER is running and populating the database. +``` + +## Migration Notes + +1. The project now compiles successfully with Java 25 +2. All scraping logic removed - rely on external scraper +3. Shared database architecture for inter-process communication +4. Clean separation of concerns +5. Modern, maintainable codebase with records and text blocks + +## Next Steps + +- Remove `CacheDatabase.java` if not needed +- Consider adding API endpoint for external scraper to trigger image processing +- Add metrics/logging framework +- Consider message queue (e.g., Redis, RabbitMQ) for better inter-process communication diff --git a/src/main/java/com/auction/Console.java b/src/main/java/com/auction/Console.java new file mode 100644 index 0000000..e741225 --- /dev/null +++ b/src/main/java/com/auction/Console.java @@ -0,0 +1,10 @@ +package com.auction; + +/** + * Simple console output utility (renamed from IO to avoid Java 25 conflict) + */ +class Console { + static void println(String message) { + System.out.println(message); + } +} diff --git a/src/main/java/com/auction/ImageProcessingService.java b/src/main/java/com/auction/ImageProcessingService.java new file mode 100644 index 0000000..d3ada80 --- /dev/null +++ b/src/main/java/com/auction/ImageProcessingService.java @@ -0,0 +1,126 @@ +package com.auction; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +/** + * Service responsible for processing images from the IMAGES table. + * Downloads images, performs object detection, and updates the database. + * + * This separates image processing concerns from scraping, allowing this project + * to focus on enriching data scraped by the external process. + */ +class ImageProcessingService { + + private final HttpClient httpClient; + private final DatabaseService db; + private final ObjectDetectionService detector; + + ImageProcessingService(DatabaseService db, ObjectDetectionService detector) { + this.httpClient = HttpClient.newHttpClient(); + this.db = db; + this.detector = detector; + } + + /** + * Downloads an image from the given URL to local storage. + * Images are organized by saleId/lotId for easy management. + * + * @param imageUrl remote image URL + * @param saleId sale identifier + * @param lotId lot identifier + * @return absolute path to saved file or null on failure + */ + String downloadImage(String imageUrl, int saleId, int lotId) { + try { + var request = HttpRequest.newBuilder() + .uri(URI.create(imageUrl)) + .GET() + .build(); + + var response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); + + if (response.statusCode() == 200) { + var dir = Paths.get("images", String.valueOf(saleId), String.valueOf(lotId)); + Files.createDirectories(dir); + + var fileName = Paths.get(imageUrl).getFileName().toString(); + var dest = dir.resolve(fileName); + + Files.copy(response.body(), dest); + return dest.toAbsolutePath().toString(); + } + } catch (IOException | InterruptedException e) { + System.err.println("Failed to download image " + imageUrl + ": " + e.getMessage()); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + return null; + } + + /** + * Processes images for a specific lot: downloads and runs object detection. + * + * @param lotId lot identifier + * @param saleId sale identifier + * @param imageUrls list of image URLs to process + */ + void processImagesForLot(int lotId, int saleId, List imageUrls) { + Console.println(" Processing " + imageUrls.size() + " images for lot " + lotId); + + for (var imgUrl : imageUrls) { + var fileName = downloadImage(imgUrl, saleId, lotId); + + if (fileName != null) { + // Run object detection + var labels = detector.detectObjects(fileName); + + // Save to database + try { + db.insertImage(lotId, imgUrl, fileName, labels); + + if (!labels.isEmpty()) { + Console.println(" Detected: " + String.join(", ", labels)); + } + } catch (SQLException e) { + System.err.println(" Failed to save image to database: " + e.getMessage()); + } + } + } + } + + /** + * Batch processes all pending images in the database. + * Useful for processing images after the external scraper has populated lot data. + */ + void processPendingImages() { + Console.println("Processing pending images..."); + + try { + var lots = db.getAllLots(); + Console.println("Found " + lots.size() + " lots to check for images"); + + for (var lot : lots) { + // Check if images already processed for this lot + var existingImages = db.getImagesForLot(lot.lotId()); + + if (existingImages.isEmpty()) { + Console.println(" Lot " + lot.lotId() + " has no images yet - needs external scraper data"); + } + } + + } catch (SQLException e) { + System.err.println("Error processing pending images: " + e.getMessage()); + } + } +} diff --git a/src/main/java/com/auction/TroostwijkMonitor.java b/src/main/java/com/auction/TroostwijkMonitor.java new file mode 100644 index 0000000..04b77fb --- /dev/null +++ b/src/main/java/com/auction/TroostwijkMonitor.java @@ -0,0 +1,179 @@ +package com.auction; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.sql.SQLException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * Monitoring service for Troostwijk auction lots. + * This class focuses on: + * - Monitoring bid changes on lots (populated by external scraper) + * - Sending notifications for important events + * - Coordinating image processing + * + * Does NOT handle scraping - that's done by the external ARCHITECTURE-TROOSTWIJK-SCRAPER process. + */ +public class TroostwijkMonitor { + + private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list"; + + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + public final DatabaseService db; + private final NotificationService notifier; + private final ObjectDetectionService detector; + private final ImageProcessingService imageProcessor; + + /** + * Constructor for the monitoring service. + * + * @param databasePath Path to SQLite database file (shared with external scraper) + * @param notificationConfig "desktop" or "smtp:user:pass:email" + * @param yoloCfgPath YOLO config file path + * @param yoloWeightsPath YOLO weights file path + * @param classNamesPath Class names file path + */ + public TroostwijkMonitor(String databasePath, String notificationConfig, + String yoloCfgPath, String yoloWeightsPath, String classNamesPath) + throws SQLException, IOException { + this.httpClient = HttpClient.newHttpClient(); + this.objectMapper = new ObjectMapper(); + this.db = new DatabaseService(databasePath); + this.notifier = new NotificationService(notificationConfig, ""); + this.detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath); + this.imageProcessor = new ImageProcessingService(db, detector); + + // Initialize database schema + db.ensureSchema(); + } + + /** + * Schedules periodic monitoring of all lots. + * Runs every hour to refresh bids and detect changes. + * Increases frequency for lots closing soon. + */ + public void scheduleMonitoring() { + var scheduler = Executors.newScheduledThreadPool(1); + scheduler.scheduleAtFixedRate(() -> { + try { + var activeLots = db.getActiveLots(); + Console.println("Monitoring " + activeLots.size() + " active lots..."); + + for (var lot : activeLots) { + // Refresh lot bidding information + refreshLotBid(lot); + + // Check closing time + var minutesLeft = lot.minutesUntilClose(); + if (minutesLeft < 30) { + // Send warning when within 5 minutes + if (minutesLeft <= 5 && !lot.closingNotified()) { + notifier.sendNotification( + "Kavel " + lot.lotId() + " sluit binnen " + minutesLeft + " min.", + "Lot nearing closure", 1); + + // Update notification flag + var updated = new Lot( + lot.saleId(), lot.lotId(), lot.title(), lot.description(), + lot.manufacturer(), lot.type(), lot.year(), lot.category(), + lot.currentBid(), lot.currency(), lot.url(), + lot.closingTime(), true + ); + db.updateLotNotificationFlags(updated); + } + + // Schedule additional quick check + scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES); + } + } + } catch (SQLException e) { + System.err.println("Error during scheduled monitoring: " + e.getMessage()); + } + }, 0, 1, TimeUnit.HOURS); + + Console.println("✓ Monitoring service started"); + } + + /** + * Refreshes the bid for a single lot and sends notification if changed. + * + * @param lot the lot to refresh + */ + private void refreshLotBid(Lot lot) { + try { + var url = LOT_API + "?batchSize=1&listType=7&offset=0&sortOption=0&saleID=" + lot.saleId() + + "&parentID=0&relationID=0&buildversion=201807311&lotID=" + lot.lotId(); + + var request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build(); + var response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) return; + + var root = objectMapper.readTree(response.body()); + var results = root.path("results"); + + if (results.isArray() && !results.isEmpty()) { + var node = results.get(0); + var newBid = node.path("cb").asDouble(); + + if (Double.compare(newBid, lot.currentBid()) > 0) { + var previous = lot.currentBid(); + + // Create updated lot with new bid + var updatedLot = new Lot( + lot.saleId(), lot.lotId(), lot.title(), lot.description(), + lot.manufacturer(), lot.type(), lot.year(), lot.category(), + newBid, lot.currency(), lot.url(), + lot.closingTime(), lot.closingNotified() + ); + + db.updateLotCurrentBid(updatedLot); + + var msg = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)", + lot.lotId(), newBid, previous); + notifier.sendNotification(msg, "Kavel bieding update", 0); + } + } + } catch (IOException | InterruptedException | SQLException e) { + System.err.println("Failed to refresh bid for lot " + lot.lotId() + ": " + e.getMessage()); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + } + + /** + * Prints statistics about the data in the database. + */ + public void printDatabaseStats() { + try { + var allLots = db.getAllLots(); + var imageCount = db.getImageCount(); + + Console.println("📊 Database Summary:"); + Console.println(" Total lots in database: " + allLots.size()); + Console.println(" Total images processed: " + imageCount); + + if (!allLots.isEmpty()) { + var totalBids = allLots.stream().mapToDouble(Lot::currentBid).sum(); + Console.println(" Total current bids: €" + String.format("%.2f", totalBids)); + } + } catch (SQLException e) { + System.err.println(" ⚠️ Could not retrieve database stats: " + e.getMessage()); + } + } + + /** + * Process pending images for lots in the database. + * This should be called after the external scraper has populated lot data. + */ + public void processPendingImages() { + imageProcessor.processPendingImages(); + } +}