start
This commit is contained in:
Tour
10
src/main/java/com/auction/Console.java
Normal file
10
src/main/java/com/auction/Console.java
Normal file
@@ -0,0 +1,10 @@
|
||||
package com.auction;
|
||||
|
||||
/**
|
||||
* Simple console output utility (renamed from IO to avoid Java 25 conflict)
|
||||
*/
|
||||
class Console {
|
||||
static void println(String message) {
|
||||
System.out.println(message);
|
||||
}
|
||||
}
|
||||
126
src/main/java/com/auction/ImageProcessingService.java
Normal file
126
src/main/java/com/auction/ImageProcessingService.java
Normal file
@@ -0,0 +1,126 @@
|
||||
package com.auction;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Service responsible for processing images from the IMAGES table.
|
||||
* Downloads images, performs object detection, and updates the database.
|
||||
*
|
||||
* This separates image processing concerns from scraping, allowing this project
|
||||
* to focus on enriching data scraped by the external process.
|
||||
*/
|
||||
class ImageProcessingService {
|
||||
|
||||
private final HttpClient httpClient;
|
||||
private final DatabaseService db;
|
||||
private final ObjectDetectionService detector;
|
||||
|
||||
ImageProcessingService(DatabaseService db, ObjectDetectionService detector) {
|
||||
this.httpClient = HttpClient.newHttpClient();
|
||||
this.db = db;
|
||||
this.detector = detector;
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads an image from the given URL to local storage.
|
||||
* Images are organized by saleId/lotId for easy management.
|
||||
*
|
||||
* @param imageUrl remote image URL
|
||||
* @param saleId sale identifier
|
||||
* @param lotId lot identifier
|
||||
* @return absolute path to saved file or null on failure
|
||||
*/
|
||||
String downloadImage(String imageUrl, int saleId, int lotId) {
|
||||
try {
|
||||
var request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(imageUrl))
|
||||
.GET()
|
||||
.build();
|
||||
|
||||
var response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||||
|
||||
if (response.statusCode() == 200) {
|
||||
var dir = Paths.get("images", String.valueOf(saleId), String.valueOf(lotId));
|
||||
Files.createDirectories(dir);
|
||||
|
||||
var fileName = Paths.get(imageUrl).getFileName().toString();
|
||||
var dest = dir.resolve(fileName);
|
||||
|
||||
Files.copy(response.body(), dest);
|
||||
return dest.toAbsolutePath().toString();
|
||||
}
|
||||
} catch (IOException | InterruptedException e) {
|
||||
System.err.println("Failed to download image " + imageUrl + ": " + e.getMessage());
|
||||
if (e instanceof InterruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes images for a specific lot: downloads and runs object detection.
|
||||
*
|
||||
* @param lotId lot identifier
|
||||
* @param saleId sale identifier
|
||||
* @param imageUrls list of image URLs to process
|
||||
*/
|
||||
void processImagesForLot(int lotId, int saleId, List<String> imageUrls) {
|
||||
Console.println(" Processing " + imageUrls.size() + " images for lot " + lotId);
|
||||
|
||||
for (var imgUrl : imageUrls) {
|
||||
var fileName = downloadImage(imgUrl, saleId, lotId);
|
||||
|
||||
if (fileName != null) {
|
||||
// Run object detection
|
||||
var labels = detector.detectObjects(fileName);
|
||||
|
||||
// Save to database
|
||||
try {
|
||||
db.insertImage(lotId, imgUrl, fileName, labels);
|
||||
|
||||
if (!labels.isEmpty()) {
|
||||
Console.println(" Detected: " + String.join(", ", labels));
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
System.err.println(" Failed to save image to database: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch processes all pending images in the database.
|
||||
* Useful for processing images after the external scraper has populated lot data.
|
||||
*/
|
||||
void processPendingImages() {
|
||||
Console.println("Processing pending images...");
|
||||
|
||||
try {
|
||||
var lots = db.getAllLots();
|
||||
Console.println("Found " + lots.size() + " lots to check for images");
|
||||
|
||||
for (var lot : lots) {
|
||||
// Check if images already processed for this lot
|
||||
var existingImages = db.getImagesForLot(lot.lotId());
|
||||
|
||||
if (existingImages.isEmpty()) {
|
||||
Console.println(" Lot " + lot.lotId() + " has no images yet - needs external scraper data");
|
||||
}
|
||||
}
|
||||
|
||||
} catch (SQLException e) {
|
||||
System.err.println("Error processing pending images: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
179
src/main/java/com/auction/TroostwijkMonitor.java
Normal file
179
src/main/java/com/auction/TroostwijkMonitor.java
Normal file
@@ -0,0 +1,179 @@
|
||||
package com.auction;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.sql.SQLException;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* Monitoring service for Troostwijk auction lots.
|
||||
* This class focuses on:
|
||||
* - Monitoring bid changes on lots (populated by external scraper)
|
||||
* - Sending notifications for important events
|
||||
* - Coordinating image processing
|
||||
*
|
||||
* Does NOT handle scraping - that's done by the external ARCHITECTURE-TROOSTWIJK-SCRAPER process.
|
||||
*/
|
||||
public class TroostwijkMonitor {
|
||||
|
||||
private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list";
|
||||
|
||||
private final HttpClient httpClient;
|
||||
private final ObjectMapper objectMapper;
|
||||
public final DatabaseService db;
|
||||
private final NotificationService notifier;
|
||||
private final ObjectDetectionService detector;
|
||||
private final ImageProcessingService imageProcessor;
|
||||
|
||||
/**
|
||||
* Constructor for the monitoring service.
|
||||
*
|
||||
* @param databasePath Path to SQLite database file (shared with external scraper)
|
||||
* @param notificationConfig "desktop" or "smtp:user:pass:email"
|
||||
* @param yoloCfgPath YOLO config file path
|
||||
* @param yoloWeightsPath YOLO weights file path
|
||||
* @param classNamesPath Class names file path
|
||||
*/
|
||||
public TroostwijkMonitor(String databasePath, String notificationConfig,
|
||||
String yoloCfgPath, String yoloWeightsPath, String classNamesPath)
|
||||
throws SQLException, IOException {
|
||||
this.httpClient = HttpClient.newHttpClient();
|
||||
this.objectMapper = new ObjectMapper();
|
||||
this.db = new DatabaseService(databasePath);
|
||||
this.notifier = new NotificationService(notificationConfig, "");
|
||||
this.detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
|
||||
this.imageProcessor = new ImageProcessingService(db, detector);
|
||||
|
||||
// Initialize database schema
|
||||
db.ensureSchema();
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedules periodic monitoring of all lots.
|
||||
* Runs every hour to refresh bids and detect changes.
|
||||
* Increases frequency for lots closing soon.
|
||||
*/
|
||||
public void scheduleMonitoring() {
|
||||
var scheduler = Executors.newScheduledThreadPool(1);
|
||||
scheduler.scheduleAtFixedRate(() -> {
|
||||
try {
|
||||
var activeLots = db.getActiveLots();
|
||||
Console.println("Monitoring " + activeLots.size() + " active lots...");
|
||||
|
||||
for (var lot : activeLots) {
|
||||
// Refresh lot bidding information
|
||||
refreshLotBid(lot);
|
||||
|
||||
// Check closing time
|
||||
var minutesLeft = lot.minutesUntilClose();
|
||||
if (minutesLeft < 30) {
|
||||
// Send warning when within 5 minutes
|
||||
if (minutesLeft <= 5 && !lot.closingNotified()) {
|
||||
notifier.sendNotification(
|
||||
"Kavel " + lot.lotId() + " sluit binnen " + minutesLeft + " min.",
|
||||
"Lot nearing closure", 1);
|
||||
|
||||
// Update notification flag
|
||||
var updated = new Lot(
|
||||
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
|
||||
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
|
||||
lot.currentBid(), lot.currency(), lot.url(),
|
||||
lot.closingTime(), true
|
||||
);
|
||||
db.updateLotNotificationFlags(updated);
|
||||
}
|
||||
|
||||
// Schedule additional quick check
|
||||
scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
|
||||
}
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
System.err.println("Error during scheduled monitoring: " + e.getMessage());
|
||||
}
|
||||
}, 0, 1, TimeUnit.HOURS);
|
||||
|
||||
Console.println("✓ Monitoring service started");
|
||||
}
|
||||
|
||||
/**
|
||||
* Refreshes the bid for a single lot and sends notification if changed.
|
||||
*
|
||||
* @param lot the lot to refresh
|
||||
*/
|
||||
private void refreshLotBid(Lot lot) {
|
||||
try {
|
||||
var url = LOT_API + "?batchSize=1&listType=7&offset=0&sortOption=0&saleID=" + lot.saleId()
|
||||
+ "&parentID=0&relationID=0&buildversion=201807311&lotID=" + lot.lotId();
|
||||
|
||||
var request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build();
|
||||
var response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
|
||||
if (response.statusCode() != 200) return;
|
||||
|
||||
var root = objectMapper.readTree(response.body());
|
||||
var results = root.path("results");
|
||||
|
||||
if (results.isArray() && !results.isEmpty()) {
|
||||
var node = results.get(0);
|
||||
var newBid = node.path("cb").asDouble();
|
||||
|
||||
if (Double.compare(newBid, lot.currentBid()) > 0) {
|
||||
var previous = lot.currentBid();
|
||||
|
||||
// Create updated lot with new bid
|
||||
var updatedLot = new Lot(
|
||||
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
|
||||
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
|
||||
newBid, lot.currency(), lot.url(),
|
||||
lot.closingTime(), lot.closingNotified()
|
||||
);
|
||||
|
||||
db.updateLotCurrentBid(updatedLot);
|
||||
|
||||
var msg = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)",
|
||||
lot.lotId(), newBid, previous);
|
||||
notifier.sendNotification(msg, "Kavel bieding update", 0);
|
||||
}
|
||||
}
|
||||
} catch (IOException | InterruptedException | SQLException e) {
|
||||
System.err.println("Failed to refresh bid for lot " + lot.lotId() + ": " + e.getMessage());
|
||||
if (e instanceof InterruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints statistics about the data in the database.
|
||||
*/
|
||||
public void printDatabaseStats() {
|
||||
try {
|
||||
var allLots = db.getAllLots();
|
||||
var imageCount = db.getImageCount();
|
||||
|
||||
Console.println("📊 Database Summary:");
|
||||
Console.println(" Total lots in database: " + allLots.size());
|
||||
Console.println(" Total images processed: " + imageCount);
|
||||
|
||||
if (!allLots.isEmpty()) {
|
||||
var totalBids = allLots.stream().mapToDouble(Lot::currentBid).sum();
|
||||
Console.println(" Total current bids: €" + String.format("%.2f", totalBids));
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
System.err.println(" ⚠️ Could not retrieve database stats: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process pending images for lots in the database.
|
||||
* This should be called after the external scraper has populated lot data.
|
||||
*/
|
||||
public void processPendingImages() {
|
||||
imageProcessor.processPendingImages();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user