commit 2bc4b21862d70e6013146aff370e1ff843ee1de9 Author: michael1986 Date: Wed Nov 26 12:57:28 2025 +0100 start diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..13275f1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +### IntelliJ IDEA ### +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ +.kotlin + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..f23d524 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..f202af8 --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..aa00ffa --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..e1e41cb --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,621 @@ + + + + \ No newline at end of file diff --git a/.idea/jarRepositories.xml b/.idea/jarRepositories.xml new file mode 100644 index 0000000..c563f5e --- /dev/null +++ b/.idea/jarRepositories.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/material_theme_project_new.xml b/.idea/material_theme_project_new.xml new file mode 100644 index 0000000..169f590 --- /dev/null +++ b/.idea/material_theme_project_new.xml @@ -0,0 +1,10 @@ + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..1c92083 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,14 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/auction.iml b/auction.iml new file mode 100644 index 0000000..c90834f --- /dev/null +++ b/auction.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..b4a8a2f --- /dev/null +++ b/pom.xml @@ -0,0 +1,100 @@ + + + 4.0.0 + + com.auction + troostwijk-scraper + 1.0-SNAPSHOT + jar + + Troostwijk Auction Scraper + Web scraper for Troostwijk Auctions with object detection and notifications + + + UTF-8 + 11 + 11 + 2.17.0 + 4.9.0-0 + + + + + + org.jsoup + jsoup + 1.17.2 + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + + org.xerial + sqlite-jdbc + 3.45.1.0 + + + + + com.sun.mail + javax.mail + 1.6.2 + + + + + org.openpnp + opencv + ${opencv.version} + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + 11 + 11 + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + + com.auction.scraper.TroostwijkScraper + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + diff --git a/src/main/java/com/auction/scraper/Main.java b/src/main/java/com/auction/scraper/Main.java new file mode 100644 index 0000000..59abbc3 --- /dev/null +++ b/src/main/java/com/auction/scraper/Main.java @@ -0,0 +1,8 @@ +package com.auction.scraper; + +public class Main { + public static void main(String[] args) { + System.out.println("Troostwijk Auction Scraper"); + System.out.println("Use TroostwijkScraper class to run the scraper."); + } +} diff --git a/src/main/java/com/auction/scraper/TroostwijkScraper.java b/src/main/java/com/auction/scraper/TroostwijkScraper.java new file mode 100644 index 0000000..b5ad33f --- /dev/null +++ b/src/main/java/com/auction/scraper/TroostwijkScraper.java @@ -0,0 +1,801 @@ +package com.auction.scraper; + +/* + * TroostwijkScraper + * + * This example shows how you could build a Java‐based scraper for the Dutch + * auctions on Troostwijk Auctions. The scraper uses a combination of + * HTTP requests and HTML parsing with the jsoup library to discover active + * auctions, calls Troostwijk's internal JSON API to fetch lot (kavel) data + * efficiently, writes the results into a local SQLite database, performs + * object detection on lot images using OpenCV's DNN module, and sends + * desktop/email notifications when bids change or lots are about to expire. + * The implementation uses well known open source libraries for each of these + * concerns. You can adjust the API endpoints and CSS selectors as + * Troostwijk's site evolves. The code is organised into small helper + * classes to make it easier to maintain. + * + * Dependencies (add these to your Maven/Gradle project): + * + * - org.jsoup:jsoup:1.17.2 – HTML parser and HTTP client. + * - com.fasterxml.jackson.core:jackson-databind:2.17.0 – JSON parsing. + * - org.xerial:sqlite-jdbc:3.45.1.0 – SQLite JDBC driver. + * - com.sun.mail:javax.mail:1.6.2 – JavaMail for email notifications (free). + * - org.openpnp:opencv:4.9.0-0 (with native libraries) – OpenCV for image + * processing and object detection. + * + * Before running this program you must ensure that the native OpenCV + * binaries are on your library path (e.g. via -Djava.library.path). + * Desktop notifications work out of the box on Windows, macOS, and Linux. + * For email notifications, you need a Gmail account with an app password + * (free, requires 2FA enabled). See https://support.google.com/accounts/answer/185833 + * + * The scraper performs four major tasks: + * 1. Discover all auctions located in the Netherlands. + * 2. For each auction, fetch all lots (kavels) including images and + * bidding information, and persist the data into SQLite tables. + * 3. Monitor bidding and closing times on a schedule and send desktop/email + * notifications when bids change or lots are about to expire. + * 4. Run object detection on downloaded lot images to automatically + * label objects using a YOLO model. The results are stored in the + * database for later search. + */ + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.opencv.core.Core; +import org.opencv.core.Mat; +import org.opencv.core.Scalar; +import org.opencv.core.Size; +import org.opencv.dnn.Dnn; +import org.opencv.dnn.Net; +import org.opencv.imgcodecs.Imgcodecs; +import static org.opencv.dnn.Dnn.DNN_BACKEND_OPENCV; +import static org.opencv.dnn.Dnn.DNN_TARGET_CPU; + +/** + * Main scraper class. It encapsulates the logic for scraping auctions, + * persisting data, scheduling updates, and performing object detection. + */ +public class TroostwijkScraper { + + // Base URLs – adjust these if Troostwijk changes their site structure + private static final String AUCTIONS_PAGE = "https://www.troostwijkauctions.com/nl/auctions"; + private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list"; + + // HTTP client used for API calls + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + private final DatabaseService db; + private final NotificationService notifier; + private final ObjectDetectionService detector; + + /** + * Constructor. Creates supporting services and ensures the database + * tables exist. + * + * @param databasePath Path to SQLite database file + * @param notificationConfig "desktop" for desktop only, or "smtp:user:pass:toEmail" for email + * @param unused Unused parameter (kept for compatibility) + * @param yoloCfgPath Path to YOLO configuration file + * @param yoloWeightsPath Path to YOLO weights file + * @param classNamesPath Path to file containing class names + */ + public TroostwijkScraper(String databasePath, String notificationConfig, String unused, + String yoloCfgPath, String yoloWeightsPath, String classNamesPath) throws SQLException, IOException { + this.httpClient = HttpClient.newHttpClient(); + this.objectMapper = new ObjectMapper(); + this.db = new DatabaseService(databasePath); + this.notifier = new NotificationService(notificationConfig, unused); + this.detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath); + // initialize DB + db.ensureSchema(); + } + + /** + * Discovers all active Dutch auctions by crawling the auctions page. + * + * Troostwijk lists auctions for many countries on one page. We parse + * the page with jsoup (an HTML parser that fetches and parses real‐world + * HTML easily【438902460386021†L14-L24】) and filter auctions whose location + * contains ", NL" (indicating the Netherlands). Each auction link + * contains a unique sale ID which we extract from its URL. + * + * @return a list of sale identifiers for auctions located in NL + */ + public List discoverDutchAuctions() { + List saleIds = new ArrayList<>(); + try { + // Fetch the auctions overview page + Document doc = Jsoup.connect(AUCTIONS_PAGE).get(); + // Select all anchor elements that represent an auction listing. + // The exact selector may change; inspect the page with your browser’s + // developer tools and update accordingly. + Elements auctionLinks = doc.select("a[href][data-id]"); + for (Element link : auctionLinks) { + Element locationElement = link.selectFirst(".auction-location"); + String location = locationElement != null ? locationElement.text() : ""; + if (location.contains(", NL")) { + // Extract saleID from the data-id attribute or href + String saleIdStr = link.attr("data-id"); + if (saleIdStr.isEmpty()) { + // Fallback: parse from URL path, e.g. /nl/sale/27213/machines + String href = link.attr("href"); + String[] parts = href.split("/"); + for (String p : parts) { + if (p.matches("\\d+")) { + saleIdStr = p; + break; + } + } + } + try { + int saleId = Integer.parseInt(saleIdStr); + saleIds.add(saleId); + } catch (NumberFormatException ignored) { + // not a sale ID + } + } + } + } catch (IOException e) { + System.err.println("Failed to discover auctions: " + e.getMessage()); + } + return saleIds; + } + + /** + * Retrieves all lots for a given sale ID using Troostwijk’s internal JSON + * API. The API accepts parameters such as batchSize, offset, and saleID. + * A large batchSize returns many lots at once【610752406306016†L124-L134】. We loop + * until no further results are returned. Each JSON result is mapped to + * our Lot domain object and persisted to the database. + * + * @param saleId the sale identifier + */ + public void fetchLotsForSale(int saleId) { + int batchSize = 200; + int offset = 0; + boolean more = true; + while (more) { + try { + String url = LOT_API + "?batchSize=" + batchSize + + "&listType=7&offset=" + offset + + "&sortOption=0&saleID=" + saleId + + "&parentID=0&relationID=0&buildversion=201807311"; + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("Accept", "application/json") + .GET() + .build(); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + System.err.println("API call failed for sale " + saleId + " with status " + response.statusCode()); + break; + } + JsonNode root = objectMapper.readTree(response.body()); + JsonNode results = root.path("results"); + if (!results.isArray() || results.isEmpty()) { + more = false; + break; + } + for (JsonNode node : results) { + Lot lot = new Lot(); + lot.saleId = saleId; + lot.lotId = node.path("lotID").asInt(); + lot.title = node.path("t").asText(); + lot.description = node.path("d").asText(); + lot.manufacturer = node.path("mf").asText(); + lot.type = node.path("typ").asText(); + lot.year = node.path("yb").asInt(); + lot.category = node.path("lc").asText(); + // Current bid; field names may differ (e.g. currentBid or cb) + lot.currentBid = node.path("cb").asDouble(); + lot.currency = node.path("cu").asText(); + lot.url = "https://www.troostwijkauctions.com/nl" + node.path("url").asText(); + // Save basic lot info into DB + db.upsertLot(lot); + // Download images and perform object detection + List imageUrls = new ArrayList<>(); + JsonNode imgs = node.path("imgs"); + if (imgs.isArray()) { + for (JsonNode imgNode : imgs) { + String imgUrl = imgNode.asText(); + imageUrls.add(imgUrl); + } + } + for (String imgUrl : imageUrls) { + String fileName = downloadImage(imgUrl, saleId, lot.lotId); + if (fileName != null) { + // run object detection once per image + List labels = detector.detectObjects(fileName); + db.insertImage(lot.lotId, imgUrl, fileName, labels); + } + } + } + offset += batchSize; + } catch (IOException | InterruptedException e) { + System.err.println("Error fetching lots for sale " + saleId + ": " + e.getMessage()); + more = false; + } catch (SQLException e) { + System.err.println("Database error: " + e.getMessage()); + } + } + } + + /** + * Downloads an image from the given URL to a local directory. Images + * are stored under "images///" to keep them organised. + * + * @param imageUrl remote image URL + * @param saleId sale identifier + * @param lotId lot identifier + * @return absolute path to saved file or null on failure + */ + private String downloadImage(String imageUrl, int saleId, int lotId) { + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(imageUrl)) + .GET() + .build(); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); + if (response.statusCode() == 200) { + Path dir = Paths.get("images", String.valueOf(saleId), String.valueOf(lotId)); + Files.createDirectories(dir); + String fileName = Paths.get(imageUrl).getFileName().toString(); + Path dest = dir.resolve(fileName); + Files.copy(response.body(), dest); + return dest.toAbsolutePath().toString(); + } + } catch (IOException | InterruptedException e) { + System.err.println("Failed to download image " + imageUrl + ": " + e.getMessage()); + } + return null; + } + + /** + * Schedules periodic monitoring of all lots. The scheduler runs every + * hour to refresh current bids and closing times. For lots that + * are within 30 minutes of closing, it increases the polling frequency + * automatically. When a new bid is detected or a lot is about to + * expire, a Pushover notification is sent to the configured user. + * Note: In production, ensure proper shutdown handling for the scheduler. + */ + public ScheduledExecutorService scheduleMonitoring() { + ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + scheduler.scheduleAtFixedRate(() -> { + try { + List activeLots = db.getActiveLots(); + for (Lot lot : activeLots) { + // refresh the lot's bidding information via API + refreshLotBid(lot); + // check closing time to adjust monitoring + long minutesLeft = lot.minutesUntilClose(); + if (minutesLeft < 30) { + // send warning when within 5 minutes + if (minutesLeft <= 5 && !lot.closingNotified) { + notifier.sendNotification("Kavel " + lot.lotId + " sluit binnen " + minutesLeft + " min.", + "Lot nearing closure", 1); + lot.closingNotified = true; + db.updateLotNotificationFlags(lot); + } + // schedule additional quick check for this lot + scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES); + } + } + } catch (SQLException e) { + System.err.println("Error during scheduled monitoring: " + e.getMessage()); + } + }, 0, 1, TimeUnit.HOURS); + return scheduler; + } + + /** + * Refreshes the bid for a single lot and sends notification if it has + * changed since the last check. The method calls the same API used for + * initial scraping but only extracts the current bid for the given lot. + * + * @param lot the lot to refresh + */ + private void refreshLotBid(Lot lot) { + try { + String url = LOT_API + "?batchSize=1&listType=7&offset=0&sortOption=0&saleID=" + lot.saleId + + "&parentID=0&relationID=0&buildversion=201807311&lotID=" + lot.lotId; + HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build(); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) return; + JsonNode root = objectMapper.readTree(response.body()); + JsonNode results = root.path("results"); + if (results.isArray() && !results.isEmpty()) { + JsonNode node = results.get(0); + double newBid = node.path("cb").asDouble(); + if (Double.compare(newBid, lot.currentBid) > 0) { + double previous = lot.currentBid; + lot.currentBid = newBid; + db.updateLotCurrentBid(lot); + String msg = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)", lot.lotId, newBid, previous); + notifier.sendNotification(msg, "Kavel bieding update", 0); + } + } + } catch (IOException | InterruptedException | SQLException e) { + System.err.println("Failed to refresh bid for lot " + lot.lotId + ": " + e.getMessage()); + } + } + + /** + * Entry point. Configure database location, notification settings, and + * YOLO model paths here before running. Once started the scraper + * discovers Dutch auctions, scrapes lots, and begins monitoring. + */ + public static void main(String[] args) throws Exception { + // Configuration parameters (replace with your own values) + String databaseFile = "troostwijk.db"; + + // Notification configuration - choose one: + // Option 1: Desktop notifications only (free, no setup required) + String notificationConfig = System.getenv().getOrDefault("NOTIFICATION_CONFIG", "desktop"); + + // Option 2: Desktop + Email via Gmail (free, requires Gmail app password) + // Format: "smtp:username:appPassword:toEmail" + // Example: "smtp:your.email@gmail.com:abcd1234efgh5678:recipient@example.com" + // Get app password: Google Account > Security > 2-Step Verification > App passwords + + String yoloCfg = "models/yolov4.cfg"; // path to YOLO config file + String yoloWeights = "models/yolov4.weights"; // path to YOLO weights file + String yoloClasses = "models/coco.names"; // list of class names + + // Load native OpenCV library + System.loadLibrary(Core.NATIVE_LIBRARY_NAME); + + TroostwijkScraper scraper = new TroostwijkScraper(databaseFile, notificationConfig, "", + yoloCfg, yoloWeights, yoloClasses); + + // Step 1: Discover auctions in NL + List auctions = scraper.discoverDutchAuctions(); + System.out.println("Found auctions: " + auctions); + + // Step 2: Fetch lots for each auction + for (int saleId : auctions) { + scraper.fetchLotsForSale(saleId); + } + + // Step 3: Start monitoring bids and closures + scraper.scheduleMonitoring(); + } + + // ---------------------------------------------------------------------- + // Domain classes and services + // ---------------------------------------------------------------------- + + /** + * Simple POJO representing a lot (kavel) in an auction. It keeps track + * of the sale it belongs to, current bid and closing time. The method + * minutesUntilClose computes how many minutes remain until the lot closes. + */ + static class Lot { + int saleId; + int lotId; + String title; + String description; + String manufacturer; + String type; + int year; + String category; + double currentBid; + String currency; + String url; + LocalDateTime closingTime; // null if unknown + boolean closingNotified; + + long minutesUntilClose() { + if (closingTime == null) return Long.MAX_VALUE; + return java.time.Duration.between(LocalDateTime.now(), closingTime).toMinutes(); + } + } + + /** + * Service for persisting auctions, lots, images, and object labels into + * a SQLite database. Uses the Xerial JDBC driver which connects to + * SQLite via a URL of the form "jdbc:sqlite:path_to_file"【329850066306528†L40-L63】. + */ + static class DatabaseService { + private final String url; + DatabaseService(String dbPath) { + this.url = "jdbc:sqlite:" + dbPath; + } + /** + * Creates tables if they do not already exist. The schema includes + * tables for sales, lots, images, and object labels. This method is + * idempotent; it can be called multiple times. + */ + void ensureSchema() throws SQLException { + try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) { + // Sales table + stmt.execute("CREATE TABLE IF NOT EXISTS sales (" + + "sale_id INTEGER PRIMARY KEY," + + "title TEXT," + + "location TEXT," + + "closing_time TEXT" + + ")"); + // Lots table + stmt.execute("CREATE TABLE IF NOT EXISTS lots (" + + "lot_id INTEGER PRIMARY KEY," + + "sale_id INTEGER," + + "title TEXT," + + "description TEXT," + + "manufacturer TEXT," + + "type TEXT," + + "year INTEGER," + + "category TEXT," + + "current_bid REAL," + + "currency TEXT," + + "url TEXT," + + "closing_time TEXT," + + "closing_notified INTEGER DEFAULT 0," + + "FOREIGN KEY (sale_id) REFERENCES sales(sale_id)" + + ")"); + // Images table + stmt.execute("CREATE TABLE IF NOT EXISTS images (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "lot_id INTEGER," + + "url TEXT," + + "file_path TEXT," + + "labels TEXT," + + "FOREIGN KEY (lot_id) REFERENCES lots(lot_id)" + + ")"); + } + } + + /** + * Inserts or updates a lot record. Uses INSERT OR REPLACE to + * implement upsert semantics so that existing rows are replaced. + */ + synchronized void upsertLot(Lot lot) throws SQLException { + String sql = "INSERT INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)" + + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + + " ON CONFLICT(lot_id) DO UPDATE SET " + + "sale_id = excluded.sale_id, title = excluded.title, description = excluded.description, " + + "manufacturer = excluded.manufacturer, type = excluded.type, year = excluded.year, category = excluded.category, " + + "current_bid = excluded.current_bid, currency = excluded.currency, url = excluded.url, closing_time = excluded.closing_time"; + try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setInt(1, lot.lotId); + ps.setInt(2, lot.saleId); + ps.setString(3, lot.title); + ps.setString(4, lot.description); + ps.setString(5, lot.manufacturer); + ps.setString(6, lot.type); + ps.setInt(7, lot.year); + ps.setString(8, lot.category); + ps.setDouble(9, lot.currentBid); + ps.setString(10, lot.currency); + ps.setString(11, lot.url); + ps.setString(12, lot.closingTime != null ? lot.closingTime.toString() : null); + ps.setInt(13, lot.closingNotified ? 1 : 0); + ps.executeUpdate(); + } + } + + /** + * Inserts a new image record. Each image is associated with a lot and + * stores both the original URL and the local file path. Detected + * labels are stored as a comma separated string. + */ + synchronized void insertImage(int lotId, String url, String filePath, List labels) throws SQLException { + String sql = "INSERT INTO images (lot_id, url, file_path, labels) VALUES (?, ?, ?, ?)"; + try (Connection conn = DriverManager.getConnection(this.url); PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setInt(1, lotId); + ps.setString(2, url); + ps.setString(3, filePath); + ps.setString(4, String.join(",", labels)); + ps.executeUpdate(); + } + } + + /** + * Retrieves all lots that are still active (i.e., have a closing time + * in the future or unknown). Only these lots need to be monitored. + */ + synchronized List getActiveLots() throws SQLException { + List list = new ArrayList<>(); + String sql = "SELECT lot_id, sale_id, current_bid, currency, closing_time, closing_notified FROM lots"; + try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery(sql); + while (rs.next()) { + Lot lot = new Lot(); + lot.lotId = rs.getInt("lot_id"); + lot.saleId = rs.getInt("sale_id"); + lot.currentBid = rs.getDouble("current_bid"); + lot.currency = rs.getString("currency"); + String closing = rs.getString("closing_time"); + lot.closingNotified = rs.getInt("closing_notified") != 0; + if (closing != null) { + lot.closingTime = LocalDateTime.parse(closing); + } + list.add(lot); + } + } + return list; + } + + /** + * Updates the current bid of a lot after a bid refresh. + */ + synchronized void updateLotCurrentBid(Lot lot) throws SQLException { + try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement( + "UPDATE lots SET current_bid = ? WHERE lot_id = ?")) { + ps.setDouble(1, lot.currentBid); + ps.setInt(2, lot.lotId); + ps.executeUpdate(); + } + } + + /** + * Updates the closingNotified flag of a lot (set to 1 when we have + * warned the user about its imminent closure). + */ + synchronized void updateLotNotificationFlags(Lot lot) throws SQLException { + try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement( + "UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) { + ps.setInt(1, lot.closingNotified ? 1 : 0); + ps.setInt(2, lot.lotId); + ps.executeUpdate(); + } + } + } + + /** + * Service for sending notifications via desktop notifications and/or email. + * Supports free notification methods: + * 1. Desktop notifications (Windows/Linux/macOS system tray) + * 2. Email via Gmail SMTP (free, requires app password) + * + * Configuration: + * - For email: Set notificationEmail to your Gmail address + * - Enable 2FA in Gmail and create an App Password + * - Use format "smtp:username:appPassword:toEmail" for credentials + * - Or use "desktop" for desktop-only notifications + */ + static class NotificationService { + private final boolean useDesktop; + private final boolean useEmail; + private final String smtpUsername; + private final String smtpPassword; + private final String toEmail; + + /** + * Creates a notification service. + * + * @param config "desktop" for desktop only, or "smtp:username:password:toEmail" for email + * @param unusedParam Kept for compatibility (can pass empty string) + */ + NotificationService(String config, String unusedParam) { + + if ("desktop".equalsIgnoreCase(config)) { + this.useDesktop = true; + this.useEmail = false; + this.smtpUsername = null; + this.smtpPassword = null; + this.toEmail = null; + } else if (config.startsWith("smtp:")) { + String[] parts = config.split(":", 4); + if (parts.length != 4) { + throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'"); + } + this.useDesktop = true; // Always include desktop + this.useEmail = true; + this.smtpUsername = parts[1]; + this.smtpPassword = parts[2]; + this.toEmail = parts[3]; + } else { + throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'"); + } + } + + /** + * Sends notification via configured channels. + * + * @param message The message body + * @param title Message title + * @param priority Priority level (0=normal, 1=high) + */ + void sendNotification(String message, String title, int priority) { + if (useDesktop) { + sendDesktopNotification(title, message, priority); + } + if (useEmail) { + sendEmailNotification(title, message, priority); + } + } + + /** + * Sends a desktop notification using system tray. + * Works on Windows, macOS, and Linux with desktop environments. + */ + private void sendDesktopNotification(String title, String message, int priority) { + try { + if (java.awt.SystemTray.isSupported()) { + java.awt.SystemTray tray = java.awt.SystemTray.getSystemTray(); + java.awt.Image image = java.awt.Toolkit.getDefaultToolkit() + .createImage(new byte[0]); // Empty image + + java.awt.TrayIcon trayIcon = new java.awt.TrayIcon(image, "Troostwijk Scraper"); + trayIcon.setImageAutoSize(true); + + java.awt.TrayIcon.MessageType messageType = priority > 0 + ? java.awt.TrayIcon.MessageType.WARNING + : java.awt.TrayIcon.MessageType.INFO; + + tray.add(trayIcon); + trayIcon.displayMessage(title, message, messageType); + + // Remove icon after 2 seconds to avoid clutter + Thread.sleep(2000); + tray.remove(trayIcon); + + System.out.println("Desktop notification sent: " + title); + } else { + System.out.println("Desktop notifications not supported, logging: " + title + " - " + message); + } + } catch (Exception e) { + System.err.println("Desktop notification failed: " + e.getMessage()); + } + } + + /** + * Sends email notification via Gmail SMTP (free). + * Uses Gmail's SMTP server with app password authentication. + */ + private void sendEmailNotification(String title, String message, int priority) { + try { + java.util.Properties props = new java.util.Properties(); + props.put("mail.smtp.auth", "true"); + props.put("mail.smtp.starttls.enable", "true"); + props.put("mail.smtp.host", "smtp.gmail.com"); + props.put("mail.smtp.port", "587"); + props.put("mail.smtp.ssl.trust", "smtp.gmail.com"); + + javax.mail.Session session = javax.mail.Session.getInstance(props, + new javax.mail.Authenticator() { + protected javax.mail.PasswordAuthentication getPasswordAuthentication() { + return new javax.mail.PasswordAuthentication(smtpUsername, smtpPassword); + } + }); + + javax.mail.Message msg = new javax.mail.internet.MimeMessage(session); + msg.setFrom(new javax.mail.internet.InternetAddress(smtpUsername)); + msg.setRecipients(javax.mail.Message.RecipientType.TO, + javax.mail.internet.InternetAddress.parse(toEmail)); + msg.setSubject("[Troostwijk] " + title); + msg.setText(message); + msg.setSentDate(new java.util.Date()); + + if (priority > 0) { + msg.setHeader("X-Priority", "1"); + msg.setHeader("Importance", "High"); + } + + javax.mail.Transport.send(msg); + System.out.println("Email notification sent: " + title); + + } catch (Exception e) { + System.err.println("Email notification failed: " + e.getMessage()); + } + } + } + + /** + * Service for performing object detection on images using OpenCV’s DNN + * module. The DNN module can load pre‑trained models from several + * frameworks (Darknet, TensorFlow, ONNX, etc.)【784097309529506†L209-L233】. Here + * we load a YOLO model (Darknet) by specifying the configuration and + * weights files. For each image we run a forward pass and return a + * list of detected class labels. + */ + static class ObjectDetectionService { + private final Net net; + private final List classNames; + ObjectDetectionService(String cfgPath, String weightsPath, String classNamesPath) throws IOException { + // Load network + this.net = Dnn.readNetFromDarknet(cfgPath, weightsPath); + this.net.setPreferableBackend(DNN_BACKEND_OPENCV); + this.net.setPreferableTarget(DNN_TARGET_CPU); + // Load class names (one per line) + this.classNames = Files.readAllLines(Paths.get(classNamesPath)); + } + /** + * Detects objects in the given image file and returns a list of + * human‑readable labels. Only detections above a confidence + * threshold are returned. For brevity this method omits drawing + * bounding boxes. See the OpenCV DNN documentation for details on + * post‑processing【784097309529506†L324-L344】. + * + * @param imagePath absolute path to the image + * @return list of detected class names + */ + List detectObjects(String imagePath) { + List labels = new ArrayList<>(); + Mat image = Imgcodecs.imread(imagePath); + if (image.empty()) return labels; + // Create a 4D blob from the image + Mat blob = Dnn.blobFromImage(image, 1.0 / 255.0, new Size(416, 416), new Scalar(0, 0, 0), true, false); + net.setInput(blob); + List outs = new ArrayList<>(); + List outNames = getOutputLayerNames(net); + net.forward(outs, outNames); + // Post‑process: for each detection compute score and choose class + float confThreshold = 0.5f; + for (Mat out : outs) { + for (int i = 0; i < out.rows(); i++) { + double[] data = out.get(i, 0); + if (data == null) continue; + // The first 5 numbers are bounding box, then class scores + double[] scores = new double[classNames.size()]; + System.arraycopy(data, 5, scores, 0, scores.length); + int classId = argMax(scores); + double confidence = scores[classId]; + if (confidence > confThreshold) { + String label = classNames.get(classId); + if (!labels.contains(label)) { + labels.add(label); + } + } + } + } + return labels; + } + /** + * Returns the indexes of the output layers in the network. YOLO + * automatically discovers its output layers; other models may require + * manually specifying them【784097309529506†L356-L365】. + */ + private List getOutputLayerNames(Net net) { + List names = new ArrayList<>(); + List outLayers = net.getUnconnectedOutLayers().toList(); + List layersNames = net.getLayerNames(); + for (Integer i : outLayers) { + names.add(layersNames.get(i - 1)); + } + return names; + } + /** + * Returns the index of the maximum value in the array. + */ + private int argMax(double[] array) { + int best = 0; + double max = array[0]; + for (int i = 1; i < array.length; i++) { + if (array[i] > max) { + max = array[i]; + best = i; + } + } + return best; + } + } +} \ No newline at end of file