diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
new file mode 100644
index 0000000..861bf2a
--- /dev/null
+++ b/.github/workflows/deploy.yml
@@ -0,0 +1,42 @@
+name: Build and Deploy
+
+on:
+ push:
+ branches: ["main"]
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Set up Java
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: '21'
+
+ - name: Build
+ run: mvn -B clean package
+
+ - name: Upload to JFrog
+ run: |
+ curl -u "${{ secrets.JFROG_USER }}:${{ secrets.JFROG_PASS }}" \
+ -T target/*.jar \
+ "http://JFROG-SERVER/artifactory/myrepo/app-latest.jar"
+
+ deploy:
+ needs: build
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Trigger remote deploy script
+ uses: appleboy/ssh-action@v0.1.7
+ with:
+ host: ${{ secrets.SERVER_IP }}
+ username: ${{ secrets.SERVER_USER }}
+ key: ${{ secrets.SERVER_SSH_KEY }}
+ script: |
+ /opt/myapp/update.sh
diff --git a/.gitignore b/.gitignore
index 13275f1..27caa23 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,4 +27,6 @@ bin/
.vscode/
### Mac OS ###
-.DS_Store
\ No newline at end of file
+.DS_Store
+
+NUL
\ No newline at end of file
diff --git a/.idea/.name b/.idea/.name
new file mode 100644
index 0000000..d543c51
--- /dev/null
+++ b/.idea/.name
@@ -0,0 +1 @@
+scrappy
\ No newline at end of file
diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml
new file mode 100644
index 0000000..53c09db
--- /dev/null
+++ b/.idea/dataSources.xml
@@ -0,0 +1,65 @@
+
+
+
+
+ sqlite.xerial
+ true
+ org.sqlite.JDBC
+ jdbc:sqlite:$PROJECT_DIR$/troostwijk.db
+ $ProjectFileDir$
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/license.txt
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar.sha1
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.pom
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.pom.sha1
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/slf4j/slf4j-api/1.7.36/_remote.repositories
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/slf4j/slf4j-parent/1.7.36/slf4j-parent-1.7.36.pom
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/slf4j/slf4j-parent/1.7.36/slf4j-parent-1.7.36.pom.sha1
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/slf4j/slf4j-parent/1.7.36/_remote.repositories
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/xerial/sqlite-jdbc/3.45.1.0/sqlite-jdbc-3.45.1.0.jar
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/xerial/sqlite-jdbc/3.45.1.0/sqlite-jdbc-3.45.1.0.jar.sha1
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/xerial/sqlite-jdbc/3.45.1.0/sqlite-jdbc-3.45.1.0.pom
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/xerial/sqlite-jdbc/3.45.1.0/sqlite-jdbc-3.45.1.0.pom.sha1
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/xerial/sqlite-jdbc/3.45.1.0/_remote.repositories
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/xerial/sqlite-jdbc/3.45.1.0/sqlite-jdbc-3.45.1.0.jar
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.45.1/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar
+
+
+ file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.43.0/org/xerial/sqlite-jdbc/3.43.0.0/sqlite-jdbc-3.43.0.0.jar
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/material_theme_project_new.xml b/.idea/material_theme_project_new.xml
index 169f590..c3c2631 100644
--- a/.idea/material_theme_project_new.xml
+++ b/.idea/material_theme_project_new.xml
@@ -3,8 +3,16 @@
+
\ No newline at end of file
diff --git a/src/main/java/com/auction/scraper/TroostwijkScraper.java b/src/main/java/com/auction/scraper/TroostwijkScraper.java
index 6d64fd8..c3d0098 100644
--- a/src/main/java/com/auction/scraper/TroostwijkScraper.java
+++ b/src/main/java/com/auction/scraper/TroostwijkScraper.java
@@ -85,760 +85,869 @@ import static org.opencv.dnn.Dnn.DNN_TARGET_CPU;
* persisting data, scheduling updates, and performing object detection.
*/
public class TroostwijkScraper {
-
- // Base URLs – adjust these if Troostwijk changes their site structure
- private static final String AUCTIONS_PAGE = "https://www.troostwijkauctions.com/nl/auctions";
- private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list";
-
- // HTTP client used for API calls
- private final HttpClient httpClient;
- private final ObjectMapper objectMapper;
- private final DatabaseService db;
- private final NotificationService notifier;
- private final ObjectDetectionService detector;
-
- /**
- * Constructor. Creates supporting services and ensures the database
- * tables exist.
- *
- * @param databasePath Path to SQLite database file
- * @param notificationConfig "desktop" for desktop only, or "smtp:user:pass:toEmail" for email
- * @param unused Unused parameter (kept for compatibility)
- * @param yoloCfgPath Path to YOLO configuration file
- * @param yoloWeightsPath Path to YOLO weights file
- * @param classNamesPath Path to file containing class names
- */
- public TroostwijkScraper(String databasePath, String notificationConfig, String unused,
- String yoloCfgPath, String yoloWeightsPath, String classNamesPath) throws SQLException, IOException {
- this.httpClient = HttpClient.newHttpClient();
- this.objectMapper = new ObjectMapper();
- this.db = new DatabaseService(databasePath);
- this.notifier = new NotificationService(notificationConfig, unused);
- this.detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
- // initialize DB
- db.ensureSchema();
- }
-
- /**
- * Discovers all active Dutch auctions by crawling the auctions page.
- *
- * Troostwijk lists auctions for many countries on one page. We parse
- * the page with jsoup (an HTML parser that fetches and parses real‐world
- * HTML easily【438902460386021†L14-L24】) and filter auctions whose location
- * contains ", NL" (indicating the Netherlands). Each auction link
- * contains a unique sale ID which we extract from its URL.
- *
- * @return a list of sale identifiers for auctions located in NL
- */
- public List discoverDutchAuctions() {
- List saleIds = new ArrayList<>();
- try {
- // Fetch the auctions overview page
- Document doc = Jsoup.connect(AUCTIONS_PAGE).get();
- // Select all anchor elements that represent an auction listing.
- // The exact selector may change; inspect the page with your browser’s
- // developer tools and update accordingly.
- Elements auctionLinks = doc.select("a[href][data-id]");
- for (Element link : auctionLinks) {
- Element locationElement = link.selectFirst(".auction-location");
- String location = locationElement != null ? locationElement.text() : "";
- if (location.contains(", NL")) {
- // Extract saleID from the data-id attribute or href
- String saleIdStr = link.attr("data-id");
- if (saleIdStr.isEmpty()) {
- // Fallback: parse from URL path, e.g. /nl/sale/27213/machines
- String href = link.attr("href");
- String[] parts = href.split("/");
- for (String p : parts) {
- if (p.matches("\\d+")) {
- saleIdStr = p;
- break;
- }
- }
- }
- try {
- int saleId = Integer.parseInt(saleIdStr);
+
+ // Base URLs – adjust these if Troostwijk changes their site structure
+ private static final String AUCTIONS_PAGE = "https://www.troostwijkauctions.com/nl/auctions";
+ private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list";
+
+ // HTTP client used for API calls
+ private final HttpClient httpClient;
+ private final ObjectMapper objectMapper;
+ private final DatabaseService db;
+ private final NotificationService notifier;
+ private final ObjectDetectionService detector;
+
+ /**
+ * Constructor. Creates supporting services and ensures the database
+ * tables exist.
+ *
+ * @param databasePath Path to SQLite database file
+ * @param notificationConfig "desktop" for desktop only, or "smtp:user:pass:toEmail" for email
+ * @param unused Unused parameter (kept for compatibility)
+ * @param yoloCfgPath Path to YOLO configuration file
+ * @param yoloWeightsPath Path to YOLO weights file
+ * @param classNamesPath Path to file containing class names
+ */
+ public TroostwijkScraper(String databasePath, String notificationConfig, String unused,
+ String yoloCfgPath, String yoloWeightsPath, String classNamesPath) throws SQLException, IOException {
+ this.httpClient = HttpClient.newHttpClient();
+ this.objectMapper = new ObjectMapper();
+ this.db = new DatabaseService(databasePath);
+ this.notifier = new NotificationService(notificationConfig, unused);
+ this.detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
+ // initialize DB
+ db.ensureSchema();
+ }
+
+ /**
+ * Discovers all active Dutch auctions by crawling the auctions page.
+ *
+ * Troostwijk lists auctions for many countries on one page. We parse
+ * the page with jsoup and filter auctions whose location contains ", NL"
+ * (indicating the Netherlands). Each auction link contains a unique sale ID
+ * in the format A1-xxxxx or A7-xxxxx which we extract from the URL.
+ *
+ * @return a list of sale identifiers for auctions located in NL
+ */
+ public List discoverDutchAuctions() {
+ List saleIds = new ArrayList<>();
+ try {
+ // Fetch the auctions overview page
+ Document doc = Jsoup.connect(AUCTIONS_PAGE).get();
+
+ // Select all anchor elements that link to auction pages
+ // The URL pattern is: /a/auction-title-A1-xxxxx or /a/auction-title-A7-xxxxx
+ Elements auctionLinks = doc.select("a[href^='/a/']");
+
+ System.out.println("Found " + auctionLinks.size() + " potential auction links");
+
+ for (Element link : auctionLinks) {
+ // Get the href to extract the auction ID
+ String href = link.attr("href");
+
+ // Check if this link contains location text with ", NL"
+ String linkText = link.text();
+
+ // Look for location in any div inside the link
+ Elements divs = link.select("div");
+ boolean isDutch = false;
+ for (Element div : divs) {
+ String text = div.text();
+ if (text.contains(", NL")) {
+ isDutch = true;
+ break;
+ }
+ }
+
+ if (isDutch) {
+ // Extract auction ID from URL
+ // Format: /a/title-A1-38375 or /a/title-A7-12345
+ // We want the number after A1- or A7-
+ java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("A[17]-(\\d+)");
+ java.util.regex.Matcher matcher = pattern.matcher(href);
+
+ if (matcher.find()) {
+ try {
+ int saleId = Integer.parseInt(matcher.group(1));
+ if (!saleIds.contains(saleId)) {
saleIds.add(saleId);
- } catch (NumberFormatException ignored) {
- // not a sale ID
- }
- }
+ System.out.println(" Found Dutch auction: " + saleId + " - " + href);
+ }
+ } catch (NumberFormatException e) {
+ // Skip invalid IDs
+ }
+ }
}
- } catch (IOException e) {
- System.err.println("Failed to discover auctions: " + e.getMessage());
- }
- return saleIds;
- }
-
- /**
- * Retrieves all lots for a given sale ID using Troostwijk’s internal JSON
- * API. The API accepts parameters such as batchSize, offset, and saleID.
- * A large batchSize returns many lots at once【610752406306016†L124-L134】. We loop
- * until no further results are returned. Each JSON result is mapped to
- * our Lot domain object and persisted to the database.
- *
- * @param saleId the sale identifier
- */
- public void fetchLotsForSale(int saleId) {
- int batchSize = 200;
- int offset = 0;
- boolean more = true;
- while (more) {
- try {
- String url = LOT_API + "?batchSize=" + batchSize
- + "&listType=7&offset=" + offset
- + "&sortOption=0&saleID=" + saleId
- + "&parentID=0&relationID=0&buildversion=201807311";
- HttpRequest request = HttpRequest.newBuilder()
- .uri(URI.create(url))
- .header("Accept", "application/json")
- .GET()
- .build();
- HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
- if (response.statusCode() != 200) {
- System.err.println("API call failed for sale " + saleId + " with status " + response.statusCode());
- break;
- }
- JsonNode root = objectMapper.readTree(response.body());
- JsonNode results = root.path("results");
- if (!results.isArray() || results.isEmpty()) {
- more = false;
- break;
- }
- for (JsonNode node : results) {
- Lot lot = new Lot();
- lot.saleId = saleId;
- lot.lotId = node.path("lotID").asInt();
- lot.title = node.path("t").asText();
- lot.description = node.path("d").asText();
- lot.manufacturer = node.path("mf").asText();
- lot.type = node.path("typ").asText();
- lot.year = node.path("yb").asInt();
- lot.category = node.path("lc").asText();
- // Current bid; field names may differ (e.g. currentBid or cb)
- lot.currentBid = node.path("cb").asDouble();
- lot.currency = node.path("cu").asText();
- lot.url = "https://www.troostwijkauctions.com/nl" + node.path("url").asText();
- // Save basic lot info into DB
- db.upsertLot(lot);
- // Download images and perform object detection
- List imageUrls = new ArrayList<>();
- JsonNode imgs = node.path("imgs");
- if (imgs.isArray()) {
- for (JsonNode imgNode : imgs) {
- String imgUrl = imgNode.asText();
- imageUrls.add(imgUrl);
- }
- }
- for (String imgUrl : imageUrls) {
- String fileName = downloadImage(imgUrl, saleId, lot.lotId);
- if (fileName != null) {
- // run object detection once per image
- List labels = detector.detectObjects(fileName);
- db.insertImage(lot.lotId, imgUrl, fileName, labels);
- }
- }
- }
- offset += batchSize;
- } catch (IOException | InterruptedException e) {
- System.err.println("Error fetching lots for sale " + saleId + ": " + e.getMessage());
- more = false;
- } catch (SQLException e) {
- System.err.println("Database error: " + e.getMessage());
- }
- }
- }
-
- /**
- * Downloads an image from the given URL to a local directory. Images
- * are stored under "images///" to keep them organised.
- *
- * @param imageUrl remote image URL
- * @param saleId sale identifier
- * @param lotId lot identifier
- * @return absolute path to saved file or null on failure
- */
- private String downloadImage(String imageUrl, int saleId, int lotId) {
- try {
+ }
+ } catch (IOException e) {
+ System.err.println("Failed to discover auctions: " + e.getMessage());
+ e.printStackTrace();
+ }
+ return saleIds;
+ }
+
+ /**
+ * Retrieves all lots for a given sale ID using Troostwijk's internal JSON
+ * API. The API accepts parameters such as batchSize, offset, and saleID.
+ * A large batchSize returns many lots at once. We loop until no further
+ * results are returned. Each JSON result is mapped to our Lot domain
+ * object and persisted to the database.
+ *
+ * @param saleId the sale identifier
+ */
+ public void fetchLotsForSale(int saleId) {
+ int batchSize = 200;
+ int offset = 0;
+ boolean more = true;
+ int totalLots = 0;
+
+ while (more) {
+ try {
+ String url = LOT_API + "?batchSize=" + batchSize
+ + "&listType=7&offset=" + offset
+ + "&sortOption=0&saleID=" + saleId
+ + "&parentID=0&relationID=0&buildversion=201807311";
+
+ System.out.println(" Fetching lots from API (offset=" + offset + ")...");
+
HttpRequest request = HttpRequest.newBuilder()
- .uri(URI.create(imageUrl))
- .GET()
- .build();
- HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
- if (response.statusCode() == 200) {
- Path dir = Paths.get("images", String.valueOf(saleId), String.valueOf(lotId));
- Files.createDirectories(dir);
- String fileName = Paths.get(imageUrl).getFileName().toString();
- Path dest = dir.resolve(fileName);
- Files.copy(response.body(), dest);
- return dest.toAbsolutePath().toString();
- }
- } catch (IOException | InterruptedException e) {
- System.err.println("Failed to download image " + imageUrl + ": " + e.getMessage());
- }
- return null;
- }
-
- /**
- * Schedules periodic monitoring of all lots. The scheduler runs every
- * hour to refresh current bids and closing times. For lots that
- * are within 30 minutes of closing, it increases the polling frequency
- * automatically. When a new bid is detected or a lot is about to
- * expire, a Pushover notification is sent to the configured user.
- * Note: In production, ensure proper shutdown handling for the scheduler.
- */
- public ScheduledExecutorService scheduleMonitoring() {
- ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
- scheduler.scheduleAtFixedRate(() -> {
- try {
- List activeLots = db.getActiveLots();
- for (Lot lot : activeLots) {
- // refresh the lot's bidding information via API
- refreshLotBid(lot);
- // check closing time to adjust monitoring
- long minutesLeft = lot.minutesUntilClose();
- if (minutesLeft < 30) {
- // send warning when within 5 minutes
- if (minutesLeft <= 5 && !lot.closingNotified) {
- notifier.sendNotification("Kavel " + lot.lotId + " sluit binnen " + minutesLeft + " min.",
- "Lot nearing closure", 1);
- lot.closingNotified = true;
- db.updateLotNotificationFlags(lot);
- }
- // schedule additional quick check for this lot
- scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
- }
- }
- } catch (SQLException e) {
- System.err.println("Error during scheduled monitoring: " + e.getMessage());
- }
- }, 0, 1, TimeUnit.HOURS);
- return scheduler;
- }
-
- /**
- * Refreshes the bid for a single lot and sends notification if it has
- * changed since the last check. The method calls the same API used for
- * initial scraping but only extracts the current bid for the given lot.
- *
- * @param lot the lot to refresh
- */
- private void refreshLotBid(Lot lot) {
- try {
- String url = LOT_API + "?batchSize=1&listType=7&offset=0&sortOption=0&saleID=" + lot.saleId
- + "&parentID=0&relationID=0&buildversion=201807311&lotID=" + lot.lotId;
- HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build();
+ .uri(URI.create(url))
+ .header("Accept", "application/json")
+ .header("User-Agent", "Mozilla/5.0")
+ .GET()
+ .build();
+
HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
- if (response.statusCode() != 200) return;
- JsonNode root = objectMapper.readTree(response.body());
+
+ if (response.statusCode() != 200) {
+ System.err.println(" ⚠️ API call failed for sale " + saleId);
+ System.err.println(" Status: " + response.statusCode());
+ System.err.println(" Response: " + response.body().substring(0, Math.min(200, response.body().length())));
+ break;
+ }
+
+ JsonNode root = objectMapper.readTree(response.body());
JsonNode results = root.path("results");
- if (results.isArray() && !results.isEmpty()) {
- JsonNode node = results.get(0);
- double newBid = node.path("cb").asDouble();
- if (Double.compare(newBid, lot.currentBid) > 0) {
- double previous = lot.currentBid;
- lot.currentBid = newBid;
- db.updateLotCurrentBid(lot);
- String msg = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)", lot.lotId, newBid, previous);
- notifier.sendNotification(msg, "Kavel bieding update", 0);
- }
+
+ if (!results.isArray() || results.isEmpty()) {
+ if (offset == 0) {
+ System.out.println(" ⚠️ No lots found for sale " + saleId);
+ System.out.println(" API Response: " + response.body().substring(0, Math.min(500, response.body().length())));
+ }
+ more = false;
+ break;
}
- } catch (IOException | InterruptedException | SQLException e) {
- System.err.println("Failed to refresh bid for lot " + lot.lotId + ": " + e.getMessage());
- }
- }
-
- /**
- * Entry point. Configure database location, notification settings, and
- * YOLO model paths here before running. Once started the scraper
- * discovers Dutch auctions, scrapes lots, and begins monitoring.
- */
- public static void main(String[] args) throws Exception {
- System.out.println("=== Troostwijk Auction Scraper ===\n");
-
- // Configuration parameters (replace with your own values)
- String databaseFile = "troostwijk.db";
-
- // Notification configuration - choose one:
- // Option 1: Desktop notifications only (free, no setup required)
- String notificationConfig = System.getenv().getOrDefault("NOTIFICATION_CONFIG", "desktop");
-
- // Option 2: Desktop + Email via Gmail (free, requires Gmail app password)
- // Format: "smtp:username:appPassword:toEmail"
- // Example: "smtp:your.email@gmail.com:abcd1234efgh5678:recipient@example.com"
- // Get app password: Google Account > Security > 2-Step Verification > App passwords
-
- // YOLO model paths (optional - scraper works without object detection)
- String yoloCfg = "models/yolov4.cfg";
- String yoloWeights = "models/yolov4.weights";
- String yoloClasses = "models/coco.names";
-
- // Load native OpenCV library
- System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
-
- System.out.println("Initializing scraper...");
- TroostwijkScraper scraper = new TroostwijkScraper(databaseFile, notificationConfig, "",
- yoloCfg, yoloWeights, yoloClasses);
-
- // Step 1: Discover auctions in NL
- System.out.println("\n[1/3] Discovering Dutch auctions...");
- List auctions = scraper.discoverDutchAuctions();
- System.out.println("✓ Found " + auctions.size() + " auctions: " + auctions);
-
- // Step 2: Fetch lots for each auction
- System.out.println("\n[2/3] Fetching lot details...");
- for (int saleId : auctions) {
- System.out.println(" Processing sale " + saleId + "...");
- scraper.fetchLotsForSale(saleId);
- }
-
- // Step 3: Start monitoring bids and closures
- System.out.println("\n[3/3] Starting monitoring service...");
- scraper.scheduleMonitoring();
- System.out.println("✓ Monitoring active. Press Ctrl+C to stop.\n");
- }
-
- // ----------------------------------------------------------------------
- // Domain classes and services
- // ----------------------------------------------------------------------
-
- /**
- * Simple POJO representing a lot (kavel) in an auction. It keeps track
- * of the sale it belongs to, current bid and closing time. The method
- * minutesUntilClose computes how many minutes remain until the lot closes.
- */
- static class Lot {
- int saleId;
- int lotId;
- String title;
- String description;
- String manufacturer;
- String type;
- int year;
- String category;
- double currentBid;
- String currency;
- String url;
- LocalDateTime closingTime; // null if unknown
- boolean closingNotified;
-
- long minutesUntilClose() {
- if (closingTime == null) return Long.MAX_VALUE;
- return java.time.Duration.between(LocalDateTime.now(), closingTime).toMinutes();
- }
- }
-
- /**
- * Service for persisting auctions, lots, images, and object labels into
- * a SQLite database. Uses the Xerial JDBC driver which connects to
- * SQLite via a URL of the form "jdbc:sqlite:path_to_file"【329850066306528†L40-L63】.
- */
- static class DatabaseService {
- private final String url;
- DatabaseService(String dbPath) {
- this.url = "jdbc:sqlite:" + dbPath;
- }
- /**
- * Creates tables if they do not already exist. The schema includes
- * tables for sales, lots, images, and object labels. This method is
- * idempotent; it can be called multiple times.
- */
- void ensureSchema() throws SQLException {
- try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
- // Sales table
- stmt.execute("CREATE TABLE IF NOT EXISTS sales ("
- + "sale_id INTEGER PRIMARY KEY,"
- + "title TEXT,"
- + "location TEXT,"
- + "closing_time TEXT"
- + ")");
- // Lots table
- stmt.execute("CREATE TABLE IF NOT EXISTS lots ("
- + "lot_id INTEGER PRIMARY KEY,"
- + "sale_id INTEGER,"
- + "title TEXT,"
- + "description TEXT,"
- + "manufacturer TEXT,"
- + "type TEXT,"
- + "year INTEGER,"
- + "category TEXT,"
- + "current_bid REAL,"
- + "currency TEXT,"
- + "url TEXT,"
- + "closing_time TEXT,"
- + "closing_notified INTEGER DEFAULT 0,"
- + "FOREIGN KEY (sale_id) REFERENCES sales(sale_id)"
- + ")");
- // Images table
- stmt.execute("CREATE TABLE IF NOT EXISTS images ("
- + "id INTEGER PRIMARY KEY AUTOINCREMENT,"
- + "lot_id INTEGER,"
- + "url TEXT,"
- + "file_path TEXT,"
- + "labels TEXT,"
- + "FOREIGN KEY (lot_id) REFERENCES lots(lot_id)"
- + ")");
+ int lotsInBatch = results.size();
+ System.out.println(" Found " + lotsInBatch + " lots in this batch");
+
+ for (JsonNode node : results) {
+ Lot lot = new Lot();
+ lot.saleId = saleId;
+ lot.lotId = node.path("lotID").asInt();
+ lot.title = node.path("t").asText();
+ lot.description = node.path("d").asText();
+ lot.manufacturer = node.path("mf").asText();
+ lot.type = node.path("typ").asText();
+ lot.year = node.path("yb").asInt();
+ lot.category = node.path("lc").asText();
+ // Current bid; field names may differ (e.g. currentBid or cb)
+ lot.currentBid = node.path("cb").asDouble();
+ lot.currency = node.path("cu").asText();
+ lot.url = "https://www.troostwijkauctions.com/nl" + node.path("url").asText();
+
+ // Save basic lot info into DB
+ db.upsertLot(lot);
+ totalLots++;
+
+ // Download images and perform object detection
+ List imageUrls = new ArrayList<>();
+ JsonNode imgs = node.path("imgs");
+ if (imgs.isArray()) {
+ for (JsonNode imgNode : imgs) {
+ String imgUrl = imgNode.asText();
+ imageUrls.add(imgUrl);
+ }
+ }
+
+ // Download and analyze images (optional, can be slow)
+ for (String imgUrl : imageUrls) {
+ String fileName = downloadImage(imgUrl, saleId, lot.lotId);
+ if (fileName != null) {
+ // run object detection once per image
+ List labels = detector.detectObjects(fileName);
+ db.insertImage(lot.lotId, imgUrl, fileName, labels);
+ }
+ }
}
- }
-
- /**
- * Inserts or updates a lot record. Uses INSERT OR REPLACE to
- * implement upsert semantics so that existing rows are replaced.
- */
- synchronized void upsertLot(Lot lot) throws SQLException {
- String sql = "INSERT INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)"
- + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
- + " ON CONFLICT(lot_id) DO UPDATE SET "
- + "sale_id = excluded.sale_id, title = excluded.title, description = excluded.description, "
- + "manufacturer = excluded.manufacturer, type = excluded.type, year = excluded.year, category = excluded.category, "
- + "current_bid = excluded.current_bid, currency = excluded.currency, url = excluded.url, closing_time = excluded.closing_time";
- try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(sql)) {
- ps.setInt(1, lot.lotId);
- ps.setInt(2, lot.saleId);
- ps.setString(3, lot.title);
- ps.setString(4, lot.description);
- ps.setString(5, lot.manufacturer);
- ps.setString(6, lot.type);
- ps.setInt(7, lot.year);
- ps.setString(8, lot.category);
- ps.setDouble(9, lot.currentBid);
- ps.setString(10, lot.currency);
- ps.setString(11, lot.url);
- ps.setString(12, lot.closingTime != null ? lot.closingTime.toString() : null);
- ps.setInt(13, lot.closingNotified ? 1 : 0);
- ps.executeUpdate();
+
+ System.out.println(" ✓ Processed " + totalLots + " lots so far");
+ offset += batchSize;
+ } catch (IOException | InterruptedException e) {
+ System.err.println("Error fetching lots for sale " + saleId + ": " + e.getMessage());
+ more = false;
+ } catch (SQLException e) {
+ System.err.println("Database error: " + e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Downloads an image from the given URL to a local directory. Images
+ * are stored under "images///" to keep them organised.
+ *
+ * @param imageUrl remote image URL
+ * @param saleId sale identifier
+ * @param lotId lot identifier
+ * @return absolute path to saved file or null on failure
+ */
+ private String downloadImage(String imageUrl, int saleId, int lotId) {
+ try {
+ HttpRequest request = HttpRequest.newBuilder()
+ .uri(URI.create(imageUrl))
+ .GET()
+ .build();
+ HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
+ if (response.statusCode() == 200) {
+ Path dir = Paths.get("images", String.valueOf(saleId), String.valueOf(lotId));
+ Files.createDirectories(dir);
+ String fileName = Paths.get(imageUrl).getFileName().toString();
+ Path dest = dir.resolve(fileName);
+ Files.copy(response.body(), dest);
+ return dest.toAbsolutePath().toString();
+ }
+ } catch (IOException | InterruptedException e) {
+ System.err.println("Failed to download image " + imageUrl + ": " + e.getMessage());
+ }
+ return null;
+ }
+
+ /**
+ * Schedules periodic monitoring of all lots. The scheduler runs every
+ * hour to refresh current bids and closing times. For lots that
+ * are within 30 minutes of closing, it increases the polling frequency
+ * automatically. When a new bid is detected or a lot is about to
+ * expire, a Pushover notification is sent to the configured user.
+ * Note: In production, ensure proper shutdown handling for the scheduler.
+ */
+ public ScheduledExecutorService scheduleMonitoring() {
+ ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
+ scheduler.scheduleAtFixedRate(() -> {
+ try {
+ List activeLots = db.getActiveLots();
+ for (Lot lot : activeLots) {
+ // refresh the lot's bidding information via API
+ refreshLotBid(lot);
+ // check closing time to adjust monitoring
+ long minutesLeft = lot.minutesUntilClose();
+ if (minutesLeft < 30) {
+ // send warning when within 5 minutes
+ if (minutesLeft <= 5 && !lot.closingNotified) {
+ notifier.sendNotification("Kavel " + lot.lotId + " sluit binnen " + minutesLeft + " min.",
+ "Lot nearing closure", 1);
+ lot.closingNotified = true;
+ db.updateLotNotificationFlags(lot);
+ }
+ // schedule additional quick check for this lot
+ scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
+ }
}
- }
-
- /**
- * Inserts a new image record. Each image is associated with a lot and
- * stores both the original URL and the local file path. Detected
- * labels are stored as a comma separated string.
- */
- synchronized void insertImage(int lotId, String url, String filePath, List labels) throws SQLException {
- String sql = "INSERT INTO images (lot_id, url, file_path, labels) VALUES (?, ?, ?, ?)";
- try (Connection conn = DriverManager.getConnection(this.url); PreparedStatement ps = conn.prepareStatement(sql)) {
- ps.setInt(1, lotId);
- ps.setString(2, url);
- ps.setString(3, filePath);
- ps.setString(4, String.join(",", labels));
- ps.executeUpdate();
+ } catch (SQLException e) {
+ System.err.println("Error during scheduled monitoring: " + e.getMessage());
+ }
+ }, 0, 1, TimeUnit.HOURS);
+ return scheduler;
+ }
+
+ /**
+ * Refreshes the bid for a single lot and sends notification if it has
+ * changed since the last check. The method calls the same API used for
+ * initial scraping but only extracts the current bid for the given lot.
+ *
+ * @param lot the lot to refresh
+ */
+ private void refreshLotBid(Lot lot) {
+ try {
+ String url = LOT_API + "?batchSize=1&listType=7&offset=0&sortOption=0&saleID=" + lot.saleId
+ + "&parentID=0&relationID=0&buildversion=201807311&lotID=" + lot.lotId;
+ HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build();
+ HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+ if (response.statusCode() != 200) return;
+ JsonNode root = objectMapper.readTree(response.body());
+ JsonNode results = root.path("results");
+ if (results.isArray() && !results.isEmpty()) {
+ JsonNode node = results.get(0);
+ double newBid = node.path("cb").asDouble();
+ if (Double.compare(newBid, lot.currentBid) > 0) {
+ double previous = lot.currentBid;
+ lot.currentBid = newBid;
+ db.updateLotCurrentBid(lot);
+ String msg = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)", lot.lotId, newBid, previous);
+ notifier.sendNotification(msg, "Kavel bieding update", 0);
}
- }
-
- /**
- * Retrieves all lots that are still active (i.e., have a closing time
- * in the future or unknown). Only these lots need to be monitored.
- */
- synchronized List getActiveLots() throws SQLException {
- List list = new ArrayList<>();
- String sql = "SELECT lot_id, sale_id, current_bid, currency, closing_time, closing_notified FROM lots";
- try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
- ResultSet rs = stmt.executeQuery(sql);
- while (rs.next()) {
- Lot lot = new Lot();
- lot.lotId = rs.getInt("lot_id");
- lot.saleId = rs.getInt("sale_id");
- lot.currentBid = rs.getDouble("current_bid");
- lot.currency = rs.getString("currency");
- String closing = rs.getString("closing_time");
- lot.closingNotified = rs.getInt("closing_notified") != 0;
- if (closing != null) {
- lot.closingTime = LocalDateTime.parse(closing);
- }
- list.add(lot);
- }
+ }
+ } catch (IOException | InterruptedException | SQLException e) {
+ System.err.println("Failed to refresh bid for lot " + lot.lotId + ": " + e.getMessage());
+ }
+ }
+
+ /**
+ * Entry point. Configure database location, notification settings, and
+ * YOLO model paths here before running. Once started the scraper
+ * discovers Dutch auctions, scrapes lots, and begins monitoring.
+ */
+ public static void main(String[] args) throws Exception {
+ System.out.println("=== Troostwijk Auction Scraper ===\n");
+
+ // Configuration parameters (replace with your own values)
+ String databaseFile = "troostwijk.db";
+
+ // Notification configuration - choose one:
+ // Option 1: Desktop notifications only (free, no setup required)
+ String notificationConfig = System.getenv().getOrDefault("NOTIFICATION_CONFIG", "desktop");
+
+ // Option 2: Desktop + Email via Gmail (free, requires Gmail app password)
+ // Format: "smtp:username:appPassword:toEmail"
+ // Example: "smtp:your.email@gmail.com:abcd1234efgh5678:recipient@example.com"
+ // Get app password: Google Account > Security > 2-Step Verification > App passwords
+
+ // YOLO model paths (optional - scraper works without object detection)
+ String yoloCfg = "models/yolov4.cfg";
+ String yoloWeights = "models/yolov4.weights";
+ String yoloClasses = "models/coco.names";
+
+ // Load native OpenCV library
+ System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
+
+ System.out.println("Initializing scraper...");
+ TroostwijkScraper scraper = new TroostwijkScraper(databaseFile, notificationConfig, "",
+ yoloCfg, yoloWeights, yoloClasses);
+
+ // Step 1: Discover auctions in NL
+ System.out.println("\n[1/3] Discovering Dutch auctions...");
+ List auctions = scraper.discoverDutchAuctions();
+ System.out.println("✓ Found " + auctions.size() + " auctions: " + auctions);
+
+ // Step 2: Fetch lots for each auction
+ System.out.println("\n[2/3] Fetching lot details...");
+ int totalAuctions = auctions.size();
+ int currentAuction = 0;
+ for (int saleId : auctions) {
+ currentAuction++;
+ System.out.println(" [" + currentAuction + "/" + totalAuctions + "] Processing sale " + saleId + "...");
+ scraper.fetchLotsForSale(saleId);
+ }
+
+ // Show database summary
+ System.out.println("\n📊 Database Summary:");
+ scraper.printDatabaseStats();
+
+ // Step 3: Start monitoring bids and closures
+ System.out.println("\n[3/3] Starting monitoring service...");
+ scraper.scheduleMonitoring();
+ System.out.println("✓ Monitoring active. Press Ctrl+C to stop.\n");
+ }
+
+ /**
+ * Prints statistics about the data in the database.
+ */
+ private void printDatabaseStats() {
+ try {
+ List allLots = db.getAllLots();
+ int imageCount = db.getImageCount();
+
+ System.out.println(" Total lots in database: " + allLots.size());
+ System.out.println(" Total images downloaded: " + imageCount);
+
+ if (!allLots.isEmpty()) {
+ double totalBids = allLots.stream().mapToDouble(l -> l.currentBid).sum();
+ System.out.println(" Total current bids: €" + String.format("%.2f", totalBids));
+ }
+ } catch (SQLException e) {
+ System.err.println(" ⚠️ Could not retrieve database stats: " + e.getMessage());
+ }
+ }
+
+ // ----------------------------------------------------------------------
+ // Domain classes and services
+ // ----------------------------------------------------------------------
+
+ /**
+ * Simple POJO representing a lot (kavel) in an auction. It keeps track
+ * of the sale it belongs to, current bid and closing time. The method
+ * minutesUntilClose computes how many minutes remain until the lot closes.
+ */
+ static class Lot {
+
+ int saleId;
+ int lotId;
+ String title;
+ String description;
+ String manufacturer;
+ String type;
+ int year;
+ String category;
+ double currentBid;
+ String currency;
+ String url;
+ LocalDateTime closingTime; // null if unknown
+ boolean closingNotified;
+
+ long minutesUntilClose() {
+ if (closingTime == null) return Long.MAX_VALUE;
+ return java.time.Duration.between(LocalDateTime.now(), closingTime).toMinutes();
+ }
+ }
+
+ /**
+ * Service for persisting auctions, lots, images, and object labels into
+ * a SQLite database. Uses the Xerial JDBC driver which connects to
+ * SQLite via a URL of the form "jdbc:sqlite:path_to_file"【329850066306528†L40-L63】.
+ */
+ static class DatabaseService {
+
+ private final String url;
+ DatabaseService(String dbPath) {
+ this.url = "jdbc:sqlite:" + dbPath;
+ }
+ /**
+ * Creates tables if they do not already exist. The schema includes
+ * tables for sales, lots, images, and object labels. This method is
+ * idempotent; it can be called multiple times.
+ */
+ void ensureSchema() throws SQLException {
+ try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
+ // Sales table
+ stmt.execute("CREATE TABLE IF NOT EXISTS sales ("
+ + "sale_id INTEGER PRIMARY KEY,"
+ + "title TEXT,"
+ + "location TEXT,"
+ + "closing_time TEXT"
+ + ")");
+ // Lots table
+ stmt.execute("CREATE TABLE IF NOT EXISTS lots ("
+ + "lot_id INTEGER PRIMARY KEY,"
+ + "sale_id INTEGER,"
+ + "title TEXT,"
+ + "description TEXT,"
+ + "manufacturer TEXT,"
+ + "type TEXT,"
+ + "year INTEGER,"
+ + "category TEXT,"
+ + "current_bid REAL,"
+ + "currency TEXT,"
+ + "url TEXT,"
+ + "closing_time TEXT,"
+ + "closing_notified INTEGER DEFAULT 0,"
+ + "FOREIGN KEY (sale_id) REFERENCES sales(sale_id)"
+ + ")");
+ // Images table
+ stmt.execute("CREATE TABLE IF NOT EXISTS images ("
+ + "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ + "lot_id INTEGER,"
+ + "url TEXT,"
+ + "file_path TEXT,"
+ + "labels TEXT,"
+ + "FOREIGN KEY (lot_id) REFERENCES lots(lot_id)"
+ + ")");
+ }
+ }
+
+ /**
+ * Inserts or updates a lot record. Uses INSERT OR REPLACE to
+ * implement upsert semantics so that existing rows are replaced.
+ */
+ synchronized void upsertLot(Lot lot) throws SQLException {
+ String sql = "INSERT INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)"
+ + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+ + " ON CONFLICT(lot_id) DO UPDATE SET "
+ + "sale_id = excluded.sale_id, title = excluded.title, description = excluded.description, "
+ + "manufacturer = excluded.manufacturer, type = excluded.type, year = excluded.year, category = excluded.category, "
+ + "current_bid = excluded.current_bid, currency = excluded.currency, url = excluded.url, closing_time = excluded.closing_time";
+ try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(sql)) {
+ ps.setInt(1, lot.lotId);
+ ps.setInt(2, lot.saleId);
+ ps.setString(3, lot.title);
+ ps.setString(4, lot.description);
+ ps.setString(5, lot.manufacturer);
+ ps.setString(6, lot.type);
+ ps.setInt(7, lot.year);
+ ps.setString(8, lot.category);
+ ps.setDouble(9, lot.currentBid);
+ ps.setString(10, lot.currency);
+ ps.setString(11, lot.url);
+ ps.setString(12, lot.closingTime != null ? lot.closingTime.toString() : null);
+ ps.setInt(13, lot.closingNotified ? 1 : 0);
+ ps.executeUpdate();
+ }
+ }
+
+ /**
+ * Inserts a new image record. Each image is associated with a lot and
+ * stores both the original URL and the local file path. Detected
+ * labels are stored as a comma separated string.
+ */
+ synchronized void insertImage(int lotId, String url, String filePath, List labels) throws SQLException {
+ String sql = "INSERT INTO images (lot_id, url, file_path, labels) VALUES (?, ?, ?, ?)";
+ try (Connection conn = DriverManager.getConnection(this.url); PreparedStatement ps = conn.prepareStatement(sql)) {
+ ps.setInt(1, lotId);
+ ps.setString(2, url);
+ ps.setString(3, filePath);
+ ps.setString(4, String.join(",", labels));
+ ps.executeUpdate();
+ }
+ }
+
+ /**
+ * Retrieves all lots that are still active (i.e., have a closing time
+ * in the future or unknown). Only these lots need to be monitored.
+ */
+ synchronized List getActiveLots() throws SQLException {
+ List list = new ArrayList<>();
+ String sql = "SELECT lot_id, sale_id, current_bid, currency, closing_time, closing_notified FROM lots";
+ try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
+ ResultSet rs = stmt.executeQuery(sql);
+ while (rs.next()) {
+ Lot lot = new Lot();
+ lot.lotId = rs.getInt("lot_id");
+ lot.saleId = rs.getInt("sale_id");
+ lot.currentBid = rs.getDouble("current_bid");
+ lot.currency = rs.getString("currency");
+ String closing = rs.getString("closing_time");
+ lot.closingNotified = rs.getInt("closing_notified") != 0;
+ if (closing != null) {
+ lot.closingTime = LocalDateTime.parse(closing);
+ }
+ list.add(lot);
}
- return list;
- }
-
- /**
- * Updates the current bid of a lot after a bid refresh.
- */
- synchronized void updateLotCurrentBid(Lot lot) throws SQLException {
- try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(
- "UPDATE lots SET current_bid = ? WHERE lot_id = ?")) {
- ps.setDouble(1, lot.currentBid);
- ps.setInt(2, lot.lotId);
- ps.executeUpdate();
+ }
+ return list;
+ }
+
+ /**
+ * Retrieves all lots from the database.
+ */
+ synchronized List getAllLots() throws SQLException {
+ List list = new ArrayList<>();
+ String sql = "SELECT lot_id, sale_id, title, current_bid, currency FROM lots";
+ try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
+ ResultSet rs = stmt.executeQuery(sql);
+ while (rs.next()) {
+ Lot lot = new Lot();
+ lot.lotId = rs.getInt("lot_id");
+ lot.saleId = rs.getInt("sale_id");
+ lot.title = rs.getString("title");
+ lot.currentBid = rs.getDouble("current_bid");
+ lot.currency = rs.getString("currency");
+ list.add(lot);
}
- }
-
- /**
- * Updates the closingNotified flag of a lot (set to 1 when we have
- * warned the user about its imminent closure).
- */
- synchronized void updateLotNotificationFlags(Lot lot) throws SQLException {
- try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(
- "UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) {
- ps.setInt(1, lot.closingNotified ? 1 : 0);
- ps.setInt(2, lot.lotId);
- ps.executeUpdate();
+ }
+ return list;
+ }
+
+ /**
+ * Gets the total number of images in the database.
+ */
+ synchronized int getImageCount() throws SQLException {
+ String sql = "SELECT COUNT(*) as count FROM images";
+ try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
+ ResultSet rs = stmt.executeQuery(sql);
+ if (rs.next()) {
+ return rs.getInt("count");
}
- }
- }
-
- /**
- * Service for sending notifications via desktop notifications and/or email.
- * Supports free notification methods:
- * 1. Desktop notifications (Windows/Linux/macOS system tray)
- * 2. Email via Gmail SMTP (free, requires app password)
- *
- * Configuration:
- * - For email: Set notificationEmail to your Gmail address
- * - Enable 2FA in Gmail and create an App Password
- * - Use format "smtp:username:appPassword:toEmail" for credentials
- * - Or use "desktop" for desktop-only notifications
- */
- static class NotificationService {
- private final boolean useDesktop;
- private final boolean useEmail;
- private final String smtpUsername;
- private final String smtpPassword;
- private final String toEmail;
-
- /**
- * Creates a notification service.
- *
- * @param config "desktop" for desktop only, or "smtp:username:password:toEmail" for email
- * @param unusedParam Kept for compatibility (can pass empty string)
- */
- NotificationService(String config, String unusedParam) {
-
- if ("desktop".equalsIgnoreCase(config)) {
- this.useDesktop = true;
- this.useEmail = false;
- this.smtpUsername = null;
- this.smtpPassword = null;
- this.toEmail = null;
- } else if (config.startsWith("smtp:")) {
- String[] parts = config.split(":", 4);
- if (parts.length != 4) {
- throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'");
- }
- this.useDesktop = true; // Always include desktop
- this.useEmail = true;
- this.smtpUsername = parts[1];
- this.smtpPassword = parts[2];
- this.toEmail = parts[3];
+ }
+ return 0;
+ }
+
+ /**
+ * Updates the current bid of a lot after a bid refresh.
+ */
+ synchronized void updateLotCurrentBid(Lot lot) throws SQLException {
+ try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(
+ "UPDATE lots SET current_bid = ? WHERE lot_id = ?")) {
+ ps.setDouble(1, lot.currentBid);
+ ps.setInt(2, lot.lotId);
+ ps.executeUpdate();
+ }
+ }
+
+ /**
+ * Updates the closingNotified flag of a lot (set to 1 when we have
+ * warned the user about its imminent closure).
+ */
+ synchronized void updateLotNotificationFlags(Lot lot) throws SQLException {
+ try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(
+ "UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) {
+ ps.setInt(1, lot.closingNotified ? 1 : 0);
+ ps.setInt(2, lot.lotId);
+ ps.executeUpdate();
+ }
+ }
+ }
+
+ /**
+ * Service for sending notifications via desktop notifications and/or email.
+ * Supports free notification methods:
+ * 1. Desktop notifications (Windows/Linux/macOS system tray)
+ * 2. Email via Gmail SMTP (free, requires app password)
+ *
+ * Configuration:
+ * - For email: Set notificationEmail to your Gmail address
+ * - Enable 2FA in Gmail and create an App Password
+ * - Use format "smtp:username:appPassword:toEmail" for credentials
+ * - Or use "desktop" for desktop-only notifications
+ */
+ static class NotificationService {
+
+ private final boolean useDesktop;
+ private final boolean useEmail;
+ private final String smtpUsername;
+ private final String smtpPassword;
+ private final String toEmail;
+
+ /**
+ * Creates a notification service.
+ *
+ * @param config "desktop" for desktop only, or "smtp:username:password:toEmail" for email
+ * @param unusedParam Kept for compatibility (can pass empty string)
+ */
+ NotificationService(String config, String unusedParam) {
+
+ if ("desktop".equalsIgnoreCase(config)) {
+ this.useDesktop = true;
+ this.useEmail = false;
+ this.smtpUsername = null;
+ this.smtpPassword = null;
+ this.toEmail = null;
+ } else if (config.startsWith("smtp:")) {
+ String[] parts = config.split(":", 4);
+ if (parts.length != 4) {
+ throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'");
+ }
+ this.useDesktop = true; // Always include desktop
+ this.useEmail = true;
+ this.smtpUsername = parts[1];
+ this.smtpPassword = parts[2];
+ this.toEmail = parts[3];
+ } else {
+ throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'");
+ }
+ }
+
+ /**
+ * Sends notification via configured channels.
+ *
+ * @param message The message body
+ * @param title Message title
+ * @param priority Priority level (0=normal, 1=high)
+ */
+ void sendNotification(String message, String title, int priority) {
+ if (useDesktop) {
+ sendDesktopNotification(title, message, priority);
+ }
+ if (useEmail) {
+ sendEmailNotification(title, message, priority);
+ }
+ }
+
+ /**
+ * Sends a desktop notification using system tray.
+ * Works on Windows, macOS, and Linux with desktop environments.
+ */
+ private void sendDesktopNotification(String title, String message, int priority) {
+ try {
+ if (java.awt.SystemTray.isSupported()) {
+ java.awt.SystemTray tray = java.awt.SystemTray.getSystemTray();
+ java.awt.Image image = java.awt.Toolkit.getDefaultToolkit()
+ .createImage(new byte[0]); // Empty image
+
+ java.awt.TrayIcon trayIcon = new java.awt.TrayIcon(image, "Troostwijk Scraper");
+ trayIcon.setImageAutoSize(true);
+
+ java.awt.TrayIcon.MessageType messageType = priority > 0
+ ? java.awt.TrayIcon.MessageType.WARNING
+ : java.awt.TrayIcon.MessageType.INFO;
+
+ tray.add(trayIcon);
+ trayIcon.displayMessage(title, message, messageType);
+
+ // Remove icon after 2 seconds to avoid clutter
+ Thread.sleep(2000);
+ tray.remove(trayIcon);
+
+ System.out.println("Desktop notification sent: " + title);
} else {
- throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'");
+ System.out.println("Desktop notifications not supported, logging: " + title + " - " + message);
}
- }
-
- /**
- * Sends notification via configured channels.
- *
- * @param message The message body
- * @param title Message title
- * @param priority Priority level (0=normal, 1=high)
- */
- void sendNotification(String message, String title, int priority) {
- if (useDesktop) {
- sendDesktopNotification(title, message, priority);
+ } catch (Exception e) {
+ System.err.println("Desktop notification failed: " + e.getMessage());
+ }
+ }
+
+ /**
+ * Sends email notification via Gmail SMTP (free).
+ * Uses Gmail's SMTP server with app password authentication.
+ */
+ private void sendEmailNotification(String title, String message, int priority) {
+ try {
+ java.util.Properties props = new java.util.Properties();
+ props.put("mail.smtp.auth", "true");
+ props.put("mail.smtp.starttls.enable", "true");
+ props.put("mail.smtp.host", "smtp.gmail.com");
+ props.put("mail.smtp.port", "587");
+ props.put("mail.smtp.ssl.trust", "smtp.gmail.com");
+
+ javax.mail.Session session = javax.mail.Session.getInstance(props,
+ new javax.mail.Authenticator() {
+
+ protected javax.mail.PasswordAuthentication getPasswordAuthentication() {
+ return new javax.mail.PasswordAuthentication(smtpUsername, smtpPassword);
+ }
+ });
+
+ javax.mail.Message msg = new javax.mail.internet.MimeMessage(session);
+ msg.setFrom(new javax.mail.internet.InternetAddress(smtpUsername));
+ msg.setRecipients(javax.mail.Message.RecipientType.TO,
+ javax.mail.internet.InternetAddress.parse(toEmail));
+ msg.setSubject("[Troostwijk] " + title);
+ msg.setText(message);
+ msg.setSentDate(new java.util.Date());
+
+ if (priority > 0) {
+ msg.setHeader("X-Priority", "1");
+ msg.setHeader("Importance", "High");
}
- if (useEmail) {
- sendEmailNotification(title, message, priority);
+
+ javax.mail.Transport.send(msg);
+ System.out.println("Email notification sent: " + title);
+
+ } catch (Exception e) {
+ System.err.println("Email notification failed: " + e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Service for performing object detection on images using OpenCV's DNN
+ * module. The DNN module can load pre‑trained models from several
+ * frameworks (Darknet, TensorFlow, ONNX, etc.)【784097309529506†L209-L233】. Here
+ * we load a YOLO model (Darknet) by specifying the configuration and
+ * weights files. For each image we run a forward pass and return a
+ * list of detected class labels.
+ *
+ * If model files are not found, the service operates in disabled mode
+ * and returns empty lists.
+ */
+ static class ObjectDetectionService {
+
+ private final Net net;
+ private final List classNames;
+ private final boolean enabled;
+
+ ObjectDetectionService(String cfgPath, String weightsPath, String classNamesPath) throws IOException {
+ // Check if model files exist
+ Path cfgFile = Paths.get(cfgPath);
+ Path weightsFile = Paths.get(weightsPath);
+ Path classNamesFile = Paths.get(classNamesPath);
+
+ if (!Files.exists(cfgFile) || !Files.exists(weightsFile) || !Files.exists(classNamesFile)) {
+ System.out.println("⚠️ Object detection disabled: YOLO model files not found");
+ System.out.println(" Expected files:");
+ System.out.println(" - " + cfgPath);
+ System.out.println(" - " + weightsPath);
+ System.out.println(" - " + classNamesPath);
+ System.out.println(" Scraper will continue without image analysis.");
+ this.enabled = false;
+ this.net = null;
+ this.classNames = new ArrayList<>();
+ return;
+ }
+
+ try {
+ // Load network
+ this.net = Dnn.readNetFromDarknet(cfgPath, weightsPath);
+ this.net.setPreferableBackend(DNN_BACKEND_OPENCV);
+ this.net.setPreferableTarget(DNN_TARGET_CPU);
+ // Load class names (one per line)
+ this.classNames = Files.readAllLines(classNamesFile);
+ this.enabled = true;
+ System.out.println("✓ Object detection enabled with YOLO");
+ } catch (Exception e) {
+ System.err.println("⚠️ Object detection disabled: " + e.getMessage());
+ throw new IOException("Failed to initialize object detection", e);
+ }
+ }
+ /**
+ * Detects objects in the given image file and returns a list of
+ * human‑readable labels. Only detections above a confidence
+ * threshold are returned. For brevity this method omits drawing
+ * bounding boxes. See the OpenCV DNN documentation for details on
+ * post‑processing【784097309529506†L324-L344】.
+ *
+ * @param imagePath absolute path to the image
+ * @return list of detected class names (empty if detection disabled)
+ */
+ List detectObjects(String imagePath) {
+ if (!enabled) {
+ return new ArrayList<>();
+ }
+
+ List labels = new ArrayList<>();
+ Mat image = Imgcodecs.imread(imagePath);
+ if (image.empty()) return labels;
+ // Create a 4D blob from the image
+ Mat blob = Dnn.blobFromImage(image, 1.0 / 255.0, new Size(416, 416), new Scalar(0, 0, 0), true, false);
+ net.setInput(blob);
+ List outs = new ArrayList<>();
+ List outNames = getOutputLayerNames(net);
+ net.forward(outs, outNames);
+ // Post‑process: for each detection compute score and choose class
+ float confThreshold = 0.5f;
+ for (Mat out : outs) {
+ for (int i = 0; i < out.rows(); i++) {
+ double[] data = out.get(i, 0);
+ if (data == null) continue;
+ // The first 5 numbers are bounding box, then class scores
+ double[] scores = new double[classNames.size()];
+ System.arraycopy(data, 5, scores, 0, scores.length);
+ int classId = argMax(scores);
+ double confidence = scores[classId];
+ if (confidence > confThreshold) {
+ String label = classNames.get(classId);
+ if (!labels.contains(label)) {
+ labels.add(label);
+ }
+ }
}
- }
-
- /**
- * Sends a desktop notification using system tray.
- * Works on Windows, macOS, and Linux with desktop environments.
- */
- private void sendDesktopNotification(String title, String message, int priority) {
- try {
- if (java.awt.SystemTray.isSupported()) {
- java.awt.SystemTray tray = java.awt.SystemTray.getSystemTray();
- java.awt.Image image = java.awt.Toolkit.getDefaultToolkit()
- .createImage(new byte[0]); // Empty image
-
- java.awt.TrayIcon trayIcon = new java.awt.TrayIcon(image, "Troostwijk Scraper");
- trayIcon.setImageAutoSize(true);
-
- java.awt.TrayIcon.MessageType messageType = priority > 0
- ? java.awt.TrayIcon.MessageType.WARNING
- : java.awt.TrayIcon.MessageType.INFO;
-
- tray.add(trayIcon);
- trayIcon.displayMessage(title, message, messageType);
-
- // Remove icon after 2 seconds to avoid clutter
- Thread.sleep(2000);
- tray.remove(trayIcon);
-
- System.out.println("Desktop notification sent: " + title);
- } else {
- System.out.println("Desktop notifications not supported, logging: " + title + " - " + message);
- }
- } catch (Exception e) {
- System.err.println("Desktop notification failed: " + e.getMessage());
+ }
+ return labels;
+ }
+ /**
+ * Returns the indexes of the output layers in the network. YOLO
+ * automatically discovers its output layers; other models may require
+ * manually specifying them【784097309529506†L356-L365】.
+ */
+ private List getOutputLayerNames(Net net) {
+ List names = new ArrayList<>();
+ List outLayers = net.getUnconnectedOutLayers().toList();
+ List layersNames = net.getLayerNames();
+ for (Integer i : outLayers) {
+ names.add(layersNames.get(i - 1));
+ }
+ return names;
+ }
+ /**
+ * Returns the index of the maximum value in the array.
+ */
+ private int argMax(double[] array) {
+ int best = 0;
+ double max = array[0];
+ for (int i = 1; i < array.length; i++) {
+ if (array[i] > max) {
+ max = array[i];
+ best = i;
}
- }
-
- /**
- * Sends email notification via Gmail SMTP (free).
- * Uses Gmail's SMTP server with app password authentication.
- */
- private void sendEmailNotification(String title, String message, int priority) {
- try {
- java.util.Properties props = new java.util.Properties();
- props.put("mail.smtp.auth", "true");
- props.put("mail.smtp.starttls.enable", "true");
- props.put("mail.smtp.host", "smtp.gmail.com");
- props.put("mail.smtp.port", "587");
- props.put("mail.smtp.ssl.trust", "smtp.gmail.com");
-
- javax.mail.Session session = javax.mail.Session.getInstance(props,
- new javax.mail.Authenticator() {
- protected javax.mail.PasswordAuthentication getPasswordAuthentication() {
- return new javax.mail.PasswordAuthentication(smtpUsername, smtpPassword);
- }
- });
-
- javax.mail.Message msg = new javax.mail.internet.MimeMessage(session);
- msg.setFrom(new javax.mail.internet.InternetAddress(smtpUsername));
- msg.setRecipients(javax.mail.Message.RecipientType.TO,
- javax.mail.internet.InternetAddress.parse(toEmail));
- msg.setSubject("[Troostwijk] " + title);
- msg.setText(message);
- msg.setSentDate(new java.util.Date());
-
- if (priority > 0) {
- msg.setHeader("X-Priority", "1");
- msg.setHeader("Importance", "High");
- }
-
- javax.mail.Transport.send(msg);
- System.out.println("Email notification sent: " + title);
-
- } catch (Exception e) {
- System.err.println("Email notification failed: " + e.getMessage());
- }
- }
- }
-
- /**
- * Service for performing object detection on images using OpenCV's DNN
- * module. The DNN module can load pre‑trained models from several
- * frameworks (Darknet, TensorFlow, ONNX, etc.)【784097309529506†L209-L233】. Here
- * we load a YOLO model (Darknet) by specifying the configuration and
- * weights files. For each image we run a forward pass and return a
- * list of detected class labels.
- *
- * If model files are not found, the service operates in disabled mode
- * and returns empty lists.
- */
- static class ObjectDetectionService {
- private final Net net;
- private final List classNames;
- private final boolean enabled;
-
- ObjectDetectionService(String cfgPath, String weightsPath, String classNamesPath) throws IOException {
- // Check if model files exist
- Path cfgFile = Paths.get(cfgPath);
- Path weightsFile = Paths.get(weightsPath);
- Path classNamesFile = Paths.get(classNamesPath);
-
- if (!Files.exists(cfgFile) || !Files.exists(weightsFile) || !Files.exists(classNamesFile)) {
- System.out.println("⚠️ Object detection disabled: YOLO model files not found");
- System.out.println(" Expected files:");
- System.out.println(" - " + cfgPath);
- System.out.println(" - " + weightsPath);
- System.out.println(" - " + classNamesPath);
- System.out.println(" Scraper will continue without image analysis.");
- this.enabled = false;
- this.net = null;
- this.classNames = new ArrayList<>();
- return;
- }
-
- try {
- // Load network
- this.net = Dnn.readNetFromDarknet(cfgPath, weightsPath);
- this.net.setPreferableBackend(DNN_BACKEND_OPENCV);
- this.net.setPreferableTarget(DNN_TARGET_CPU);
- // Load class names (one per line)
- this.classNames = Files.readAllLines(classNamesFile);
- this.enabled = true;
- System.out.println("✓ Object detection enabled with YOLO");
- } catch (Exception e) {
- System.err.println("⚠️ Object detection disabled: " + e.getMessage());
- throw new IOException("Failed to initialize object detection", e);
- }
- }
- /**
- * Detects objects in the given image file and returns a list of
- * human‑readable labels. Only detections above a confidence
- * threshold are returned. For brevity this method omits drawing
- * bounding boxes. See the OpenCV DNN documentation for details on
- * post‑processing【784097309529506†L324-L344】.
- *
- * @param imagePath absolute path to the image
- * @return list of detected class names (empty if detection disabled)
- */
- List detectObjects(String imagePath) {
- if (!enabled) {
- return new ArrayList<>();
- }
-
- List labels = new ArrayList<>();
- Mat image = Imgcodecs.imread(imagePath);
- if (image.empty()) return labels;
- // Create a 4D blob from the image
- Mat blob = Dnn.blobFromImage(image, 1.0 / 255.0, new Size(416, 416), new Scalar(0, 0, 0), true, false);
- net.setInput(blob);
- List outs = new ArrayList<>();
- List outNames = getOutputLayerNames(net);
- net.forward(outs, outNames);
- // Post‑process: for each detection compute score and choose class
- float confThreshold = 0.5f;
- for (Mat out : outs) {
- for (int i = 0; i < out.rows(); i++) {
- double[] data = out.get(i, 0);
- if (data == null) continue;
- // The first 5 numbers are bounding box, then class scores
- double[] scores = new double[classNames.size()];
- System.arraycopy(data, 5, scores, 0, scores.length);
- int classId = argMax(scores);
- double confidence = scores[classId];
- if (confidence > confThreshold) {
- String label = classNames.get(classId);
- if (!labels.contains(label)) {
- labels.add(label);
- }
- }
- }
- }
- return labels;
- }
- /**
- * Returns the indexes of the output layers in the network. YOLO
- * automatically discovers its output layers; other models may require
- * manually specifying them【784097309529506†L356-L365】.
- */
- private List getOutputLayerNames(Net net) {
- List names = new ArrayList<>();
- List outLayers = net.getUnconnectedOutLayers().toList();
- List layersNames = net.getLayerNames();
- for (Integer i : outLayers) {
- names.add(layersNames.get(i - 1));
- }
- return names;
- }
- /**
- * Returns the index of the maximum value in the array.
- */
- private int argMax(double[] array) {
- int best = 0;
- double max = array[0];
- for (int i = 1; i < array.length; i++) {
- if (array[i] > max) {
- max = array[i];
- best = i;
- }
- }
- return best;
- }
- }
+ }
+ return best;
+ }
+ }
}
\ No newline at end of file
diff --git a/target/classes/com/auction/scraper/TroostwijkScraper$DatabaseService.class b/target/classes/com/auction/scraper/TroostwijkScraper$DatabaseService.class
index 7069d86..55d52d5 100644
Binary files a/target/classes/com/auction/scraper/TroostwijkScraper$DatabaseService.class and b/target/classes/com/auction/scraper/TroostwijkScraper$DatabaseService.class differ
diff --git a/target/classes/com/auction/scraper/TroostwijkScraper$Lot.class b/target/classes/com/auction/scraper/TroostwijkScraper$Lot.class
index 9af6273..23d3787 100644
Binary files a/target/classes/com/auction/scraper/TroostwijkScraper$Lot.class and b/target/classes/com/auction/scraper/TroostwijkScraper$Lot.class differ
diff --git a/target/classes/com/auction/scraper/TroostwijkScraper$NotificationService$1.class b/target/classes/com/auction/scraper/TroostwijkScraper$NotificationService$1.class
index 3ebb139..7539e9d 100644
Binary files a/target/classes/com/auction/scraper/TroostwijkScraper$NotificationService$1.class and b/target/classes/com/auction/scraper/TroostwijkScraper$NotificationService$1.class differ
diff --git a/target/classes/com/auction/scraper/TroostwijkScraper$NotificationService.class b/target/classes/com/auction/scraper/TroostwijkScraper$NotificationService.class
index 526cd75..f1950ab 100644
Binary files a/target/classes/com/auction/scraper/TroostwijkScraper$NotificationService.class and b/target/classes/com/auction/scraper/TroostwijkScraper$NotificationService.class differ
diff --git a/target/classes/com/auction/scraper/TroostwijkScraper$ObjectDetectionService.class b/target/classes/com/auction/scraper/TroostwijkScraper$ObjectDetectionService.class
index f82c6a4..a123796 100644
Binary files a/target/classes/com/auction/scraper/TroostwijkScraper$ObjectDetectionService.class and b/target/classes/com/auction/scraper/TroostwijkScraper$ObjectDetectionService.class differ
diff --git a/target/classes/com/auction/scraper/TroostwijkScraper.class b/target/classes/com/auction/scraper/TroostwijkScraper.class
index c04fa8f..479f691 100644
Binary files a/target/classes/com/auction/scraper/TroostwijkScraper.class and b/target/classes/com/auction/scraper/TroostwijkScraper.class differ