package com.auction;

/*
 * TroostwijkScraper
 *
 * This example shows how you could build a Java‐based scraper for the Dutch
 * auctions on Troostwijk Auctions.  The scraper uses a combination of
 * HTTP requests and HTML parsing with the jsoup library to discover active
 * auctions, calls Troostwijk's internal JSON API to fetch lot (kavel) data
 * efficiently, writes the results into a local SQLite database, performs
 * object detection on lot images using OpenCV's DNN module, and sends
 * desktop/email notifications when bids change or lots are about to expire.
 * The implementation uses well known open source libraries for each of these
 * concerns.  You can adjust the API endpoints and CSS selectors as
 * Troostwijk's site evolves.  The code is organised into small helper
 * classes to make it easier to maintain.
 *
 * Dependencies (add these to your Maven/Gradle project):
 *
 * - org.jsoup:jsoup:1.17.2 – HTML parser and HTTP client.
 * - com.fasterxml.jackson.core:jackson-databind:2.17.0 – JSON parsing.
 * - org.xerial:sqlite-jdbc:3.45.1.0 – SQLite JDBC driver.
 * - com.sun.mail:javax.mail:1.6.2 – JavaMail for email notifications (free).
 * - org.openpnp:opencv:4.9.0-0 (with native libraries) – OpenCV for image
 *   processing and object detection.
 *
 * Before running this program you must ensure that the native OpenCV
 * binaries are on your library path (e.g. via -Djava.library.path).
 * Desktop notifications work out of the box on Windows, macOS, and Linux.
 * For email notifications, you need a Gmail account with an app password
 * (free, requires 2FA enabled). See https://support.google.com/accounts/answer/185833
 *
 * The scraper performs four major tasks:
 * 1. Discover all auctions located in the Netherlands.
 * 2. For each auction, fetch all lots (kavels) including images and
 *    bidding information, and persist the data into SQLite tables.
 * 3. Monitor bidding and closing times on a schedule and send desktop/email
 *    notifications when bids change or lots are about to expire.
 * 4. Run object detection on downloaded lot images to automatically
 *    label objects using a YOLO model.  The results are stored in the
 *    database for later search.
 */

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgcodecs.Imgcodecs;
import static org.opencv.dnn.Dnn.DNN_BACKEND_OPENCV;
import static org.opencv.dnn.Dnn.DNN_TARGET_CPU;

/**
 * Main scraper class.  It encapsulates the logic for scraping auctions,
 * persisting data, scheduling updates, and performing object detection.
 */
public class TroostwijkScraper {
   
   // Base URLs – adjust these if Troostwijk changes their site structure
   private static final String AUCTIONS_PAGE = "https://www.troostwijkauctions.com/nl/auctions";
   private static final String LOT_API       = "https://api.troostwijkauctions.com/lot/7/list";
   
   // HTTP client used for API calls
   private final HttpClient             httpClient;
   private final ObjectMapper           objectMapper;
   public final  DatabaseService        db;
   private final NotificationService    notifier;
   private final ObjectDetectionService detector;
   
   /**
    * Constructor.  Creates supporting services and ensures the database
    * tables exist.
    *
    * @param databasePath Path to SQLite database file
    * @param notificationConfig "desktop" for desktop only, or "smtp:user:pass:toEmail" for email
    * @param unused Unused parameter (kept for compatibility)
    * @param yoloCfgPath Path to YOLO configuration file
    * @param yoloWeightsPath Path to YOLO weights file
    * @param classNamesPath Path to file containing class names
    */
   public TroostwijkScraper(String databasePath, String notificationConfig, String unused,
                            String yoloCfgPath, String yoloWeightsPath, String classNamesPath) throws SQLException, IOException {
      this.httpClient   = HttpClient.newHttpClient();
      this.objectMapper = new ObjectMapper();
      this.db           = new DatabaseService(databasePath);
      this.notifier     = new NotificationService(notificationConfig, unused);
      this.detector     = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
      // initialize DB
      db.ensureSchema();
   }
   
   /**
    * Discovers all active Dutch auctions by crawling the auctions page.
    *
    * Troostwijk lists auctions for many countries on one page.  We parse
    * the page with jsoup and filter auctions whose location contains ", NL"
    * (indicating the Netherlands).  Each auction link contains a unique sale ID
    * in the format A1-xxxxx or A7-xxxxx which we extract from the URL.
    *
    * @return a list of sale identifiers for auctions located in NL
    */
   public List<Integer> discoverDutchAuctions() {
      List<Integer> saleIds = new ArrayList<>();
      try {
         // Fetch the auctions overview page
         Document doc = Jsoup.connect(AUCTIONS_PAGE).get();
         
         // Select all anchor elements that link to auction pages
         // The URL pattern is: /a/auction-title-A1-xxxxx or /a/auction-title-A7-xxxxx
         Elements auctionLinks = doc.select("a[href^='/a/']");
         
         System.out.println("Found " + auctionLinks.size() + " potential auction links");
         
         for (Element link : auctionLinks) {
            // Get the href to extract the auction ID
            String href = link.attr("href");
            
            // Check if this link contains location text with ", NL"
            String linkText = link.text();
            
            // Look for location in any div inside the link
            Elements divs    = link.select("div");
            boolean  isDutch = false;
            for (Element div : divs) {
               String text = div.text();
               if (text.contains(", NL")) {
                  isDutch = true;
                  break;
               }
            }
            
            if (isDutch) {
               // Extract auction ID from URL
               // Format: /a/title-A1-38375 or /a/title-A7-12345
               // We want the number after A1- or A7-
               java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("A[17]-(\\d+)");
               java.util.regex.Matcher matcher = pattern.matcher(href);
               
               if (matcher.find()) {
                  try {
                     int saleId = Integer.parseInt(matcher.group(1));
                     if (!saleIds.contains(saleId)) {
                        saleIds.add(saleId);
                        System.out.println("  Found Dutch auction: " + saleId + " - " + href);
                     }
                  } catch (NumberFormatException e) {
                     // Skip invalid IDs
                  }
               }
            }
         }
      } catch (IOException e) {
         System.err.println("Failed to discover auctions: " + e.getMessage());
         e.printStackTrace();
      }
      return saleIds;
   }
   
   /**
    * Retrieves all lots for a given sale ID using Troostwijk's internal JSON
    * API.  The API accepts parameters such as batchSize, offset, and saleID.
    * A large batchSize returns many lots at once.  We loop until no further
    * results are returned.  Each JSON result is mapped to our Lot domain
    * object and persisted to the database.
    *
    * @param saleId the sale identifier
    */
   public void fetchLotsForSale(int saleId) {
      int     batchSize = 200;
      int     offset    = 0;
      boolean more      = true;
      int     totalLots = 0;
      
      while (more) {
         try {
            String url = LOT_API + "?batchSize=" + batchSize
                         + "&listType=7&offset=" + offset
                         + "&sortOption=0&saleID=" + saleId
                         + "&parentID=0&relationID=0&buildversion=201807311";
            
            System.out.println("    Fetching lots from API (offset=" + offset + ")...");
            
            HttpRequest request = HttpRequest.newBuilder()
                                             .uri(URI.create(url))
                                             .header("Accept", "application/json")
                                             .header("User-Agent", "Mozilla/5.0")
                                             .GET()
                                             .build();
            
            HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
            
            if (response.statusCode() != 200) {
               System.err.println("    ⚠️  API call failed for sale " + saleId);
               System.err.println("    Status: " + response.statusCode());
               System.err.println("    Response: " + response.body().substring(0, Math.min(200, response.body().length())));
               break;
            }
            
            JsonNode root    = objectMapper.readTree(response.body());
            JsonNode results = root.path("results");
            
            if (!results.isArray() || results.isEmpty()) {
               if (offset == 0) {
                  System.out.println("    ⚠️  No lots found for sale " + saleId);
                  System.out.println("    API Response: " + response.body().substring(0, Math.min(500, response.body().length())));
               }
               more = false;
               break;
            }
            int lotsInBatch = results.size();
            System.out.println("    Found " + lotsInBatch + " lots in this batch");
            
            for (JsonNode node : results) {
               Lot lot = new Lot();
               lot.saleId       = saleId;
               lot.lotId        = node.path("lotID").asInt();
               lot.title        = node.path("t").asText();
               lot.description  = node.path("d").asText();
               lot.manufacturer = node.path("mf").asText();
               lot.type         = node.path("typ").asText();
               lot.year         = node.path("yb").asInt();
               lot.category     = node.path("lc").asText();
               // Current bid; field names may differ (e.g. currentBid or cb)
               lot.currentBid = node.path("cb").asDouble();
               lot.currency   = node.path("cu").asText();
               lot.url        = "https://www.troostwijkauctions.com/nl" + node.path("url").asText();
               
               // Save basic lot info into DB
               db.upsertLot(lot);
               totalLots++;
               
               // Download images and perform object detection
               List<String> imageUrls = new ArrayList<>();
               JsonNode     imgs      = node.path("imgs");
               if (imgs.isArray()) {
                  for (JsonNode imgNode : imgs) {
                     String imgUrl = imgNode.asText();
                     imageUrls.add(imgUrl);
                  }
               }
               
               // Download and analyze images (optional, can be slow)
               for (String imgUrl : imageUrls) {
                  String fileName = downloadImage(imgUrl, saleId, lot.lotId);
                  if (fileName != null) {
                     // run object detection once per image
                     List<String> labels = detector.detectObjects(fileName);
                     db.insertImage(lot.lotId, imgUrl, fileName, labels);
                  }
               }
            }
            
            System.out.println("    ✓ Processed " + totalLots + " lots so far");
            offset += batchSize;
         } catch (IOException | InterruptedException e) {
            System.err.println("Error fetching lots for sale " + saleId + ": " + e.getMessage());
            more = false;
         } catch (SQLException e) {
            System.err.println("Database error: " + e.getMessage());
         }
      }
   }
   
   /**
    * Downloads an image from the given URL to a local directory.  Images
    * are stored under "images/<saleId>/<lotId>/" to keep them organised.
    *
    * @param imageUrl remote image URL
    * @param saleId   sale identifier
    * @param lotId    lot identifier
    * @return absolute path to saved file or null on failure
    */
   private String downloadImage(String imageUrl, int saleId, int lotId) {
      try {
         HttpRequest request = HttpRequest.newBuilder()
                                          .uri(URI.create(imageUrl))
                                          .GET()
                                          .build();
         HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
         if (response.statusCode() == 200) {
            Path dir = Paths.get("images", String.valueOf(saleId), String.valueOf(lotId));
            Files.createDirectories(dir);
            String fileName = Paths.get(imageUrl).getFileName().toString();
            Path   dest     = dir.resolve(fileName);
            Files.copy(response.body(), dest);
            return dest.toAbsolutePath().toString();
         }
      } catch (IOException | InterruptedException e) {
         System.err.println("Failed to download image " + imageUrl + ": " + e.getMessage());
      }
      return null;
   }
   
   /**
    * Schedules periodic monitoring of all lots.  The scheduler runs every
    * hour to refresh current bids and closing times.  For lots that
    * are within 30 minutes of closing, it increases the polling frequency
    * automatically.  When a new bid is detected or a lot is about to
    * expire, a Pushover notification is sent to the configured user.
    * Note: In production, ensure proper shutdown handling for the scheduler.
    */
   public ScheduledExecutorService scheduleMonitoring() {
      ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
      scheduler.scheduleAtFixedRate(() -> {
         try {
            List<Lot> activeLots = db.getActiveLots();
            for (Lot lot : activeLots) {
               // refresh the lot's bidding information via API
               refreshLotBid(lot);
               // check closing time to adjust monitoring
               long minutesLeft = lot.minutesUntilClose();
               if (minutesLeft < 30) {
                  // send warning when within 5 minutes
                  if (minutesLeft <= 5 && !lot.closingNotified) {
                     notifier.sendNotification("Kavel " + lot.lotId + " sluit binnen " + minutesLeft + " min.",
                                               "Lot nearing closure", 1);
                     lot.closingNotified = true;
                     db.updateLotNotificationFlags(lot);
                  }
                  // schedule additional quick check for this lot
                  scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
               }
            }
         } catch (SQLException e) {
            System.err.println("Error during scheduled monitoring: " + e.getMessage());
         }
      }, 0, 1, TimeUnit.HOURS);
      return scheduler;
   }
   
   /**
    * Refreshes the bid for a single lot and sends notification if it has
    * changed since the last check.  The method calls the same API used for
    * initial scraping but only extracts the current bid for the given lot.
    *
    * @param lot the lot to refresh
    */
   private void refreshLotBid(Lot lot) {
      try {
         String url = LOT_API + "?batchSize=1&listType=7&offset=0&sortOption=0&saleID=" + lot.saleId
                      + "&parentID=0&relationID=0&buildversion=201807311&lotID=" + lot.lotId;
         HttpRequest          request  = HttpRequest.newBuilder().uri(URI.create(url)).GET().build();
         HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
         if (response.statusCode() != 200) return;
         JsonNode root    = objectMapper.readTree(response.body());
         JsonNode results = root.path("results");
         if (results.isArray() && !results.isEmpty()) {
            JsonNode node   = results.get(0);
            double   newBid = node.path("cb").asDouble();
            if (Double.compare(newBid, lot.currentBid) > 0) {
               double previous = lot.currentBid;
               lot.currentBid = newBid;
               db.updateLotCurrentBid(lot);
               String msg = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)", lot.lotId, newBid, previous);
               notifier.sendNotification(msg, "Kavel bieding update", 0);
            }
         }
      } catch (IOException | InterruptedException | SQLException e) {
         System.err.println("Failed to refresh bid for lot " + lot.lotId + ": " + e.getMessage());
      }
   }
   
   /**
    * Entry point.  Configure database location, notification settings, and
    * YOLO model paths here before running.  Once started the scraper
    * discovers Dutch auctions, scrapes lots, and begins monitoring.
    */
   public static void main(String[] args) throws Exception {
      System.out.println("=== Troostwijk Auction Scraper ===\n");
      
      // Configuration parameters (replace with your own values)
      String databaseFile = "troostwijk.db";
      
      // Notification configuration - choose one:
      // Option 1: Desktop notifications only (free, no setup required)
      String notificationConfig = System.getenv().getOrDefault("NOTIFICATION_CONFIG", "desktop");
      
      // Option 2: Desktop + Email via Gmail (free, requires Gmail app password)
      // Format: "smtp:username:appPassword:toEmail"
      // Example: "smtp:your.email@gmail.com:abcd1234efgh5678:recipient@example.com"
      // Get app password: Google Account > Security > 2-Step Verification > App passwords
      
      // YOLO model paths (optional - scraper works without object detection)
      String yoloCfg     = "models/yolov4.cfg";
      String yoloWeights = "models/yolov4.weights";
      String yoloClasses = "models/coco.names";
      
      // Load native OpenCV library
      System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
      
      System.out.println("Initializing scraper...");
      TroostwijkScraper scraper = new TroostwijkScraper(databaseFile, notificationConfig, "",
                                                        yoloCfg, yoloWeights, yoloClasses);
      
      // Step 1: Discover auctions in NL
      System.out.println("\n[1/3] Discovering Dutch auctions...");
      List<Integer> auctions = scraper.discoverDutchAuctions();
      System.out.println("✓ Found " + auctions.size() + " auctions: " + auctions);
      
      // Step 2: Fetch lots for each auction
      System.out.println("\n[2/3] Fetching lot details...");
      int totalAuctions  = auctions.size();
      int currentAuction = 0;
      for (int saleId : auctions) {
         currentAuction++;
         System.out.println("  [Page " + currentAuction + "] Fetching auctions...");
         System.out.println("  [" + currentAuction + "/" + totalAuctions + "] Processing sale " + saleId + "...");
         scraper.fetchLotsForSale(saleId);
      }
      
      // Show database summary
      System.out.println("\n📊 Database Summary:");
      scraper.printDatabaseStats();
      
      // Step 3: Start monitoring bids and closures
      System.out.println("\n[3/3] Starting monitoring service...");
      scraper.scheduleMonitoring();
      System.out.println("✓ Monitoring active. Press Ctrl+C to stop.\n");
   }
   
   /**
    * Prints statistics about the data in the database.
    */
   private void printDatabaseStats() {
      try {
         List<Lot> allLots    = db.getAllLots();
         int       imageCount = db.getImageCount();
         
         System.out.println("  Total lots in database: " + allLots.size());
         System.out.println("  Total images downloaded: " + imageCount);
         
         if (!allLots.isEmpty()) {
            double totalBids = allLots.stream().mapToDouble(l -> l.currentBid).sum();
            System.out.println("  Total current bids: €" + String.format("%.2f", totalBids));
         }
      } catch (SQLException e) {
         System.err.println("  ⚠️  Could not retrieve database stats: " + e.getMessage());
      }
   }
   
   // ----------------------------------------------------------------------
   // Domain classes and services
   // ----------------------------------------------------------------------
   
   /**
    * Simple POJO representing a lot (kavel) in an auction.  It keeps track
    * of the sale it belongs to, current bid and closing time.  The method
    * minutesUntilClose computes how many minutes remain until the lot closes.
    */
   static class Lot {
      
      int           saleId;
      int           lotId;
      String        title;
      String        description;
      String        manufacturer;
      String        type;
      int           year;
      String        category;
      double        currentBid;
      String        currency;
      String        url;
      LocalDateTime closingTime; // null if unknown
      boolean       closingNotified;
      
      long minutesUntilClose() {
         if (closingTime == null) return Long.MAX_VALUE;
         return java.time.Duration.between(LocalDateTime.now(), closingTime).toMinutes();
      }
   }
   
   /**
    * Service for persisting auctions, lots, images, and object labels into
    * a SQLite database.  Uses the Xerial JDBC driver which connects to
    * SQLite via a URL of the form "jdbc:sqlite:path_to_file"【329850066306528†L40-L63】.
    */
   static public class DatabaseService {
      
      private final String url;
      DatabaseService(String dbPath) {
         this.url = "jdbc:sqlite:" + dbPath;
      }
      /**
       * Creates tables if they do not already exist.  The schema includes
       * tables for sales, lots, images, and object labels.  This method is
       * idempotent; it can be called multiple times.
       */
      void ensureSchema() throws SQLException {
         try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
            // Sales table
            stmt.execute("CREATE TABLE IF NOT EXISTS sales ("
                         + "sale_id INTEGER PRIMARY KEY,"
                         + "title TEXT,"
                         + "location TEXT,"
                         + "closing_time TEXT"
                         + ")");
            // Lots table
            stmt.execute("CREATE TABLE IF NOT EXISTS lots ("
                         + "lot_id INTEGER PRIMARY KEY,"
                         + "sale_id INTEGER,"
                         + "title TEXT,"
                         + "description TEXT,"
                         + "manufacturer TEXT,"
                         + "type TEXT,"
                         + "year INTEGER,"
                         + "category TEXT,"
                         + "current_bid REAL,"
                         + "currency TEXT,"
                         + "url TEXT,"
                         + "closing_time TEXT,"
                         + "closing_notified INTEGER DEFAULT 0,"
                         + "FOREIGN KEY (sale_id) REFERENCES sales(sale_id)"
                         + ")");
            // Images table
            stmt.execute("CREATE TABLE IF NOT EXISTS images ("
                         + "id INTEGER PRIMARY KEY AUTOINCREMENT,"
                         + "lot_id INTEGER,"
                         + "url TEXT,"
                         + "file_path TEXT,"
                         + "labels TEXT,"
                         + "FOREIGN KEY (lot_id) REFERENCES lots(lot_id)"
                         + ")");
         }
      }
      
      /**
       * Inserts or updates a lot record.  Uses INSERT OR REPLACE to
       * implement upsert semantics so that existing rows are replaced.
       */
      synchronized void upsertLot(Lot lot) throws SQLException {
         String sql = "INSERT INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)"
                      + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
                      + " ON CONFLICT(lot_id) DO UPDATE SET "
                      + "sale_id = excluded.sale_id, title = excluded.title, description = excluded.description, "
                      + "manufacturer = excluded.manufacturer, type = excluded.type, year = excluded.year, category = excluded.category, "
                      + "current_bid = excluded.current_bid, currency = excluded.currency, url = excluded.url, closing_time = excluded.closing_time";
         try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(sql)) {
            ps.setInt(1, lot.lotId);
            ps.setInt(2, lot.saleId);
            ps.setString(3, lot.title);
            ps.setString(4, lot.description);
            ps.setString(5, lot.manufacturer);
            ps.setString(6, lot.type);
            ps.setInt(7, lot.year);
            ps.setString(8, lot.category);
            ps.setDouble(9, lot.currentBid);
            ps.setString(10, lot.currency);
            ps.setString(11, lot.url);
            ps.setString(12, lot.closingTime != null ? lot.closingTime.toString() : null);
            ps.setInt(13, lot.closingNotified ? 1 : 0);
            ps.executeUpdate();
         }
      }
      
      /**
       * Inserts a new image record.  Each image is associated with a lot and
       * stores both the original URL and the local file path.  Detected
       * labels are stored as a comma separated string.
       */
      synchronized void insertImage(int lotId, String url, String filePath, List<String> labels) throws SQLException {
         String sql = "INSERT INTO images (lot_id, url, file_path, labels) VALUES (?, ?, ?, ?)";
         try (Connection conn = DriverManager.getConnection(this.url); PreparedStatement ps = conn.prepareStatement(sql)) {
            ps.setInt(1, lotId);
            ps.setString(2, url);
            ps.setString(3, filePath);
            ps.setString(4, String.join(",", labels));
            ps.executeUpdate();
         }
      }
      
      /**
       * Retrieves all lots that are still active (i.e., have a closing time
       * in the future or unknown).  Only these lots need to be monitored.
       */
      synchronized List<Lot> getActiveLots() throws SQLException {
         List<Lot> list = new ArrayList<>();
         String    sql  = "SELECT lot_id, sale_id, current_bid, currency, closing_time, closing_notified FROM lots";
         try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
            ResultSet rs = stmt.executeQuery(sql);
            while (rs.next()) {
               Lot lot = new Lot();
               lot.lotId      = rs.getInt("lot_id");
               lot.saleId     = rs.getInt("sale_id");
               lot.currentBid = rs.getDouble("current_bid");
               lot.currency   = rs.getString("currency");
               String closing = rs.getString("closing_time");
               lot.closingNotified = rs.getInt("closing_notified") != 0;
               if (closing != null) {
                  lot.closingTime = LocalDateTime.parse(closing);
               }
               list.add(lot);
            }
         }
         return list;
      }
      
      /**
       * Retrieves all lots from the database.
       */
      synchronized List<Lot> getAllLots() throws SQLException {
         List<Lot> list = new ArrayList<>();
         String    sql  = "SELECT lot_id, sale_id, title, current_bid, currency FROM lots";
         try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
            ResultSet rs = stmt.executeQuery(sql);
            while (rs.next()) {
               Lot lot = new Lot();
               lot.lotId      = rs.getInt("lot_id");
               lot.saleId     = rs.getInt("sale_id");
               lot.title      = rs.getString("title");
               lot.currentBid = rs.getDouble("current_bid");
               lot.currency   = rs.getString("currency");
               list.add(lot);
            }
         }
         return list;
      }
      
      /**
       * Gets the total number of images in the database.
       */
      synchronized int getImageCount() throws SQLException {
         String sql = "SELECT COUNT(*) as count FROM images";
         try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
            ResultSet rs = stmt.executeQuery(sql);
            if (rs.next()) {
               return rs.getInt("count");
            }
         }
         return 0;
      }
      
      /**
       * Updates the current bid of a lot after a bid refresh.
       */
      synchronized void updateLotCurrentBid(Lot lot) throws SQLException {
         try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(
                 "UPDATE lots SET current_bid = ? WHERE lot_id = ?")) {
            ps.setDouble(1, lot.currentBid);
            ps.setInt(2, lot.lotId);
            ps.executeUpdate();
         }
      }
      
      /**
       * Updates the closingNotified flag of a lot (set to 1 when we have
       * warned the user about its imminent closure).
       */
      synchronized void updateLotNotificationFlags(Lot lot) throws SQLException {
         try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(
                 "UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) {
            ps.setInt(1, lot.closingNotified ? 1 : 0);
            ps.setInt(2, lot.lotId);
            ps.executeUpdate();
         }
      }
   }
   
   /**
    * Service for sending notifications via desktop notifications and/or email.
    * Supports free notification methods:
    * 1. Desktop notifications (Windows/Linux/macOS system tray)
    * 2. Email via Gmail SMTP (free, requires app password)
    *
    * Configuration:
    * - For email: Set notificationEmail to your Gmail address
    * - Enable 2FA in Gmail and create an App Password
    * - Use format "smtp:username:appPassword:toEmail" for credentials
    * - Or use "desktop" for desktop-only notifications
    */
   static class NotificationService {
      
      private final boolean useDesktop;
      private final boolean useEmail;
      private final String  smtpUsername;
      private final String  smtpPassword;
      private final String  toEmail;
      
      /**
       * Creates a notification service.
       *
       * @param config "desktop" for desktop only, or "smtp:username:password:toEmail" for email
       * @param unusedParam Kept for compatibility (can pass empty string)
       */
      NotificationService(String config, String unusedParam) {
         
         if ("desktop".equalsIgnoreCase(config)) {
            this.useDesktop   = true;
            this.useEmail     = false;
            this.smtpUsername = null;
            this.smtpPassword = null;
            this.toEmail      = null;
         } else if (config.startsWith("smtp:")) {
            String[] parts = config.split(":", 4);
            if (parts.length != 4) {
               throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'");
            }
            this.useDesktop   = true;  // Always include desktop
            this.useEmail     = true;
            this.smtpUsername = parts[1];
            this.smtpPassword = parts[2];
            this.toEmail      = parts[3];
         } else {
            throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'");
         }
      }
      
      /**
       * Sends notification via configured channels.
       *
       * @param message The message body
       * @param title Message title
       * @param priority Priority level (0=normal, 1=high)
       */
      void sendNotification(String message, String title, int priority) {
         if (useDesktop) {
            sendDesktopNotification(title, message, priority);
         }
         if (useEmail) {
            sendEmailNotification(title, message, priority);
         }
      }
      
      /**
       * Sends a desktop notification using system tray.
       * Works on Windows, macOS, and Linux with desktop environments.
       */
      private void sendDesktopNotification(String title, String message, int priority) {
         try {
            if (java.awt.SystemTray.isSupported()) {
               java.awt.SystemTray tray = java.awt.SystemTray.getSystemTray();
               java.awt.Image image = java.awt.Toolkit.getDefaultToolkit()
                                                      .createImage(new byte[0]); // Empty image
               
               java.awt.TrayIcon trayIcon = new java.awt.TrayIcon(image, "Troostwijk Scraper");
               trayIcon.setImageAutoSize(true);
               
               java.awt.TrayIcon.MessageType messageType = priority > 0
                                                           ? java.awt.TrayIcon.MessageType.WARNING
                                                           : java.awt.TrayIcon.MessageType.INFO;
               
               tray.add(trayIcon);
               trayIcon.displayMessage(title, message, messageType);
               
               // Remove icon after 2 seconds to avoid clutter
               Thread.sleep(2000);
               tray.remove(trayIcon);
               
               System.out.println("Desktop notification sent: " + title);
            } else {
               System.out.println("Desktop notifications not supported, logging: " + title + " - " + message);
            }
         } catch (Exception e) {
            System.err.println("Desktop notification failed: " + e.getMessage());
         }
      }
      
      /**
       * Sends email notification via Gmail SMTP (free).
       * Uses Gmail's SMTP server with app password authentication.
       */
      private void sendEmailNotification(String title, String message, int priority) {
         try {
            java.util.Properties props = new java.util.Properties();
            props.put("mail.smtp.auth", "true");
            props.put("mail.smtp.starttls.enable", "true");
            props.put("mail.smtp.host", "smtp.gmail.com");
            props.put("mail.smtp.port", "587");
            props.put("mail.smtp.ssl.trust", "smtp.gmail.com");
            
            javax.mail.Session session = javax.mail.Session.getInstance(props,
                                                                        new javax.mail.Authenticator() {
                                                                           
                                                                           protected javax.mail.PasswordAuthentication getPasswordAuthentication() {
                                                                              return new javax.mail.PasswordAuthentication(smtpUsername, smtpPassword);
                                                                           }
                                                                        });
            
            javax.mail.Message msg = new javax.mail.internet.MimeMessage(session);
            msg.setFrom(new javax.mail.internet.InternetAddress(smtpUsername));
            msg.setRecipients(javax.mail.Message.RecipientType.TO,
                              javax.mail.internet.InternetAddress.parse(toEmail));
            msg.setSubject("[Troostwijk] " + title);
            msg.setText(message);
            msg.setSentDate(new java.util.Date());
            
            if (priority > 0) {
               msg.setHeader("X-Priority", "1");
               msg.setHeader("Importance", "High");
            }
            
            javax.mail.Transport.send(msg);
            System.out.println("Email notification sent: " + title);
            
         } catch (Exception e) {
            System.err.println("Email notification failed: " + e.getMessage());
         }
      }
   }
   
   /**
    * Service for performing object detection on images using OpenCV's DNN
    * module.  The DNN module can load pre‑trained models from several
    * frameworks (Darknet, TensorFlow, ONNX, etc.)【784097309529506†L209-L233】.  Here
    * we load a YOLO model (Darknet) by specifying the configuration and
    * weights files.  For each image we run a forward pass and return a
    * list of detected class labels.
    *
    * If model files are not found, the service operates in disabled mode
    * and returns empty lists.
    */
   static class ObjectDetectionService {
      
      private final Net          net;
      private final List<String> classNames;
      private final boolean      enabled;
      
      ObjectDetectionService(String cfgPath, String weightsPath, String classNamesPath) throws IOException {
         // Check if model files exist
         Path cfgFile        = Paths.get(cfgPath);
         Path weightsFile    = Paths.get(weightsPath);
         Path classNamesFile = Paths.get(classNamesPath);
         
         if (!Files.exists(cfgFile) || !Files.exists(weightsFile) || !Files.exists(classNamesFile)) {
            System.out.println("⚠️  Object detection disabled: YOLO model files not found");
            System.out.println("   Expected files:");
            System.out.println("   - " + cfgPath);
            System.out.println("   - " + weightsPath);
            System.out.println("   - " + classNamesPath);
            System.out.println("   Scraper will continue without image analysis.");
            this.enabled    = false;
            this.net        = null;
            this.classNames = new ArrayList<>();
            return;
         }
         
         try {
            // Load network
            this.net = Dnn.readNetFromDarknet(cfgPath, weightsPath);
            this.net.setPreferableBackend(DNN_BACKEND_OPENCV);
            this.net.setPreferableTarget(DNN_TARGET_CPU);
            // Load class names (one per line)
            this.classNames = Files.readAllLines(classNamesFile);
            this.enabled    = true;
            System.out.println("✓ Object detection enabled with YOLO");
         } catch (Exception e) {
            System.err.println("⚠️  Object detection disabled: " + e.getMessage());
            throw new IOException("Failed to initialize object detection", e);
         }
      }
      /**
       * Detects objects in the given image file and returns a list of
       * human‑readable labels.  Only detections above a confidence
       * threshold are returned.  For brevity this method omits drawing
       * bounding boxes.  See the OpenCV DNN documentation for details on
       * post‑processing【784097309529506†L324-L344】.
       *
       * @param imagePath absolute path to the image
       * @return list of detected class names (empty if detection disabled)
       */
      List<String> detectObjects(String imagePath) {
         if (!enabled) {
            return new ArrayList<>();
         }
         
         List<String> labels = new ArrayList<>();
         Mat          image  = Imgcodecs.imread(imagePath);
         if (image.empty()) return labels;
         // Create a 4D blob from the image
         Mat blob = Dnn.blobFromImage(image, 1.0 / 255.0, new Size(416, 416), new Scalar(0, 0, 0), true, false);
         net.setInput(blob);
         List<Mat>    outs     = new ArrayList<>();
         List<String> outNames = getOutputLayerNames(net);
         net.forward(outs, outNames);
         // Post‑process: for each detection compute score and choose class
         float confThreshold = 0.5f;
         for (Mat out : outs) {
            for (int i = 0; i < out.rows(); i++) {
               double[] data = out.get(i, 0);
               if (data == null) continue;
               // The first 5 numbers are bounding box, then class scores
               double[] scores = new double[classNames.size()];
               System.arraycopy(data, 5, scores, 0, scores.length);
               int    classId    = argMax(scores);
               double confidence = scores[classId];
               if (confidence > confThreshold) {
                  String label = classNames.get(classId);
                  if (!labels.contains(label)) {
                     labels.add(label);
                  }
               }
            }
         }
         return labels;
      }
      /**
       * Returns the indexes of the output layers in the network.  YOLO
       * automatically discovers its output layers; other models may require
       * manually specifying them【784097309529506†L356-L365】.
       */
      private List<String> getOutputLayerNames(Net net) {
         List<String>  names       = new ArrayList<>();
         List<Integer> outLayers   = net.getUnconnectedOutLayers().toList();
         List<String>  layersNames = net.getLayerNames();
         for (Integer i : outLayers) {
            names.add(layersNames.get(i - 1));
         }
         return names;
      }
      /**
       * Returns the index of the maximum value in the array.
       */
      private int argMax(double[] array) {
         int    best = 0;
         double max  = array[0];
         for (int i = 1; i < array.length; i++) {
            if (array[i] > max) {
               max  = array[i];
               best = i;
            }
         }
         return best;
      }
   }
}