package com.auction;

/*
 * TroostwijkScraper
 *
 * This example shows how you could build a Java‐based scraper for the Dutch
 * auctions on Troostwijk Auctions.  The scraper uses a combination of
 * HTTP requests and HTML parsing with the jsoup library to discover active
 * auctions, calls Troostwijk's internal JSON API to fetch lot (kavel) data
 * efficiently, writes the results into a local SQLite database, performs
 * object detection on lot images using OpenCV's DNN module, and sends
 * desktop/email notifications when bids change or lots are about to expire.
 * The implementation uses well known open source libraries for each of these
 * concerns.  You can adjust the API endpoints and CSS selectors as
 * Troostwijk's site evolves.  The code is organised into small helper
 * classes to make it easier to maintain.
 *
 * Dependencies (add these to your Maven/Gradle project):
 *
 * - org.jsoup:jsoup:1.17.2 – HTML parser and HTTP client.
 * - com.fasterxml.jackson.core:jackson-databind:2.17.0 – JSON parsing.
 * - org.xerial:sqlite-jdbc:3.45.1.0 – SQLite JDBC driver.
 * - com.sun.mail:javax.mail:1.6.2 – JavaMail for email notifications (free).
 * - org.openpnp:opencv:4.9.0-0 (with native libraries) – OpenCV for image
 *   processing and object detection.
 *
 * Before running this program you must ensure that the native OpenCV
 * binaries are on your library path (e.g. via -Djava.library.path).
 * Desktop notifications work out of the box on Windows, macOS, and Linux.
 * For email notifications, you need a Gmail account with an app password
 * (free, requires 2FA enabled). See https://support.google.com/accounts/answer/185833
 *
 * The scraper performs four major tasks:
 * 1. Discover all auctions located in the Netherlands.
 * 2. For each auction, fetch all lots (kavels) including images and
 *    bidding information, and persist the data into SQLite tables.
 * 3. Monitor bidding and closing times on a schedule and send desktop/email
 *    notifications when bids change or lots are about to expire.
 * 4. Run object detection on downloaded lot images to automatically
 *    label objects using a YOLO model.  The results are stored in the
 *    database for later search.
 */

import com.fasterxml.jackson.databind.ObjectMapper;
import com.microsoft.playwright.Browser;
import com.microsoft.playwright.BrowserType;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.Playwright;
import com.microsoft.playwright.options.WaitUntilState;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

/**
 * Main scraper class.  It encapsulates the logic for scraping auctions,
 * persisting data, scheduling updates, and performing object detection.
 */
public class TroostwijkScraper {

   // Base URLs – adjust these if Troostwijk changes their site structure
   private static final String AUCTIONS_PAGE = "https://www.troostwijkauctions.com/auctions";
   private static final String LOT_API       = "https://api.troostwijkauctions.com/lot/7/list";
   private static final String CACHE_DB_PATH = "cache/page_cache.db";
   private static final long CACHE_EXPIRATION_HOURS = 24;
   private static final int RATE_LIMIT_MS = 200;

   // HTTP client used for API calls
   private final HttpClient             httpClient;
   private final ObjectMapper           objectMapper;
   public final  DatabaseService        db;
   private final NotificationService    notifier;
   private final ObjectDetectionService detector;
   private final CacheDatabase          cacheDb;
   private final boolean                useCache;
   private Playwright                   playwright;
   private Browser                      browser;
   
   /**
    * Constructor.  Creates supporting services and ensures the database
    * tables exist.
    *
    * @param databasePath Path to SQLite database file
    * @param notificationConfig "desktop" for desktop only, or "smtp:user:pass:toEmail" for email
    * @param unused Unused parameter (kept for compatibility)
    * @param yoloCfgPath Path to YOLO configuration file
    * @param yoloWeightsPath Path to YOLO weights file
    * @param classNamesPath Path to file containing class names
    */
   public TroostwijkScraper(String databasePath, String notificationConfig, String unused,
                            String yoloCfgPath, String yoloWeightsPath, String classNamesPath) throws SQLException, IOException {
      this(databasePath, notificationConfig, unused, yoloCfgPath, yoloWeightsPath, classNamesPath, true);
   }

   /**
    * Constructor with cache control.
    *
    * @param databasePath Path to SQLite database file
    * @param notificationConfig "desktop" for desktop only, or "smtp:user:pass:toEmail" for email
    * @param unused Unused parameter (kept for compatibility)
    * @param yoloCfgPath Path to YOLO configuration file
    * @param yoloWeightsPath Path to YOLO weights file
    * @param classNamesPath Path to file containing class names
    * @param useCache Enable page caching
    */
   public TroostwijkScraper(String databasePath, String notificationConfig, String unused,
                            String yoloCfgPath, String yoloWeightsPath, String classNamesPath,
                            boolean useCache) throws SQLException, IOException {
      this.httpClient   = HttpClient.newHttpClient();
      this.objectMapper = new ObjectMapper();
      this.db           = new DatabaseService(databasePath);
      this.notifier     = new NotificationService(notificationConfig, unused);
      this.detector     = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
      this.useCache     = useCache;
      this.cacheDb      = useCache ? new CacheDatabase(CACHE_DB_PATH) : null;

      // initialize DB
      db.ensureSchema();
      if (useCache) {
         cacheDb.initialize();
      }
   }

   /**
    * Initializes Playwright browser for JavaScript-rendered pages.
    * Call this before using discoverDutchAuctions().
    */
   public void initializeBrowser() {
      if (playwright == null) {
         IO.println("Initializing Playwright browser...");
         this.playwright = Playwright.create();
         this.browser = playwright.chromium().launch(new BrowserType.LaunchOptions()
            .setHeadless(true)
            .setArgs(Arrays.asList("--no-sandbox", "--disable-setuid-sandbox")));
         IO.println("✓ Browser ready");
      }
   }

   /**
    * Closes browser and cache resources.
    */
   public void close() {
      if (browser != null) {
         browser.close();
         browser = null;
      }
      if (playwright != null) {
         playwright.close();
         playwright = null;
      }
      if (cacheDb != null) {
         cacheDb.close();
      }
   }
   
   /**
    * Discovers all active Dutch auctions by crawling the auctions page.
    *
    * Uses Playwright to render JavaScript-heavy pages and extract auction data.
    * Supports caching to avoid unnecessary page fetches.  Filters auctions whose
    * location contains ", NL" (indicating the Netherlands).  Each auction link
    * contains a unique sale ID in the format A1-xxxxx or A7-xxxxx.
    *
    * Auctions are saved to the database and can be retrieved with getDutchAuctions().
    *
    * @return a list of sale identifiers for auctions located in NL (legacy compatibility)
    */
   public List<Integer> discoverDutchAuctions() {
      Set<Integer> saleIds = new HashSet<>();

      // Check if browser is initialized
      if (browser == null) {
         initializeBrowser();
      }
      
      var pageNumber   = 1;
      var hasMorePages = true;

      IO.println("Starting Dutch auction discovery from " + AUCTIONS_PAGE);

      while (hasMorePages) {
         IO.println("\n[Page " + pageNumber + "] Fetching auctions...");

         // Check cache first
         var html = loadFromCache(pageNumber);

         if (html != null) {
            IO.println("  ✓ Loaded from cache");
         } else {
            // Fetch with Playwright
            html = fetchPageWithPlaywright(pageNumber);

            if (html == null || html.isEmpty()) {
               IO.println("  ⚠️  Failed to fetch page, stopping pagination");
               break;
            }

            IO.println("  ✓ Fetched from website");

            // Save to cache
            if (useCache) {
               saveToCache(pageNumber, html);
            }

            // Rate limiting
            try {
               Thread.sleep(RATE_LIMIT_MS);
            } catch (InterruptedException e) {
               Thread.currentThread().interrupt();
               break;
            }
         }

         // Parse auctions from HTML (saves Dutch auctions to database)
         var foundOnPage = parseAuctionsFromHtml(html, saleIds);

         if (foundOnPage == 0) {
            IO.println("  ⚠️  No Dutch auctions found on page, stopping pagination");
            hasMorePages = false;
         } else {
            IO.println("  ✓ Found " + foundOnPage + " Dutch auctions");
            pageNumber++;
         }
      }

      IO.println("\n✓ Total Dutch auctions discovered: " + saleIds.size());
      return new ArrayList<>(saleIds);
   }


   /**
    * Fetches a single page using Playwright
    */
   private String fetchPageWithPlaywright(int pageNumber) {
      var url = pageNumber == 1
         ? AUCTIONS_PAGE
         : AUCTIONS_PAGE + "?page=" + pageNumber;

      try {
         var page = browser.newPage();

         // Set user agent
         page.setExtraHTTPHeaders(Map.of(
            "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
         ));

         // Navigate to page
         page.navigate(url, new Page.NavigateOptions()
            .setTimeout(30000)
            .setWaitUntil(WaitUntilState.NETWORKIDLE));

         // Wait for auction listings to appear
         try {
            page.waitForSelector("a[href^='/a/']", new Page.WaitForSelectorOptions()
               .setTimeout(10000));
         } catch (Exception e) {
            // Continue even if selector not found
            IO.println("  ⚠️  Auction selector not found");
         }

         // Get HTML content
         var html = page.content();
         page.close();

         return html;

      } catch (Exception e) {
         System.err.println("  ⚠️  Playwright error: " + e.getMessage());
         return null;
      }
   }

   /**
    * Parses auctions from HTML using JSoup and saves Dutch auctions to database.
    * Uses proper HTML parsing instead of regex for more reliable extraction.
    * @return number of Dutch auctions found on this page
    */
   private int parseAuctionsFromHtml(String html, Set<Integer> saleIds) {
      var foundCount = 0;

      try {
         var doc = org.jsoup.Jsoup.parse(html);

         // Find all auction links (format: /a/title-A1-12345 or /a/title-A7-12345)
         var auctionLinks = doc.select("a[href^='/a/']");

         for (var link : auctionLinks) {
            var href = link.attr("href");

            // Extract auction ID from URL
            var pattern = java.util.regex.Pattern.compile("/a/.*?-A([17])-(\\d+)");
            var matcher = pattern.matcher(href);

            if (!matcher.find()) {
               continue;
            }
            
            var typeNum   = matcher.group(1);
            var auctionId = Integer.parseInt(matcher.group(2));

            // Skip duplicates
            if (saleIds.contains(auctionId)) {
               continue;
            }

            // Extract auction info using JSoup
            var auction = extractAuctionInfo(link, href, auctionId, "A" + typeNum);

            // Only keep Dutch auctions
            if (auction != null && "NL".equals(auction.country)) {
               saleIds.add(auctionId);
               foundCount++;

               // Save to database
               try {
                  db.upsertAuction(auction);
                  IO.println("    Found Dutch auction: " + auctionId + " - " + auction.title + " (" + auction.location + ")");
               } catch (SQLException e) {
                  System.err.println("    Failed to save auction: " + e.getMessage());
               }
            }
         }
      } catch (Exception e) {
         System.err.println("    Error parsing HTML: " + e.getMessage());
      }

      return foundCount;
   }

   /**
    * Extracts auction information from a link element using JSoup
    * This method intelligently parses the HTML structure to extract:
    * - Title
    * - Location (city and country)
    * - Lot count (if available)
    */
   private AuctionInfo extractAuctionInfo(org.jsoup.nodes.Element link, String href, int auctionId, String type) {
      var auction = new AuctionInfo();
      auction.auctionId = auctionId;
      auction.type = type;
      auction.url = "https://www.troostwijkauctions.com" + href;

      // Extract title from href (convert kebab-case to title)
      var titlePattern = java.util.regex.Pattern.compile("/a/(.+?)-A[17]-");
      var titleMatcher = titlePattern.matcher(href);
      if (titleMatcher.find()) {
         var slug = titleMatcher.group(1);
         auction.title = slug.replace("-", " ");
         // Capitalize first letter
         if (!auction.title.isEmpty()) {
            auction.title = auction.title.substring(0, 1).toUpperCase() + auction.title.substring(1);
         }
      } else {
         auction.title = "Unknown Auction";
      }

      // Try to find title in link text (more accurate)
      var linkText = link.text();
      if (!linkText.isEmpty() && !linkText.matches(".*\\d+.*")) {
         // If link text doesn't contain numbers, it's likely the title
         var parts = linkText.split(",|\\d+");
         if (parts.length > 0 && parts[0].trim().length() > 5) {
            auction.title = parts[0].trim();
         }
      }

      // Extract location using JSoup selectors
      // Look for <p> tags that contain location info
      var locationElements = link.select("p");
      for (var p : locationElements) {
         var text = p.text();

         // Pattern: "City, Country" or "City, Region, Country"
         if (text.matches(".*[A-Z]{2}$")) {
            // Ends with 2-letter country code
            var countryCode = text.substring(text.length() - 2);
            var cityPart    = text.substring(0, text.length() - 2).trim();

            // Remove trailing comma or whitespace
            cityPart = cityPart.replaceAll("[,\\s]+$", "");

            auction.country = countryCode;
            auction.city = cityPart;
            auction.location = cityPart + ", " + countryCode;
            break;
         }
      }

      // Fallback: check HTML content directly
      if (auction.country == null) {
         var html = link.html();
         var locPattern = java.util.regex.Pattern.compile(
            "([A-Za-z][A-Za-z\\s,\\-']+?)\\s*(?:<!--.*?-->)?\\s*</span>\\s*([A-Z]{2})(?![A-Za-z])");
         var locMatcher = locPattern.matcher(html);

         if (locMatcher.find()) {
            var city    = locMatcher.group(1).trim().replaceAll(",$", "");
            var country = locMatcher.group(2);
            auction.city = city;
            auction.country = country;
            auction.location = city + ", " + country;
         }
      }

      // Extract lot count if available (kavels/lots)
      var textElements = link.select("*");
      for (var elem : textElements) {
         var text = elem.ownText();
         if (text.matches("\\d+\\s+(?:kavel|lot|item)s?.*")) {
            var countPattern = java.util.regex.Pattern.compile("(\\d+)");
            var countMatcher = countPattern.matcher(text);
            if (countMatcher.find()) {
               auction.lotCount = Integer.parseInt(countMatcher.group(1));
               break;
            }
         }
      }

      return auction;
   }

   /**
    * Loads cached HTML for a page
    */
   private String loadFromCache(int pageNumber) {
      if (!useCache || cacheDb == null) return null;
      
      var url = pageNumber == 1
         ? AUCTIONS_PAGE
         : AUCTIONS_PAGE + "?page=" + pageNumber;

      return cacheDb.get(url);
   }

   /**
    * Saves HTML to cache
    */
   private void saveToCache(int pageNumber, String html) {
      if (!useCache || cacheDb == null) return;
      
      var url = pageNumber == 1
         ? AUCTIONS_PAGE
         : AUCTIONS_PAGE + "?page=" + pageNumber;

      cacheDb.put(url, html, CACHE_EXPIRATION_HOURS);
   }
   
   /**
    * Retrieves all lots for a given sale ID using Troostwijk's internal JSON
    * API.  The API accepts parameters such as batchSize, offset, and saleID.
    * A large batchSize returns many lots at once.  We loop until no further
    * results are returned.  Each JSON result is mapped to our Lot domain
    * object and persisted to the database.
    *
    * @param saleId the sale identifier
    */
   public void fetchLotsForSale(int saleId) {
      var batchSize = 200;
      var offset    = 0;
      var more      = true;
      var totalLots = 0;
      
      while (more) {
         try {
            var url = LOT_API + "?batchSize=" + batchSize
                      + "&listType=7&offset=" + offset
                      + "&sortOption=0&saleID=" + saleId
                      + "&parentID=0&relationID=0&buildversion=201807311";
            
            IO.println("    Fetching lots from API (offset=" + offset + ")...");
            
            var request = HttpRequest.newBuilder()
                                     .uri(URI.create(url))
                                     .header("Accept", "application/json")
                                     .header("User-Agent", "Mozilla/5.0")
                                     .GET()
                                     .build();
            
            var response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
            
            if (response.statusCode() != 200) {
               System.err.println("    ⚠️  API call failed for sale " + saleId);
               System.err.println("    Status: " + response.statusCode());
               System.err.println("    Response: " + response.body().substring(0, Math.min(200, response.body().length())));
               break;
            }
            
            var root    = objectMapper.readTree(response.body());
            var results = root.path("results");
            
            if (!results.isArray() || results.isEmpty()) {
               if (offset == 0) {
                  IO.println("    ⚠️  No lots found for sale " + saleId);
                  IO.println("    API Response: " + response.body().substring(0, Math.min(500, response.body().length())));
               }
               more = false;
               break;
            }
            var lotsInBatch = results.size();
            IO.println("    Found " + lotsInBatch + " lots in this batch");
            
            for (var node : results) {
               var lot = new Lot();
               lot.saleId       = saleId;
               lot.lotId        = node.path("lotID").asInt();
               lot.title        = node.path("t").asText();
               lot.description  = node.path("d").asText();
               lot.manufacturer = node.path("mf").asText();
               lot.type         = node.path("typ").asText();
               lot.year         = node.path("yb").asInt();
               lot.category     = node.path("lc").asText();
               // Current bid; field names may differ (e.g. currentBid or cb)
               lot.currentBid = node.path("cb").asDouble();
               lot.currency   = node.path("cu").asText();
               lot.url        = "https://www.troostwijkauctions.com/nl" + node.path("url").asText();
               
               // Save basic lot info into DB
               db.upsertLot(lot);
               totalLots++;
               
               // Download images and perform object detection
               List<String> imageUrls = new ArrayList<>();
               var          imgs      = node.path("imgs");
               if (imgs.isArray()) {
                  for (var imgNode : imgs) {
                     var imgUrl = imgNode.asText();
                     imageUrls.add(imgUrl);
                  }
               }
               
               // Download and analyze images (optional, can be slow)
               for (var imgUrl : imageUrls) {
                  var fileName = downloadImage(imgUrl, saleId, lot.lotId);
                  if (fileName != null) {
                     // run object detection once per image
                     var labels = detector.detectObjects(fileName);
                     db.insertImage(lot.lotId, imgUrl, fileName, labels);
                  }
               }
            }
            
            IO.println("    ✓ Processed " + totalLots + " lots so far");
            offset += batchSize;
         } catch (IOException | InterruptedException e) {
            System.err.println("Error fetching lots for sale " + saleId + ": " + e.getMessage());
            more = false;
         } catch (SQLException e) {
            System.err.println("Database error: " + e.getMessage());
         }
      }
   }
   
   /**
    * Downloads an image from the given URL to a local directory.  Images
    * are stored under "images/<saleId>/<lotId>/" to keep them organised.
    *
    * @param imageUrl remote image URL
    * @param saleId   sale identifier
    * @param lotId    lot identifier
    * @return absolute path to saved file or null on failure
    */
   private String downloadImage(String imageUrl, int saleId, int lotId) {
      try {
         var request = HttpRequest.newBuilder()
                                  .uri(URI.create(imageUrl))
                                  .GET()
                                  .build();
         var response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
         if (response.statusCode() == 200) {
            var dir = Paths.get("images", String.valueOf(saleId), String.valueOf(lotId));
            Files.createDirectories(dir);
            var fileName = Paths.get(imageUrl).getFileName().toString();
            var dest     = dir.resolve(fileName);
            Files.copy(response.body(), dest);
            return dest.toAbsolutePath().toString();
         }
      } catch (IOException | InterruptedException e) {
         System.err.println("Failed to download image " + imageUrl + ": " + e.getMessage());
      }
      return null;
   }
   
   /**
    * Schedules periodic monitoring of all lots.  The scheduler runs every
    * hour to refresh current bids and closing times.  For lots that
    * are within 30 minutes of closing, it increases the polling frequency
    * automatically.  When a new bid is detected or a lot is about to
    * expire, a Pushover notification is sent to the configured user.
    * Note: In production, ensure proper shutdown handling for the scheduler.
    */
   public void scheduleMonitoring() {
      var scheduler = Executors.newScheduledThreadPool(1);
      scheduler.scheduleAtFixedRate(() -> {
         try {
            var activeLots = db.getActiveLots();
            for (var lot : activeLots) {
               // refresh the lot's bidding information via API
               refreshLotBid(lot);
               // check closing time to adjust monitoring
               var minutesLeft = lot.minutesUntilClose();
               if (minutesLeft < 30) {
                  // send warning when within 5 minutes
                  if (minutesLeft <= 5 && !lot.closingNotified) {
                     notifier.sendNotification("Kavel " + lot.lotId + " sluit binnen " + minutesLeft + " min.",
                                               "Lot nearing closure", 1);
                     lot.closingNotified = true;
                     db.updateLotNotificationFlags(lot);
                  }
                  // schedule additional quick check for this lot
                  scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
               }
            }
         } catch (SQLException e) {
            System.err.println("Error during scheduled monitoring: " + e.getMessage());
         }
      }, 0, 1, TimeUnit.HOURS);
   }
   
   /**
    * Refreshes the bid for a single lot and sends notification if it has
    * changed since the last check.  The method calls the same API used for
    * initial scraping but only extracts the current bid for the given lot.
    *
    * @param lot the lot to refresh
    */
   private void refreshLotBid(Lot lot) {
      try {
         var url = LOT_API + "?batchSize=1&listType=7&offset=0&sortOption=0&saleID=" + lot.saleId
                   + "&parentID=0&relationID=0&buildversion=201807311&lotID=" + lot.lotId;
         var request  = HttpRequest.newBuilder().uri(URI.create(url)).GET().build();
         var response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
         if (response.statusCode() != 200) return;
         var root    = objectMapper.readTree(response.body());
         var results = root.path("results");
         if (results.isArray() && !results.isEmpty()) {
            var node   = results.get(0);
            var newBid = node.path("cb").asDouble();
            if (Double.compare(newBid, lot.currentBid) > 0) {
               var previous = lot.currentBid;
               lot.currentBid = newBid;
               db.updateLotCurrentBid(lot);
               var msg = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)", lot.lotId, newBid, previous);
               notifier.sendNotification(msg, "Kavel bieding update", 0);
            }
         }
      } catch (IOException | InterruptedException | SQLException e) {
         System.err.println("Failed to refresh bid for lot " + lot.lotId + ": " + e.getMessage());
      }
   }
   
   /**
    * Prints statistics about the data in the database.
    */
   public void printDatabaseStats() {
      try {
         var allLots    = db.getAllLots();
         var imageCount = db.getImageCount();
         
         IO.println("  Total lots in database: " + allLots.size());
         IO.println("  Total images downloaded: " + imageCount);
         
         if (!allLots.isEmpty()) {
            var totalBids = allLots.stream().mapToDouble(l -> l.currentBid).sum();
            IO.println("  Total current bids: €" + String.format("%.2f", totalBids));
         }
      } catch (SQLException e) {
         System.err.println("  ⚠️  Could not retrieve database stats: " + e.getMessage());
      }
   }
   
   // ----------------------------------------------------------------------
   // Domain classes and services
   // ----------------------------------------------------------------------
   
}