From bde45e0dc947f7c0377d667458450d87301d85b6 Mon Sep 17 00:00:00 2001 From: michael1986 Date: Fri, 28 Nov 2025 05:39:23 +0100 Subject: [PATCH] start --- .idea/data_source_mapping.xml | 6 + pom.xml | 7 + .../java/com/auction/TroostwijkScraper.java | 5 +- src/main/resources/test.html | 205 ++++++++++++++++++ .../com/auction/TroostwijkScraperTest.java | 155 +++++++++++++ 5 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 .idea/data_source_mapping.xml create mode 100644 src/main/resources/test.html create mode 100644 src/test/java/com/auction/TroostwijkScraperTest.java diff --git a/.idea/data_source_mapping.xml b/.idea/data_source_mapping.xml new file mode 100644 index 0000000..4e6c19e --- /dev/null +++ b/.idea/data_source_mapping.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 34c722b..372a40b 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,12 @@ slf4j-simple 2.0.9 + + org.junit.jupiter + junit-jupiter + 5.10.1 + test + @@ -100,6 +106,7 @@ 3 + ${project.build.outputDirectory} --enable-native-access=ALL-UNNAMED diff --git a/src/main/java/com/auction/TroostwijkScraper.java b/src/main/java/com/auction/TroostwijkScraper.java index 0456e66..bff5531 100644 --- a/src/main/java/com/auction/TroostwijkScraper.java +++ b/src/main/java/com/auction/TroostwijkScraper.java @@ -92,7 +92,7 @@ public class TroostwijkScraper { // HTTP client used for API calls private final HttpClient httpClient; private final ObjectMapper objectMapper; - private final DatabaseService db; + public final DatabaseService db; private final NotificationService notifier; private final ObjectDetectionService detector; @@ -432,6 +432,7 @@ public class TroostwijkScraper { int currentAuction = 0; for (int saleId : auctions) { currentAuction++; + System.out.println(" [Page " + currentAuction + "] Fetching auctions..."); System.out.println(" [" + currentAuction + "/" + totalAuctions + "] Processing sale " + saleId + "..."); scraper.fetchLotsForSale(saleId); } @@ -502,7 +503,7 @@ public class TroostwijkScraper { * a SQLite database. Uses the Xerial JDBC driver which connects to * SQLite via a URL of the form "jdbc:sqlite:path_to_file"【329850066306528†L40-L63】. */ - static class DatabaseService { + static public class DatabaseService { private final String url; DatabaseService(String dbPath) { diff --git a/src/main/resources/test.html b/src/main/resources/test.html new file mode 100644 index 0000000..ce92f8f --- /dev/null +++ b/src/main/resources/test.html @@ -0,0 +1,205 @@ +Alle veilingen | Troostwijk Auctions
Startpagina

Alle veilingen

522 resultaten

Sorteren op:

  • Sluitingsdatum (eerst)
  • Sluitingsdatum (laatst)
  • Startdatum (eerst)
  • Startdatum (laatst)

Woensdag 3 dec 25

Download nu de Troostwijk Auctions app!

App storeGoogle play
\ No newline at end of file diff --git a/src/test/java/com/auction/TroostwijkScraperTest.java b/src/test/java/com/auction/TroostwijkScraperTest.java new file mode 100644 index 0000000..471f286 --- /dev/null +++ b/src/test/java/com/auction/TroostwijkScraperTest.java @@ -0,0 +1,155 @@ +package com.auction; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opencv.core.Core; + +import java.io.File; +import java.sql.SQLException; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Test case for TroostwijkScraper that verifies auction discovery and data persistence. + * Uses a temporary test database to verify that: + * 1. Dutch auctions are correctly discovered from the live page + * 2. Auction properties (sale IDs, titles, etc.) are valid + * 3. Data is correctly persisted to the database + */ +public class TroostwijkScraperTest { + + private TroostwijkScraper scraper; + private String testDatabasePath; + + @BeforeAll + public static void loadOpenCV() { + // Load native OpenCV library before any tests run + try { + System.loadLibrary(Core.NATIVE_LIBRARY_NAME); + System.out.println("✓ OpenCV native library loaded successfully"); + } catch (UnsatisfiedLinkError e) { + System.err.println("⚠️ Warning: Could not load OpenCV native library"); + System.err.println(" Tests will run without object detection support"); + } + } + + @BeforeEach + public void setUp() throws SQLException, java.io.IOException { + // Create temporary test database + testDatabasePath = "test_auctions_" + System.currentTimeMillis() + ".db"; + + // Initialize scraper with test database (no YOLO models needed for this test) + // Using non-existent paths for YOLO files will disable object detection + scraper = new TroostwijkScraper( + testDatabasePath, + "desktop", + "", + "nonexistent/yolov4.cfg", + "nonexistent/yolov4.weights", + "nonexistent/coco.names" + ); + } + + @AfterEach + public void tearDown() { + // Clean up test database + File dbFile = new File(testDatabasePath); + if (dbFile.exists()) { + dbFile.delete(); + } + } + + @Test + public void testDiscoverDutchAuctions() { + // Discover auctions from the live page + List auctions = scraper.discoverDutchAuctions(); + + // Verify that auctions were found + assertNotNull(auctions, "Auctions list should not be null"); + assertFalse(auctions.isEmpty(), "Should find at least one Dutch auction"); + + // Verify that all sale IDs are positive integers + for (Integer saleId : auctions) { + assertNotNull(saleId, "Sale ID should not be null"); + assertTrue(saleId > 0, "Sale ID should be positive: " + saleId); + } + + // Verify no duplicate sale IDs + long uniqueCount = auctions.stream().distinct().count(); + assertEquals(auctions.size(), uniqueCount, "All sale IDs should be unique"); + + System.out.println("✓ Found " + auctions.size() + " unique Dutch auctions"); + System.out.println("✓ Sale IDs: " + auctions); + } + + @Test + public void testFetchAndPersistAuctionData() throws SQLException { + // First, discover auctions + List auctions = scraper.discoverDutchAuctions(); + assertFalse(auctions.isEmpty(), "Need at least one auction to test"); + + // Take the first auction and fetch its lots + Integer firstSaleId = auctions.get(0); + System.out.println("Testing with sale ID: " + firstSaleId); + + scraper.fetchLotsForSale(firstSaleId); + + // Verify data was persisted to database + List lotsInDb = scraper.db.getAllLots(); + + assertNotNull(lotsInDb, "Lots list should not be null"); + assertFalse(lotsInDb.isEmpty(), "Should have persisted at least one lot"); + + // Verify lot properties + for (TroostwijkScraper.Lot lot : lotsInDb) { + assertEquals(firstSaleId.intValue(), lot.saleId, "Lot should belong to the correct sale"); + assertTrue(lot.lotId > 0, "Lot ID should be positive"); + assertNotNull(lot.title, "Lot title should not be null"); + assertFalse(lot.title.isEmpty(), "Lot title should not be empty"); + assertNotNull(lot.url, "Lot URL should not be null"); + assertTrue(lot.url.startsWith("https://"), "Lot URL should be valid"); + assertTrue(lot.currentBid >= 0, "Current bid should be non-negative"); + } + + System.out.println("✓ Successfully persisted " + lotsInDb.size() + " lots to database"); + System.out.println("✓ All lot properties are valid"); + } + + @Test + public void testDatabaseSchema() throws SQLException { + // Verify that the database schema was created correctly + List lots = scraper.db.getAllLots(); + assertNotNull(lots, "Should be able to query lots table"); + + int imageCount = scraper.db.getImageCount(); + assertTrue(imageCount >= 0, "Image count should be non-negative"); + + List activeLots = scraper.db.getActiveLots(); + assertNotNull(activeLots, "Should be able to query active lots"); + + System.out.println("✓ Database schema is valid and queryable"); + } + + @Test + public void testAuctionProperties() { + List auctions = scraper.discoverDutchAuctions(); + assertFalse(auctions.isEmpty(), "Should find auctions"); + + // Test that we can fetch data for multiple auctions + int auctionsToTest = Math.min(2, auctions.size()); + + for (int i = 0; i < auctionsToTest; i++) { + Integer saleId = auctions.get(i); + System.out.println("Testing auction " + (i + 1) + ": " + saleId); + + // This should not throw an exception + assertDoesNotThrow(() -> scraper.fetchLotsForSale(saleId), + "Should be able to fetch lots for sale " + saleId); + } + + System.out.println("✓ Successfully tested " + auctionsToTest + " auctions"); + } +}