From b1a13e97471ed9540152d1e2678af2ff59ef380a Mon Sep 17 00:00:00 2001 From: Tour Date: Tue, 9 Dec 2025 12:32:30 +0100 Subject: [PATCH] Initial-Commit --- .aiassistant/rules/rules.md | 4 + .aiignore | 18 + .dockerignore | 27 + .env | 1 + .github/workflows/__deploy.old | 42 + .github/workflows/_oldbuild.nothing | 50 + .gitignore | 35 + Dockerfile | 41 + README.md | 563 ++++++ docker-compose.yml | 56 + docs/ARCHITECTURE-TROOSTWIJK-SCRAPER.md | 326 ++++ docs/DATABASE_ARCHITECTURE.md | 258 +++ docs/DATA_SYNC_SETUP.md | 109 ++ docs/EMAIL_CONFIGURATION.md | 226 +++ docs/EXPERT_ANALITICS.sql | 153 ++ docs/EXPERT_ANALITICS_PRIORITY.md | 38 + docs/GraphQL.md | 126 ++ docs/INTEGRATION_FLOWCHART.md | 393 ++++ docs/QUARKUS_GUIDE.md | 650 +++++++ docs/RATE_LIMITING.md | 209 +++ docs/VALUATION.md | 304 ++++ nginx.conf | 22 + pom.xml | 465 +++++ src/main/java/auctiora/AuctionInfo.java | 19 + .../auctiora/AuctionMonitorHealthCheck.java | 82 + .../java/auctiora/AuctionMonitorProducer.java | 61 + .../java/auctiora/AuctionMonitorResource.java | 869 +++++++++ src/main/java/auctiora/BidHistory.java | 16 + src/main/java/auctiora/DatabaseService.java | 270 +++ .../java/auctiora/ImageProcessingService.java | 55 + src/main/java/auctiora/Lot.java | 153 ++ .../java/auctiora/LotEnrichmentScheduler.java | 81 + .../java/auctiora/LotEnrichmentService.java | 201 ++ src/main/java/auctiora/LotIntelligence.java | 33 + .../java/auctiora/NotificationService.java | 292 +++ .../java/auctiora/ObjectDetectionService.java | 244 +++ .../auctiora/QuarkusWorkflowScheduler.java | 309 ++++ .../java/auctiora/RateLimitedHttpClient.java | 246 +++ .../java/auctiora/ScraperDataAdapter.java | 196 ++ src/main/java/auctiora/StatusResource.java | 74 + .../auctiora/TroostwijkGraphQLClient.java | 378 ++++ src/main/java/auctiora/TroostwijkMonitor.java | 132 ++ .../auctiora/ValuationAnalyticsResource.java | 378 ++++ .../java/auctiora/WorkflowOrchestrator.java | 433 +++++ .../java/auctiora/db/AuctionRepository.java | 152 ++ src/main/java/auctiora/db/DatabaseSchema.java | 154 ++ .../java/auctiora/db/ImageRepository.java | 137 ++ src/main/java/auctiora/db/LotRepository.java | 275 +++ .../resources/META-INF/resources/favicon.ico | Bin 0 -> 3138 bytes .../resources/META-INF/resources/favicon.svg | 7 + .../resources/META-INF/resources/index.html | 1611 +++++++++++++++++ .../resources/META-INF/resources/script.js | 0 .../resources/META-INF/resources/status.html | 224 +++ .../resources/META-INF/resources/style.css | 0 .../resources/valuation-analytics.html | 1086 +++++++++++ src/main/resources/application.properties | 75 + src/main/resources/simplelogger.properties | 20 + .../auctiora/ClosingTimeCalculationTest.java | 138 ++ .../java/auctiora/DatabaseServiceTest.java | 390 ++++ .../auctiora/ImageProcessingServiceTest.java | 186 ++ src/test/java/auctiora/IntegrationTest.java | 461 +++++ .../auctiora/NotificationServiceTest.java | 243 +++ .../auctiora/ObjectDetectionServiceTest.java | 185 ++ src/test/java/auctiora/ParserTest.java | 62 + .../java/auctiora/ScraperDataAdapterTest.java | 275 +++ .../java/auctiora/TroostwijkMonitorTest.java | 380 ++++ src/test/resources/application.properties | 65 + workflows/maven.yml | 20 + 68 files changed, 14784 insertions(+) create mode 100644 .aiassistant/rules/rules.md create mode 100644 .aiignore create mode 100644 .dockerignore create mode 100644 .env create mode 100644 .github/workflows/__deploy.old create mode 100644 .github/workflows/_oldbuild.nothing create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 docs/ARCHITECTURE-TROOSTWIJK-SCRAPER.md create mode 100644 docs/DATABASE_ARCHITECTURE.md create mode 100644 docs/DATA_SYNC_SETUP.md create mode 100644 docs/EMAIL_CONFIGURATION.md create mode 100644 docs/EXPERT_ANALITICS.sql create mode 100644 docs/EXPERT_ANALITICS_PRIORITY.md create mode 100644 docs/GraphQL.md create mode 100644 docs/INTEGRATION_FLOWCHART.md create mode 100644 docs/QUARKUS_GUIDE.md create mode 100644 docs/RATE_LIMITING.md create mode 100644 docs/VALUATION.md create mode 100644 nginx.conf create mode 100644 pom.xml create mode 100644 src/main/java/auctiora/AuctionInfo.java create mode 100644 src/main/java/auctiora/AuctionMonitorHealthCheck.java create mode 100644 src/main/java/auctiora/AuctionMonitorProducer.java create mode 100644 src/main/java/auctiora/AuctionMonitorResource.java create mode 100644 src/main/java/auctiora/BidHistory.java create mode 100644 src/main/java/auctiora/DatabaseService.java create mode 100644 src/main/java/auctiora/ImageProcessingService.java create mode 100644 src/main/java/auctiora/Lot.java create mode 100644 src/main/java/auctiora/LotEnrichmentScheduler.java create mode 100644 src/main/java/auctiora/LotEnrichmentService.java create mode 100644 src/main/java/auctiora/LotIntelligence.java create mode 100644 src/main/java/auctiora/NotificationService.java create mode 100644 src/main/java/auctiora/ObjectDetectionService.java create mode 100644 src/main/java/auctiora/QuarkusWorkflowScheduler.java create mode 100644 src/main/java/auctiora/RateLimitedHttpClient.java create mode 100644 src/main/java/auctiora/ScraperDataAdapter.java create mode 100644 src/main/java/auctiora/StatusResource.java create mode 100644 src/main/java/auctiora/TroostwijkGraphQLClient.java create mode 100644 src/main/java/auctiora/TroostwijkMonitor.java create mode 100644 src/main/java/auctiora/ValuationAnalyticsResource.java create mode 100644 src/main/java/auctiora/WorkflowOrchestrator.java create mode 100644 src/main/java/auctiora/db/AuctionRepository.java create mode 100644 src/main/java/auctiora/db/DatabaseSchema.java create mode 100644 src/main/java/auctiora/db/ImageRepository.java create mode 100644 src/main/java/auctiora/db/LotRepository.java create mode 100644 src/main/resources/META-INF/resources/favicon.ico create mode 100644 src/main/resources/META-INF/resources/favicon.svg create mode 100644 src/main/resources/META-INF/resources/index.html create mode 100644 src/main/resources/META-INF/resources/script.js create mode 100644 src/main/resources/META-INF/resources/status.html create mode 100644 src/main/resources/META-INF/resources/style.css create mode 100644 src/main/resources/META-INF/resources/valuation-analytics.html create mode 100644 src/main/resources/application.properties create mode 100644 src/main/resources/simplelogger.properties create mode 100644 src/test/java/auctiora/ClosingTimeCalculationTest.java create mode 100644 src/test/java/auctiora/DatabaseServiceTest.java create mode 100644 src/test/java/auctiora/ImageProcessingServiceTest.java create mode 100644 src/test/java/auctiora/IntegrationTest.java create mode 100644 src/test/java/auctiora/NotificationServiceTest.java create mode 100644 src/test/java/auctiora/ObjectDetectionServiceTest.java create mode 100644 src/test/java/auctiora/ParserTest.java create mode 100644 src/test/java/auctiora/ScraperDataAdapterTest.java create mode 100644 src/test/java/auctiora/TroostwijkMonitorTest.java create mode 100644 src/test/resources/application.properties create mode 100644 workflows/maven.yml diff --git a/.aiassistant/rules/rules.md b/.aiassistant/rules/rules.md new file mode 100644 index 0000000..523e7e2 --- /dev/null +++ b/.aiassistant/rules/rules.md @@ -0,0 +1,4 @@ +--- +apply: always +--- + diff --git a/.aiignore b/.aiignore new file mode 100644 index 0000000..3236782 --- /dev/null +++ b/.aiignore @@ -0,0 +1,18 @@ +# An .aiignore file follows the same syntax as a .gitignore file. +# .gitignore documentation: https://git-scm.com/docs/gitignore + +# you can ignore files +.DS_Store +*.log +*.tmp + +# or folders +dist/ +build/ +out/ +.idea +node_modules/ +.vscode/ +.git +.github +scripts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0ad7b8c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,27 @@ +# Exclude large model files from Docker build +models/ +*.weights + +# Exclude build artifacts +target/ +build/ +*.class +*.jar + +# Exclude version control +.git/ +.gitignore + +# Exclude IDE files +.idea/ +*.iml +.vscode/ + +# Exclude data files +images/ +*.db +*.db-journal + +# Exclude docs +README.md +*.md diff --git a/.env b/.env new file mode 100644 index 0000000..1f6cd0f --- /dev/null +++ b/.env @@ -0,0 +1 @@ +GMAIL_APP_PASSWORD=agrepolhlnvhipkv \ No newline at end of file diff --git a/.github/workflows/__deploy.old b/.github/workflows/__deploy.old new file mode 100644 index 0000000..861bf2a --- /dev/null +++ b/.github/workflows/__deploy.old @@ -0,0 +1,42 @@ +name: Build and Deploy + +on: + push: + branches: ["main"] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + + - name: Build + run: mvn -B clean package + + - name: Upload to JFrog + run: | + curl -u "${{ secrets.JFROG_USER }}:${{ secrets.JFROG_PASS }}" \ + -T target/*.jar \ + "http://JFROG-SERVER/artifactory/myrepo/app-latest.jar" + + deploy: + needs: build + runs-on: ubuntu-latest + + steps: + - name: Trigger remote deploy script + uses: appleboy/ssh-action@v0.1.7 + with: + host: ${{ secrets.SERVER_IP }} + username: ${{ secrets.SERVER_USER }} + key: ${{ secrets.SERVER_SSH_KEY }} + script: | + /opt/myapp/update.sh diff --git a/.github/workflows/_oldbuild.nothing b/.github/workflows/_oldbuild.nothing new file mode 100644 index 0000000..7932d23 --- /dev/null +++ b/.github/workflows/_oldbuild.nothing @@ -0,0 +1,50 @@ +name: Build and Deploy Auction App + +on: + push: + branches: + - main + +jobs: + build_and_deploy: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Maven + run: | + apt update + apt install -y maven + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + + - name: Build with Maven + run: mvn -B clean package + + - name: Copy jar to server (no tar) + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.SERVER_IP }} + username: ${{ secrets.SERVER_USER }} + key: ${{ secrets.SERVER_SSH_KEY }} + source: "target/*.jar" + target: "/opt/auction/" + overwrite: true + strip_components: 1 # Strips the 'target/' directory + timeout: 60s + + - name: Restart service + uses: appleboy/ssh-action@v0.1.7 + with: + host: ${{ secrets.SERVER_IP }} + username: ${{ secrets.SERVER_USER }} + key: ${{ secrets.SERVER_SSH_KEY }} + script: | + systemctl restart auction + echo "Deploy complete" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a4011b8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,35 @@ +### IntelliJ IDEA ### +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ +.kotlin + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### MacOS ### +.DS_Store + +NUL +target/ +build/ +.idea/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ca64df2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +# Stage 1: Build +FROM maven:3.9-eclipse-temurin-25-alpine AS builder +WORKDIR /app +# Copy POM first (allows for cached dependency layer) +COPY pom.xml . +RUN mvn dependency:resolve -B + +COPY src ./src +# Updated with both properties to avoid the warning +RUN mvn package -DskipTests -Dquarkus.package.jar.type=uber-jar -Dquarkus.package.jar.enabled=true + +# Stage 2: Runtime (DEBIAN-based for OpenCV native libs) +FROM eclipse-temurin:25-jre +WORKDIR /app + +# Install dependencies + wget for health checks +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + libstdc++6 \ + libgomp1 \ + wget \ + && rm -rf /var/lib/apt/lists/* + +# Create user (Debian syntax) +RUN groupadd -r quarkus && useradd -r -g quarkus quarkus +# Create user with explicit UID 1000 (Debian syntax) +# RUN groupadd -r -g 1000 quarkus && useradd -r -u 1000 -g quarkus quarkus +# Create the data directory and set ownership BEFORE switching user +# RUN mkdir -p /mnt/okcomputer/output && chown -R quarkus:quarkus /mnt/okcomputer/output + +# Copy the built jar with correct pattern +COPY --from=builder --chown=quarkus:quarkus /app/target/auctiora-*.jar app.jar + +USER quarkus +EXPOSE 8081 + +ENTRYPOINT ["java", \ + "-Dio.netty.tryReflectionSetAccessible=true", \ + "--enable-native-access=ALL-UNNAMED", \ + "--add-opens", "java.base/java.nio=ALL-UNNAMED", \ + "-jar", "app.jar"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..b2300d7 --- /dev/null +++ b/README.md @@ -0,0 +1,563 @@ +# Troostwijk Auction Scraper + +A Java-based web scraper for Dutch auctions on Troostwijk Auctions with **100% free** desktop/email notifications, SQLite persistence, and AI-powered object detection. + +## Features + +- **Auction Discovery**: Automatically discovers active Dutch auctions +- **Data Scraping**: Fetches detailed lot information via Troostwijk's JSON API +- **SQLite Storage**: Persists auction data, lots, images, and detected objects +- **Image Processing**: Downloads and analyzes lot images using OpenCV YOLO object detection +- **Free Notifications**: Real-time notifications when: + - Bids change on monitored lots + - Auctions are closing soon (within 5 minutes) + - Via desktop notifications (Windows/macOS/Linux system tray) βœ… + - Optionally via email (Gmail SMTP - free) βœ… + +## Dependencies + +All dependencies are managed via Maven (see `pom.xml`): + +- **jsoup 1.17.2** - HTML parsing and HTTP client +- **Jackson 2.17.0** - JSON processing +- **SQLite JDBC 3.45.1.0** - Database operations +- **JavaMail 1.6.2** - Email notifications (free) +- **OpenCV 4.9.0** - Image processing and object detection + +## Quick Start + +### Development: Sync Production Data + +To work with real production data locally: + +```powershell +# Linux/Mac (Bash) +./scripts/sync-production-data.sh --db-only +``` + +See [scripts/README.md](scripts/README.md) for full documentation. + +## Setup + +### 1. Notification Options (Choose One) + +#### Option A: Desktop Notifications Only ⭐ (Recommended - Zero Setup) + +Desktop notifications work out of the box on: +- **Windows**: System tray notifications +- **macOS**: Notification Center +- **Linux**: Desktop environment notifications (GNOME, KDE, etc.) + +**No configuration required!** Just run with default settings: +```bash +export NOTIFICATION_CONFIG="desktop" +# Or simply don't set it - desktop is the default +``` + +#### Option B: Desktop + Email Notifications πŸ“§ (Free Gmail) + +1. Enable 2-Factor Authentication in your Google Account +2. Go to: **Google Account β†’ Security β†’ 2-Step Verification β†’ App passwords** +3. Generate an app password for "Mail" +4. Set environment variable: + ```bash + export NOTIFICATION_CONFIG="smtp:your.email@gmail.com:your_app_password:recipient@example.com" + ``` + +**Format**: `smtp:username:app_password:recipient_email` + +**Example**: +```bash +export NOTIFICATION_CONFIG="smtp:john.doe@gmail.com:abcd1234efgh5678:john.doe@gmail.com" +``` + +**Note**: This is completely free using Gmail's SMTP server. No paid services required! + +### 2. OpenCV Native Libraries + +Download and install OpenCV native libraries for your platform: + +**Windows:** +```bash +# Download from https://opencv.org/releases/ +# Extract and add to PATH or use: +java -Djava.library.path="C:\opencv\build\java\x64" -jar scraper.jar +``` + +**Linux:** +```bash +sudo apt-get install libopencv-dev +``` + +**macOS:** +```bash +brew install opencv +``` + +### 3. YOLO Model Files + +Download YOLO model files for object detection: + +```bash +mkdir /mnt/okcomputer/output/models +cd /mnt/okcomputer/output/models + +# Download YOLOv4 config +wget https://raw.githubusercontent.com/AlexeyAB/darknet/master/cfg/yolov4.cfg + +# Download YOLOv4 weights (245 MB) +wget https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v3_optimal/yolov4.weights + +# Download COCO class names +wget https://raw.githubusercontent.com/AlexeyAB/darknet/master/data/coco.names +``` + +## Building + +```bash +mvn clean package +``` + +This creates: +- `../build/auctiora/auctiora-1.0-SNAPSHOT.jar` - Regular JAR +- `../build/auctiora/auctiora-1.0-SNAPSHOT-jar-with-dependencies.jar` - Executable JAR with all dependencies + +## Running + +### Quick Start (Desktop Notifications Only) + +```bash +java -Djava.library.path="/path/to/opencv/lib" \ + -jar ../build/auctiora/troostwijk-scraper-1.0-SNAPSHOT-jar-with-dependencies.jar +``` + +### With Email Notifications + +```bash +export NOTIFICATION_CONFIG="smtp:your@gmail.com:app_password:your@gmail.com" + +java -Djava.library.path="/path/to/opencv/lib" \ + -jar ../build/auctiora/troostwijk-scraper-1.0-SNAPSHOT-jar-with-dependencies.jar +``` + +### Using Maven + +```bash +mvn exec:java -Dexec.mainClass="com.auction.scraper.TroostwijkScraper" +``` + +## System Architecture & Integration Flow + +> **πŸ“Š Complete Integration Flowchart**: See [docs/INTEGRATION_FLOWCHART.md](docs/INTEGRATION_FLOWCHART.md) for the detailed intelligence integration diagram with GraphQL API fields, analytics, and dashboard features. + +### Quick Overview + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ COMPLETE SYSTEM INTEGRATION DIAGRAM β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ PHASE 1: EXTERNAL SCRAPER (Python/Playwright) - ARCHITECTURE-TROOSTWIJK β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β–Ό β–Ό β–Ό + [Listing Pages] [Auction Pages] [Lot Pages] + /auctions?page=N /a/auction-id /l/lot-id + β”‚ β”‚ β”‚ + β”‚ Extract URLs β”‚ Parse __NEXT_DATA__ β”‚ Parse __NEXT_DATA__ + β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Άβ”‚ JSON β”‚ JSON + β”‚ β”‚ β”‚ + β”‚ β–Ό β–Ό + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ β”‚ INSERT auctionsβ”‚ β”‚ INSERT lots β”‚ + β”‚ β”‚ to SQLite β”‚ β”‚ INSERT images β”‚ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ (URLs only) β”‚ + β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ SQLITE DATABASE β”‚ + β”‚ output/cache.db β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β–Ό β–Ό β–Ό + [auctions table] [lots table] [images table] + - auction_id - lot_id - id + - title - auction_id - lot_id + - location - title - url + - lots_count - current_bid - local_path + - closing_time - bid_count - downloaded=0 + - closing_time + β”‚ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ PHASE 2: MONITORING & PROCESSING (Java) - THIS PROJECT β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β–Ό β–Ό β–Ό + [TroostwijkMonitor] [DatabaseService] [ScraperDataAdapter] + β”‚ β”‚ β”‚ + β”‚ Read lots β”‚ Query lots β”‚ Transform data + β”‚ every hour β”‚ Import images β”‚ TEXT β†’ INTEGER + β”‚ β”‚ β”‚ "€123" β†’ 123.0 + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β–Ό β–Ό β–Ό + [Bid Monitoring] [Image Processing] [Closing Alerts] + Check API every 1h Download images Check < 5 min + β”‚ β”‚ β”‚ + β”‚ New bid? β”‚ Process via β”‚ Time critical? + β”œβ”€[YES]──────────┐ β”‚ ObjectDetection β”œβ”€[YES]────┐ + β”‚ β”‚ β”‚ β”‚ β”‚ + β–Ό β”‚ β–Ό β”‚ β”‚ + [Update current_bid] β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ + in database β”‚ β”‚ YOLO Detection β”‚ β”‚ β”‚ + β”‚ β”‚ OpenCV DNN β”‚ β”‚ β”‚ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ + β”‚ β”‚ β”‚ β”‚ + β”‚ β”‚ Detect objects β”‚ β”‚ + β”‚ β”œβ”€[vehicle] β”‚ β”‚ + β”‚ β”œβ”€[furniture] β”‚ β”‚ + β”‚ β”œβ”€[machinery] β”‚ β”‚ + β”‚ β”‚ β”‚ β”‚ + β”‚ β–Ό β”‚ β”‚ + β”‚ [Save labels to DB] β”‚ β”‚ + β”‚ [Estimate value] β”‚ β”‚ + β”‚ β”‚ β”‚ β”‚ + β”‚ β”‚ β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ PHASE 3: NOTIFICATION SYSTEM - USER INTERACTION TRIGGERS β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β–Ό β–Ό + [NotificationService] [User Decision Points] + β”‚ β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ + β–Ό β–Ό β–Ό β”‚ + [Desktop Notify] [Email Notify] [Priority Level] β”‚ + Windows/macOS/ Gmail SMTP 0=Normal β”‚ + Linux system (FREE) 1=High β”‚ + tray β”‚ + β”‚ β”‚ β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β”‚ β”‚ + β–Ό β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ USER INTERACTION β”‚ β”‚ TRIGGER EVENTS: β”‚ + β”‚ NOTIFICATIONS β”‚ β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ + β–Ό β–Ό β–Ό β”‚ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ 1. BID CHANGE β”‚ β”‚ 2. OBJECT β”‚ β”‚ 3. CLOSING β”‚ β”‚ +β”‚ β”‚ β”‚ DETECTED β”‚ β”‚ ALERT β”‚ β”‚ +β”‚ "Nieuw bod op β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ +β”‚ kavel 12345: β”‚ β”‚ "Lot contains: β”‚ β”‚ "Kavel 12345 β”‚ β”‚ +β”‚ €150 (was €125)"β”‚ β”‚ - Vehicle β”‚ β”‚ sluit binnen β”‚ β”‚ +β”‚ β”‚ β”‚ - Machinery β”‚ β”‚ 5 min." β”‚ β”‚ +β”‚ Priority: NORMAL β”‚ β”‚ Est: €5000" β”‚ β”‚ Priority: HIGH β”‚ β”‚ +β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ +β”‚ Action needed: β”‚ β”‚ Action needed: β”‚ β”‚ Action needed: β”‚ β”‚ +β”‚ β–Έ Place bid? β”‚ β”‚ β–Έ Review item? β”‚ β”‚ β–Έ Place final β”‚ β”‚ +β”‚ β–Έ Monitor? β”‚ β”‚ β–Έ Confirm value? β”‚ β”‚ bid? β”‚ β”‚ +β”‚ β–Έ Ignore? β”‚ β”‚ β–Έ Add to watch? β”‚ β”‚ β–Έ Let expire? β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β”‚ β”‚ β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ USER ACTIONS & EXCEPTIONS β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ Additional interaction points: β”‚ +β”‚ β”‚ +β”‚ 4. VIEWING DAY QUESTIONS β”‚ +β”‚ "Bezichtiging op [date] - kunt u aanwezig zijn?" β”‚ +β”‚ Action: β–Έ Confirm attendance β–Έ Request alternative β–Έ Decline β”‚ +β”‚ β”‚ +β”‚ 5. ITEM RECOGNITION CONFIRMATION β”‚ +β”‚ "Detected: [object] - Is deze correcte identificatie?" β”‚ +β”‚ Action: β–Έ Confirm β–Έ Correct label β–Έ Add notes β”‚ +β”‚ β”‚ +β”‚ 6. VALUE ESTIMATE APPROVAL β”‚ +β”‚ "Geschatte waarde: €X - Akkoord?" β”‚ +β”‚ Action: β–Έ Accept β–Έ Adjust β–Έ Request manual review β”‚ +β”‚ β”‚ +β”‚ 7. EXCEPTION HANDLING β”‚ +β”‚ "Afwijkende sluitingstijd / locatiewijziging / special terms" β”‚ +β”‚ Action: β–Έ Acknowledge β–Έ Update preferences β–Έ Withdraw interest β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ OBJECT DETECTION & VALUE ESTIMATION PIPELINE β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +[Downloaded Image] β†’ [ImageProcessingService] + β”‚ β”‚ + β”‚ β–Ό + β”‚ [ObjectDetectionService] + β”‚ β”‚ + β”‚ β”œβ”€ Load YOLO model + β”‚ β”œβ”€ Run inference (416x416) + β”‚ β”œβ”€ Post-process detections + β”‚ β”‚ (confidence > 0.5) + β”‚ β”‚ + β”‚ β–Ό + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ β”‚ Detected Objects: β”‚ + β”‚ β”‚ - person β”‚ + β”‚ β”‚ - car β”‚ + β”‚ β”‚ - truck β”‚ + β”‚ β”‚ - furniture β”‚ + β”‚ β”‚ - machinery β”‚ + β”‚ β”‚ - electronics β”‚ + β”‚ β”‚ (80 COCO classes) β”‚ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β”‚ β–Ό + β”‚ [Value Estimation Logic] + β”‚ (Future enhancement) + β”‚ β”‚ + β”‚ β”œβ”€ Match objects to auction categories + β”‚ β”œβ”€ Historical price analysis + β”‚ β”œβ”€ Condition assessment + β”‚ β”œβ”€ Market trends + β”‚ β”‚ + β”‚ β–Ό + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ β”‚ Estimated Value: β”‚ + β”‚ β”‚ €X - €Y range β”‚ + β”‚ β”‚ Confidence: 75% β”‚ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + └──────────────────────┴─ [Save to DB] + β”‚ + β–Ό + [Trigger notification if + value > threshold] + +``` +```mermaid +flowchart TD + subgraph P1["PHASE 1: EXTERNAL SCRAPER (Python/Playwright)"] + direction LR + A1[Listing Pages
/auctions?page=N] --> A2[Extract URLs] + B1[Auction Pages
/a/auction-id] --> B2[Parse __NEXT_DATA__ JSON] + C1[Lot Pages
/l/lot-id] --> C2[Parse __NEXT_DATA__ JSON] + + A2 --> D1[INSERT auctions to SQLite] + B2 --> D1 + C2 --> D2[INSERT lots & image URLs] + + D1 --> DB[(SQLite Database
output/cache.db)] + D2 --> DB + end + + DB --> P2_Entry + + subgraph P2["PHASE 2: MONITORING & PROCESSING (Java)"] + direction TB + P2_Entry[Data Ready] --> Monitor[TroostwijkMonitor
Read lots every hour] + P2_Entry --> DBService[DatabaseService
Query & Import] + P2_Entry --> Adapter[ScraperDataAdapter
Transform TEXT β†’ INTEGER] + + Monitor --> BM[Bid Monitoring
Check API every 1h] + DBService --> IP[Image Processing
Download & Analyze] + Adapter --> DataForNotify[Formatted Data] + + BM --> BidUpdate{New bid?} + BidUpdate -->|Yes| UpdateDB[Update current_bid in DB] + UpdateDB --> NotifyTrigger1 + + IP --> Detection[Object Detection
YOLO/OpenCV DNN] + Detection --> ObjectCheck{Detect objects?} + ObjectCheck -->|Vehicle| Save1[Save labels & estimate value] + ObjectCheck -->|Furniture| Save2[Save labels & estimate value] + ObjectCheck -->|Machinery| Save3[Save labels & estimate value] + Save1 --> NotifyTrigger2 + Save2 --> NotifyTrigger2 + Save3 --> NotifyTrigger2 + + CA[Closing Alerts
Check < 5 min] --> TimeCheck{Time critical?} + TimeCheck -->|Yes| NotifyTrigger3 + end + + NotifyTrigger1 --> NS + NotifyTrigger2 --> NS + NotifyTrigger3 --> NS + + subgraph P3["PHASE 3: NOTIFICATION SYSTEM"] + NS[NotificationService] --> DN[Desktop Notify
Windows/macOS/Linux] + NS --> EN[Email Notify
Gmail SMTP] + NS --> PL[Set Priority Level
0=Normal, 1=High] + end + + DN --> UI[User Interaction & Decisions] + EN --> UI + PL --> UI + + subgraph UI_Details[User Decision Points / Trigger Events] + direction LR + E1["1. BID CHANGE
'Nieuw bod op kavel 12345...'
Actions: Place bid? Monitor? Ignore?"] + E2["2. OBJECT DETECTED
'Lot contains: Vehicle...'
Actions: Review? Confirm value?"] + E3["3. CLOSING ALERT
'Kavel 12345 sluit binnen 5 min.'
Actions: Place final bid? Let expire?"] + E4["4. VIEWING DAY QUESTIONS
'Bezichtiging op [date]...'"] + E5["5. ITEM RECOGNITION CONFIRMATION
'Detected: [object]...'"] + E6["6. VALUE ESTIMATE APPROVAL
'Geschatte waarde: €X...'"] + E7["7. EXCEPTION HANDLING
'Afwijkende sluitingstijd...'"] + end + + UI --> UI_Details + + %% Object Detection Sub-Flow Detail + subgraph P2_Detail["Object Detection & Value Estimation Pipeline"] + direction LR + DI[Downloaded Image] --> IPS[ImageProcessingService] + IPS --> ODS[ObjectDetectionService] + ODS --> Load[Load YOLO model] + ODS --> Run[Run inference] + ODS --> Post[Post-process detections
confidence > 0.5] + Post --> ObjList["Detected Objects List
(80 COCO classes)"] + ObjList --> VEL[Value Estimation Logic
Future enhancement] + VEL --> Match[Match to categories] + VEL --> History[Historical price analysis] + VEL --> Condition[Condition assessment] + VEL --> Market[Market trends] + Market --> ValueEst["Estimated Value Range
Confidence: 75%"] + ValueEst --> SaveToDB[Save to Database] + SaveToDB --> TriggerNotify{Value > threshold?} + end + + IP -.-> P2_Detail + TriggerNotify -.-> NotifyTrigger2 +``` +## Integration Hooks & Timing + +| Event | Frequency | Trigger | Notification Type | User Action Required | +|--------------------------------|-------------------|----------------------------|----------------------------|------------------------| +| **New auction discovered** | On scrape | Scraper finds new auction | Desktop + Email (optional) | Review auction | +| **Bid change detected** | Every 1 hour | Monitor detects higher bid | Desktop + Email | Place counter-bid? | +| **Closing soon (< 30 min)** | When detected | Time-based check | Desktop + Email | Review lot | +| **Closing imminent (< 5 min)** | When detected | Time-based check | Desktop + Email (HIGH) | Final bid decision | +| **Object detected** | On image process | YOLO finds objects | Desktop + Email | Confirm identification | +| **Value estimated** | After detection | Estimation complete | Desktop + Email | Approve estimate | +| **Viewing day scheduled** | From lot metadata | Scraper extracts date | Desktop + Email | Confirm attendance | +| **Exception/Change** | On update | Scraper detects change | Desktop + Email (HIGH) | Acknowledge | + +## Project Structure + +``` +src/main/java/com/auction/ +β”œβ”€β”€ Main.java # Entry point +β”œβ”€β”€ TroostwijkMonitor.java # Monitoring & orchestration +β”œβ”€β”€ DatabaseService.java # SQLite operations +β”œβ”€β”€ ScraperDataAdapter.java # Schema translation (TEXTβ†’INT, €→float) +β”œβ”€β”€ ImageProcessingService.java # Downloads & processes images +β”œβ”€β”€ ObjectDetectionService.java # OpenCV YOLO detection +β”œβ”€β”€ NotificationService.java # Desktop + Email notifications (FREE) +β”œβ”€β”€ Lot.java # Domain model for auction lots +β”œβ”€β”€ AuctionInfo.java # Domain model for auctions +└── Console.java # Logging utility +``` + +## Configuration + +Edit `TroostwijkScraper.main()` to customize: + +- **Database file**: `troostwijk.db` (SQLite database location) +- **YOLO paths**: Model configuration and weights files +- **Monitoring frequency**: Default is every 1 hour +- **Closing alerts**: Default is 5 minutes before closing + +## Database Schema + +The scraper creates three tables: + +**sales** +- `sale_id` (PRIMARY KEY) +- `title`, `location`, `closing_time` + +**lots** +- `lot_id` (PRIMARY KEY) +- `sale_id`, `title`, `description`, `manufacturer`, `type`, `year` +- `category`, `current_bid`, `currency`, `url` +- `closing_time`, `closing_notified` + +**images** +- `id` (PRIMARY KEY) +- `lot_id`, `url`, `local_path`, `labels` (detected objects) + +## Notification Examples + +### Desktop Notification +![System Tray Notification] +``` +πŸ”” Kavel bieding update +Nieuw bod op kavel 12345: €150.00 (was €125.00) +``` + +### Email Notification +``` +From: your.email@gmail.com +To: your.email@gmail.com +Subject: [Troostwijk] Kavel bieding update + +Nieuw bod op kavel 12345: €150.00 (was €125.00) +``` + +**High Priority Alerts** (closing soon): +``` +⚠️ Lot nearing closure +Kavel 12345 sluit binnen 5 min. +``` + +## Why This Approach? + +βœ… **100% Free** - No paid services (Twilio, Pushover, etc.) +βœ… **No External Dependencies** - Desktop notifications built into Java +βœ… **Works Offline** - Desktop notifications don't need internet +βœ… **Privacy First** - Your data stays on your machine +βœ… **Cross-Platform** - Windows, macOS, Linux supported +βœ… **Optional Email** - Add Gmail notifications if you want + +## Troubleshooting + +### Desktop Notifications Not Showing + +- **Windows**: Check if Java has notification permissions +- **Linux**: Ensure you have a desktop environment running (not headless) +- **macOS**: Check System Preferences β†’ Notifications + +### Email Not Sending + +1. Verify 2FA is enabled in Google Account +2. Confirm you're using an **App Password** (not your regular Gmail password) +3. Check that "Less secure app access" is NOT needed (app passwords work with 2FA) +4. Verify the SMTP format: `smtp:username:app_password:recipient` + +## Notes + +- Desktop notifications require a graphical environment (not headless servers) +- For headless servers, use email-only notifications +- Gmail SMTP is free and has generous limits (500 emails/day) +- OpenCV native libraries must match your platform architecture +- YOLO weights file is ~245 MB + + +```shell +ssh tour@athena.lan "docker run --rm -v shared-auction-data:/data -v /tmp:/tmp alpine cp /data/cache.db /tmp/cache.db" && scp tour@athena.lan:/tmp/cache.db c:/mnt/okcomputer/cache.db +``` +## License + +This is example code for educational purposes. + + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c21cae2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,56 @@ +services: + auctiora: + user: "1000:1000" + build: + context: /opt/apps/auctiora + dockerfile: Dockerfile + container_name: auctiora + restart: unless-stopped + networks: + - traefik_net + environment: + # Database configuration + - AUCTION_DATABASE_PATH=/mnt/okcomputer/output/cache.db + - AUCTION_IMAGES_PATH=/mnt/okcomputer/output/images + + # Notification configuration + # - AUCTION_NOTIFICATION_CONFIG=desktop + - AUCTION_NOTIFICATION_CONFIG=smtp:michael.bakker1986@gmail.com:agrepolhlnvhipkv:michael.bakker1986@gmail.com + # Quarkus configuration + - QUARKUS_HTTP_PORT=8081 + - QUARKUS_HTTP_HOST=0.0.0.0 + - QUARKUS_LOG_CONSOLE_LEVEL=INFO + + # Scheduler configuration (cron expressions) + - AUCTION_WORKFLOW_SCRAPER_IMPORT_CRON=0 */30 * * * ? + - AUCTION_WORKFLOW_IMAGE_PROCESSING_CRON=0 0 * * * ? + - AUCTION_WORKFLOW_BID_MONITORING_CRON=0 */15 * * * ? + - AUCTION_WORKFLOW_CLOSING_ALERTS_CRON=0 */5 * * * ? + + volumes: + # Mount database and images directory1 + - shared-auction-data:/mnt/okcomputer/output + + labels: + - "traefik.enable=true" + - "traefik.http.routers.auctiora.rule=Host(`auctiora.appmodel.nl`)" + - "traefik.http.routers.auctiora.entrypoints=websecure" + - "traefik.http.routers.auctiora.tls=true" + - "traefik.http.routers.auctiora.tls.certresolver=letsencrypt" + - "traefik.http.services.auctiora.loadbalancer.server.port=8081" + + #healthcheck: + # test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8081/q/health/live"] + # interval: 30s + # timeout: 3s + # retries: 3 + # start_period: 10s + +networks: + traefik_net: + external: true + name: traefik_net + +volumes: + shared-auction-data: + external: true \ No newline at end of file diff --git a/docs/ARCHITECTURE-TROOSTWIJK-SCRAPER.md b/docs/ARCHITECTURE-TROOSTWIJK-SCRAPER.md new file mode 100644 index 0000000..98a9682 --- /dev/null +++ b/docs/ARCHITECTURE-TROOSTWIJK-SCRAPER.md @@ -0,0 +1,326 @@ +# Troostwijk Scraper - Architecture & Data Flow + +## System Overview + +The scraper follows a **3-phase hierarchical crawling pattern** to extract auction and lot data from Troostwijk Auctions website. + +## Architecture Diagram + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ TROOSTWIJK SCRAPER β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ PHASE 1: COLLECT AUCTION URLs β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Listing Page │────────▢│ Extract /a/ β”‚ β”‚ +β”‚ β”‚ /auctions? β”‚ β”‚ auction URLs β”‚ β”‚ +β”‚ β”‚ page=1..N β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ +β”‚ β–Ό β”‚ +β”‚ [ List of Auction URLs ] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ PHASE 2: EXTRACT LOT URLs FROM AUCTIONS β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Auction Page │────────▢│ Parse β”‚ β”‚ +β”‚ β”‚ /a/... β”‚ β”‚ __NEXT_DATA__β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ JSON β”‚ β”‚ +β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”‚ β”‚ β”‚ +β”‚ β–Ό β–Ό β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Save Auction β”‚ β”‚ Extract /l/ β”‚ β”‚ +β”‚ β”‚ Metadata β”‚ β”‚ lot URLs β”‚ β”‚ +β”‚ β”‚ to DB β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ +β”‚ β–Ό β”‚ +β”‚ [ List of Lot URLs ] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ PHASE 3: SCRAPE LOT DETAILS β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Lot Page │────────▢│ Parse β”‚ β”‚ +β”‚ β”‚ /l/... β”‚ β”‚ __NEXT_DATA__β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ JSON β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”‚ β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β–Ό β–Ό β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Save Lot β”‚ β”‚ Save Images β”‚ β”‚ +β”‚ β”‚ Details β”‚ β”‚ URLs to DB β”‚ β”‚ +β”‚ β”‚ to DB β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ +β”‚ β–Ό β”‚ +β”‚ [Optional Download] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## Database Schema + +```sql +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ CACHE TABLE (HTML Storage with Compression) β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ cache β”‚ +β”‚ β”œβ”€β”€ url (TEXT, PRIMARY KEY) β”‚ +β”‚ β”œβ”€β”€ content (BLOB) -- Compressed HTML (zlib) β”‚ +β”‚ β”œβ”€β”€ timestamp (REAL) β”‚ +β”‚ β”œβ”€β”€ status_code (INTEGER) β”‚ +β”‚ └── compressed (INTEGER) -- 1=compressed, 0=plain β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ AUCTIONS TABLE β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ auctions β”‚ +β”‚ β”œβ”€β”€ auction_id (TEXT, PRIMARY KEY) -- e.g. "A7-39813" β”‚ +β”‚ β”œβ”€β”€ url (TEXT, UNIQUE) β”‚ +β”‚ β”œβ”€β”€ title (TEXT) β”‚ +β”‚ β”œβ”€β”€ location (TEXT) -- e.g. "Cluj-Napoca, RO" β”‚ +β”‚ β”œβ”€β”€ lots_count (INTEGER) β”‚ +β”‚ β”œβ”€β”€ first_lot_closing_time (TEXT) β”‚ +β”‚ └── scraped_at (TEXT) β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ LOTS TABLE β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ lots β”‚ +β”‚ β”œβ”€β”€ lot_id (TEXT, PRIMARY KEY) -- e.g. "A1-28505-5" β”‚ +β”‚ β”œβ”€β”€ auction_id (TEXT) -- FK to auctions β”‚ +β”‚ β”œβ”€β”€ url (TEXT, UNIQUE) β”‚ +β”‚ β”œβ”€β”€ title (TEXT) β”‚ +β”‚ β”œβ”€β”€ current_bid (TEXT) -- "€123.45" or "No bids" β”‚ +β”‚ β”œβ”€β”€ bid_count (INTEGER) β”‚ +β”‚ β”œβ”€β”€ closing_time (TEXT) β”‚ +β”‚ β”œβ”€β”€ viewing_time (TEXT) β”‚ +β”‚ β”œβ”€β”€ pickup_date (TEXT) β”‚ +β”‚ β”œβ”€β”€ location (TEXT) -- e.g. "Dongen, NL" β”‚ +β”‚ β”œβ”€β”€ description (TEXT) β”‚ +β”‚ β”œβ”€β”€ category (TEXT) β”‚ +β”‚ └── scraped_at (TEXT) β”‚ +β”‚ FOREIGN KEY (auction_id) β†’ auctions(auction_id) β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ IMAGES TABLE (Image URLs & Download Status) β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ images ◀── THIS TABLE HOLDS IMAGE LINKSβ”‚ +β”‚ β”œβ”€β”€ id (INTEGER, PRIMARY KEY AUTOINCREMENT) β”‚ +β”‚ β”œβ”€β”€ lot_id (TEXT) -- FK to lots β”‚ +β”‚ β”œβ”€β”€ url (TEXT) -- Image URL β”‚ +β”‚ β”œβ”€β”€ local_path (TEXT) -- Path after download β”‚ +β”‚ └── downloaded (INTEGER) -- 0=pending, 1=downloaded β”‚ +β”‚ FOREIGN KEY (lot_id) β†’ lots(lot_id) β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## Sequence Diagram + +``` +User Scraper Playwright Cache DB Data Tables + β”‚ β”‚ β”‚ β”‚ β”‚ + β”‚ Run β”‚ β”‚ β”‚ β”‚ + β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Άβ”‚ β”‚ β”‚ β”‚ + β”‚ β”‚ β”‚ β”‚ β”‚ + β”‚ β”‚ Phase 1: Listing Pages β”‚ β”‚ + β”‚ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Άβ”‚ β”‚ β”‚ + β”‚ β”‚ goto() β”‚ β”‚ β”‚ + β”‚ │◀──────────────── β”‚ β”‚ + β”‚ β”‚ HTML β”‚ β”‚ β”‚ + β”‚ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Άβ”‚ β”‚ + β”‚ β”‚ compress & cache β”‚ β”‚ + β”‚ β”‚ β”‚ β”‚ β”‚ + β”‚ β”‚ Phase 2: Auction Pages β”‚ β”‚ + β”‚ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Άβ”‚ β”‚ β”‚ + β”‚ │◀──────────────── β”‚ β”‚ + β”‚ β”‚ HTML β”‚ β”‚ β”‚ + β”‚ β”‚ β”‚ β”‚ β”‚ + β”‚ β”‚ Parse __NEXT_DATA__ JSON β”‚ β”‚ + β”‚ │────────────────────────────────────────────────▢│ + β”‚ β”‚ β”‚ β”‚ INSERT auctions + β”‚ β”‚ β”‚ β”‚ β”‚ + β”‚ β”‚ Phase 3: Lot Pages β”‚ β”‚ + β”‚ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Άβ”‚ β”‚ β”‚ + β”‚ │◀──────────────── β”‚ β”‚ + β”‚ β”‚ HTML β”‚ β”‚ β”‚ + β”‚ β”‚ β”‚ β”‚ β”‚ + β”‚ β”‚ Parse __NEXT_DATA__ JSON β”‚ β”‚ + β”‚ │────────────────────────────────────────────────▢│ + β”‚ β”‚ β”‚ β”‚ INSERT lots β”‚ + β”‚ │────────────────────────────────────────────────▢│ + β”‚ β”‚ β”‚ β”‚ INSERT imagesβ”‚ + β”‚ β”‚ β”‚ β”‚ β”‚ + β”‚ β”‚ Export to CSV/JSON β”‚ β”‚ + β”‚ │◀───────────────────────────────────────────────── + β”‚ β”‚ Query all data β”‚ β”‚ + │◀─────────────── β”‚ β”‚ β”‚ + β”‚ Results β”‚ β”‚ β”‚ β”‚ +``` + +## Data Flow Details + +### 1. **Page Retrieval & Caching** +``` +Request URL + β”‚ + β”œβ”€β”€β–Ά Check cache DB (with timestamp validation) + β”‚ β”‚ + β”‚ β”œβ”€[HIT]──▢ Decompress (if compressed=1) + β”‚ β”‚ └──▢ Return HTML + β”‚ β”‚ + β”‚ └─[MISS]─▢ Fetch via Playwright + β”‚ β”‚ + β”‚ β”œβ”€β”€β–Ά Compress HTML (zlib level 9) + β”‚ β”‚ ~70-90% size reduction + β”‚ β”‚ + β”‚ └──▢ Store in cache DB (compressed=1) + β”‚ + └──▢ Return HTML for parsing +``` + +### 2. **JSON Parsing Strategy** +``` +HTML Content + β”‚ + └──▢ Extract + + + + + + +
+ + +
+
+
+
+

+ + Auctiora Intelligence Dashboard +

+

Real-time Auction Analytics & Predictive Monitoring

+
+
+
+ + System Online + Uptime: 00:00:00 +
+
Last updated: --:--
+
+ + +
+
+
+
+
+ + +
+ +
+
+
+
+
Active Auctions
+
+ +
+
+ -- +
+
+
+ +
+
+
+ +
+
+
+
Total Lots
+
+ +
+
+ -- +
+
+
+ +
+
+
+ +
+
+
+
Image Assets
+
+ +
+
+ -- +
+
+
+ +
+
+
+ +
+
+
+
Active Bidding
+
+ +
+
+ -- +
+
+
+ +
+
+
+ +
+
+
+
Closing Soon
+
+ +
+
+ -- +
+
+
+ +
+
+
+
+ + +
+
+ +

Live Insights

+
+
+
+ Loading insights... +
+
+
+ + +
+ +
+
+
+

+ Sleeper Lots +

+

High interest, low bids

+
+
+ +
+
+ +
+ + +
+
+
+

+ Bargains +

+

Below estimate

+
+
+ +
+
+ +
+ + +
+
+
+

+ Hot Lots +

+

High competition

+
+
+ +
+
+ +
+
+ + +
+
+
+

+ + Bidding Intelligence +

+
+ + -- lots/hour +
+
+
+
+
Lots with Bids
+
+ +
+
--% conversion
+
+
+
Total Bid Value
+
+ +
+
--% vs avg
+
+
+
Average Bid
+
+ +
+
--% premium
+
+
+
+ +
+

+ + API Performance +

+
+
+ + Total Requests + + + + +
+
+ + Success Rate + + + + +
+
+ + Failed + + + + +
+
+ + Avg Latency + + + + +
+
+
+
+ + +
+
+

+ + Workflow Orchestration +

+
+ All systems ready +
+
+
+ + + + + + + +
+
+ + +
+ +
+
+

+ + Geographic Distribution +

+
+ Live +
+
+
+
+
+ +
+
+
+
+
--
+
Top Market
+
+
+
--
+
Countries
+
+
+
--
+
Growth
+
+
+
+ + +
+
+

+ + Category Intelligence +

+
+ Live +
+
+
+
+
+ +
+
+
+
+
--
+
Top Category
+
+
+
--
+
Categories
+
+
+
--
+
Avg Bids
+
+
+
+
+ + +
+

+ + Market Activity Trend (Last 24h) +

+
+
+
+ +
+
+
+ + +
+
+

+ + Critical Time Alerts + + -- + +

+
+ + + + + + + +
+
+
+ + + + + + + + + + + + + + + + + + +
+ Lot ID + + Title + + Watchers + + Current Bid + + Est. Range + + Total Cost + + Time Left + + Actions +
+ + Loading critical alerts... +
+
+
+
Showing 0 of 0 lots
+ +
+
+ + +
+
+

+ + Activity Intelligence +

+
+ + +
+
+
+
+ + --:-- + Dashboard initialized and monitoring started... +
+
+
+
Last 50 events | Auto-scroll enabled
+
Errors: 0
+
+
+
+ + + + + + + \ No newline at end of file diff --git a/src/main/resources/META-INF/resources/script.js b/src/main/resources/META-INF/resources/script.js new file mode 100644 index 0000000..e69de29 diff --git a/src/main/resources/META-INF/resources/status.html b/src/main/resources/META-INF/resources/status.html new file mode 100644 index 0000000..0746686 --- /dev/null +++ b/src/main/resources/META-INF/resources/status.html @@ -0,0 +1,224 @@ + + + + + + Scrape-UI 1 - Enterprise + + + + + +
+
+

Scrape-UI Enterprise

+

Powered by Quarkus + Modern Frontend

+
+
+ + +
+ + +
+

Build & Runtime Status

+
+
+ +
+

πŸ“¦ Maven Build

+
+
+ Group: + - +
+
+ Artifact: + - +
+
+ Version: + - +
+
+
+ + +
+

πŸš€ Runtime

+
+
+ Status: + - +
+
+ Java: + - +
+
+ Platform: + - +
+
+
+
+ + +
+
+
+

Last Updated

+

-

+
+ +
+
+
+
+ + +
+

API Test

+ +
+ 
Click the button to test the API
+
+
+ + +
+
+

⚑ Quarkus Backend

+

Fast startup, low memory footprint, optimized for containers

+
+
+

πŸš€ REST API

+

RESTful endpoints with JSON responses

+
+
+

🎨 Modern UI

+

Responsive design with Tailwind CSS

+
+
+
+ + + + diff --git a/src/main/resources/META-INF/resources/style.css b/src/main/resources/META-INF/resources/style.css new file mode 100644 index 0000000..e69de29 diff --git a/src/main/resources/META-INF/resources/valuation-analytics.html b/src/main/resources/META-INF/resources/valuation-analytics.html new file mode 100644 index 0000000..da25d6d --- /dev/null +++ b/src/main/resources/META-INF/resources/valuation-analytics.html @@ -0,0 +1,1086 @@ + + + + + <!DOCTYPE html> + <html lang="en"> + <head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>Auctiora - Valuation Analytics + + + + + + + +
+
+
+
+ + Back to Dashboard + +
+

+ + Valuation Analytics Engine +

+

Mathematical Framework for Auction Intelligence

+
+
+
+ +
+
+
+
+ + +
+ + +
+ + +
+

+ + Input Parameters +

+ +
+ +
+

Current State

+ +
+ + +
+ +
+ + + 7.5 +
+ +
+ + +
+ +
+ + +
+
+ + +
+

Market Context

+ +
+ + + 0.15 +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+
+
+ + +
+

Auction Estimates

+
+
+ + +
+
+ + +
+
+
+
+ + +
+ +
+
+ Fair Market Value (FMV) + +
+
€8,500.00
+
87% confidence
+
+ + +
+
+ Undervaluation Score + +
+
0.153
+
+ 15.3% below fair value +
+
+ + +
+
+ Predicted Final Price + +
+
€8,900.00
+
+ 95% CI: €8,200 - €9,600 +
+
+
+
+ + +
+ + +
+

+ + 1. Fair Market Value (FMV) +

+
+ FMV = Ξ£(Pi Β· Ο‰c Β· Ο‰t Β· Ο‰p Β· Ο‰h) / Ξ£(Ο‰c Β· Ο‰t Β· Ο‰p Β· Ο‰h) +
+
+

Explanation: Weighted average of comparable sales where each comparable is weighted by:

+
    +
  • Ο‰_c Condition similarity (exponential decay)
    • +
  • Ο‰_t Age proximity (exponential decay)
    • +
  • Ο‰_p Provenance premium (linear boost)
    • +
  • Ο‰_h Historical relevance (logistic function)
    • +
+
+
+ + +
+

+ + 2. Undervaluation Detection +

+
+ Uscore = (FMV - Pcurrent)/FMV Β· Οƒmarket Β· (1 + Bvelocity/10) Β· ln(1 + Wwatch/Wbid) +
+
+

Explanation: Quantifies mispricing opportunity factoring:

+
    +
  • Price gap percentage
    • +
  • Market volatility multiplier
    • +
  • Bid velocity acceleration
    • +
  • Watch-to-bid ratio (buyer intent)
    • +
+

Alert threshold: Uscore > 0.25

+
+
+ + +
+

+ + 3. Final Price Prediction +

+
+ PΜ‚final = FMV Β· (1 + Ξ΅bid + Ξ΅time + Ξ΅comp) +
+
+

Explanation: Adjusts FMV for auction dynamics:

+
    +
  • Ξ΅_bid Bid momentum (tanh function)
    • +
  • Ξ΅_time Time pressure (exponential decay)
    • +
  • Ξ΅_comp Competition level (logarithmic)
    • +
+
+
+ + +
+

+ + 4. Condition Multiplier +

+
+ Mcond = exp(αc · √Ctarget - βc) +
+
+

Explanation: Normalizes prices across condition states using square-root scaling:

+
    +
  • C = 10 (mint): 1.48x premium
    • +
  • C = 7.5 (good): 1.12x premium
    • +
  • C = 5 (avg): 0.91x discount
    • +
+
+
+
+ + +
+

+ + Bidding Strategy Recommendations +

+ +
+
+
€9,200
+
Recommended Max Bid
+
Standard strategy
+
+ +
+
10 min
+
Optimal Bid Timing
+
Before close
+
+ +
+
Medium
+
Competition Level
+
2.3 bids/min
+
+
+ + +
+
+
+ +
+ Analysis: Bid velocity is moderate (2.3 bids/min) with high watch count indicating potential sniping. +
+
+
+ +
+ Recommendation: Wait until final 10 minutes. Set max bid at €9,200 (7% above FMV) to secure lot while avoiding bidding war. +
+
+
+ +
+ Risk Factors: High watch-to-bid ratio (87:8) suggests aggressive sniping likely. Reserve may be close to current bid. +
+
+
+
+
+ + +
+ +
+

+ + Condition Sensitivity +

+
+
+

How FMV changes with condition score. Current: 7.5

+
+
+ + +
+

+ + Time Pressure Impact +

+
+
+

Final price prediction vs. time to close. Current: 45 min

+
+
+
+ + +
+

+ + Calculation Log +

+
+
+ + System initialized. Ready for calculations... +
+
+
+
+ + + + + + + \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..cc6c6b9 --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,75 @@ +# Application Configuration +# Values will be injected from pom.xml during build +quarkus.application.name=${project.artifactId} +quarkus.application.version=${project.version} +# Custom properties for groupId if needed +application.groupId=${project.groupId} +application.artifactId=${project.artifactId} +application.version=${project.version} + + +# HTTP Configuration +quarkus.http.port=8081 +# ========== DEVELOPMENT (quarkus:dev) ========== +%dev.quarkus.http.host=127.0.0.1 +# ========== PRODUCTION (Docker/JAR) ========== +%prod.quarkus.http.host=0.0.0.0 +# ========== TEST PROFILE ========== +%test.quarkus.http.host=localhost + +# Enable CORS for frontend development +quarkus.http.cors=true +quarkus.http.cors.origins=* +quarkus.http.cors.methods=GET,POST,PUT,DELETE,OPTIONS +quarkus.http.cors.headers=accept,authorization,content-type,x-requested-with + +# Logging Configuration +quarkus.log.console.format=%d{HH:mm:ss} %-5p [%c{2.}] (%t) %s%e%n +quarkus.log.console.level=INFO + +# Development mode settings +%dev.quarkus.log.console.level=DEBUG +%dev.quarkus.live-reload.instrumentation=true + +# JVM Arguments for native access (Jansi, OpenCV, etc.) +quarkus.native.additional-build-args=--enable-native-access=ALL-UNNAMED + +# Production optimizations +%prod.quarkus.package.type=fast-jar +%prod.quarkus.http.enable-compression=true + +# Static resources +quarkus.http.enable-compression=true +quarkus.rest.path=/ +quarkus.http.root-path=/ + +# Auction Monitor Configuration +auction.database.path=/mnt/okcomputer/output/cache.db +auction.images.path=/mnt/okcomputer/output/images +# auction.notification.config=desktop +# Format: smtp:username:password:recipient_email +auction.notification.config=smtp:michael.bakker1986@gmail.com:agrepolhlnvhipkv:michael.bakker1986@gmail.com + +auction.yolo.config=/mnt/okcomputer/output/models/yolov4.cfg +auction.yolo.weights=/mnt/okcomputer/output/models/yolov4.weights +auction.yolo.classes=/mnt/okcomputer/output/models/coco.names + +# Scheduler Configuration +quarkus.scheduler.enabled=true +quarkus.scheduler.start-halted=false + +# Workflow Schedules +auction.workflow.scraper-import.cron=0 */30 * * * ? +auction.workflow.image-processing.cron=0 0 * * * ? +auction.workflow.bid-monitoring.cron=0 */15 * * * ? +auction.workflow.closing-alerts.cron=0 */5 * * * ? + +# HTTP Rate Limiting Configuration +# Prevents overloading external services and getting blocked +auction.http.rate-limit.default-max-rps=2 +auction.http.rate-limit.troostwijk-max-rps=1 +auction.http.timeout-seconds=30 + +# Health Check Configuration +quarkus.smallrye-health.root-path=/health + diff --git a/src/main/resources/simplelogger.properties b/src/main/resources/simplelogger.properties new file mode 100644 index 0000000..e410fe2 --- /dev/null +++ b/src/main/resources/simplelogger.properties @@ -0,0 +1,20 @@ +# SLF4J Simple Logger Configuration +# Set default log level (trace, debug, info, warn, error, off) +org.slf4j.simpleLogger.defaultLogLevel=warn + +# Show date/time in logs +org.slf4j.simpleLogger.showDateTime=true +org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss + +# Show thread name +org.slf4j.simpleLogger.showThreadName=false + +# Show log name (logger name) +org.slf4j.simpleLogger.showLogName=false + +# Show short log name +org.slf4j.simpleLogger.showShortLogName=true + +# Set specific logger levels +org.slf4j.simpleLogger.log.com.microsoft.playwright=warn +org.slf4j.simpleLogger.log.org.sqlite=warn diff --git a/src/test/java/auctiora/ClosingTimeCalculationTest.java b/src/test/java/auctiora/ClosingTimeCalculationTest.java new file mode 100644 index 0000000..40d75ba --- /dev/null +++ b/src/test/java/auctiora/ClosingTimeCalculationTest.java @@ -0,0 +1,138 @@ +package auctiora; + +import org.junit.jupiter.api.*; + +import java.time.LocalDateTime; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests for closing time calculations that power the UI + * Tests the minutesUntilClose() logic used in dashboard and alerts + */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@DisplayName("Closing Time Calculation Tests") +class ClosingTimeCalculationTest { + + @Test + @Order(1) + @DisplayName("Should calculate minutes until close for lot closing in 15 minutes") + void testMinutesUntilClose15Minutes() { + var lot = createLot(LocalDateTime.now().plusMinutes(15)); + long minutes = lot.minutesUntilClose(); + + assertTrue(minutes >= 14 && minutes <= 16, + "Should be approximately 15 minutes, was: " + minutes); + } + + @Test + @Order(2) + @DisplayName("Should calculate minutes until close for lot closing in 2 hours") + void testMinutesUntilClose2Hours() { + var lot = createLot(LocalDateTime.now().plusHours(2)); + long minutes = lot.minutesUntilClose(); + + assertTrue(minutes >= 119 && minutes <= 121, + "Should be approximately 120 minutes, was: " + minutes); + } + + @Test + @Order(3) + @DisplayName("Should return negative value for already closed lot") + void testMinutesUntilCloseNegative() { + var lot = createLot(LocalDateTime.now().minusHours(1)); + long minutes = lot.minutesUntilClose(); + + assertTrue(minutes < 0, + "Should be negative for closed lots, was: " + minutes); + } + + @Test + @Order(4) + @DisplayName("Should return MAX_VALUE when lot has no closing time") + void testMinutesUntilCloseNoTime() { + var lot = Lot.basic(100, 1001, "No closing time", "", "", "", 0, "General", + 100.0, "EUR", "http://test.com/1001", null, false); + long minutes = lot.minutesUntilClose(); + + assertEquals(Long.MAX_VALUE, minutes, + "Should return MAX_VALUE when no closing time set"); + } + + @Test + @Order(5) + @DisplayName("Should identify lots closing within 5 minutes (critical threshold)") + void testCriticalClosingThreshold() { + var closing4Min = createLot(LocalDateTime.now().plusMinutes(4)); + var closing5Min = createLot(LocalDateTime.now().plusMinutes(5)); + var closing6Min = createLot(LocalDateTime.now().plusMinutes(6)); + + assertTrue(closing4Min.minutesUntilClose() < 5, + "Lot closing in 4 min should be < 5 minutes"); + assertTrue(closing5Min.minutesUntilClose() >= 5, + "Lot closing in 5 min should be >= 5 minutes"); + assertTrue(closing6Min.minutesUntilClose() > 5, + "Lot closing in 6 min should be > 5 minutes"); + } + + @Test + @Order(6) + @DisplayName("Should identify lots closing within 30 minutes (dashboard threshold)") + void testDashboardClosingThreshold() { + var closing20Min = createLot(LocalDateTime.now().plusMinutes(20)); + var closing31Min = createLot(LocalDateTime.now().plusMinutes(31)); // Use 31 to avoid boundary timing issue + var closing40Min = createLot(LocalDateTime.now().plusMinutes(40)); + + assertTrue(closing20Min.minutesUntilClose() < 30, + "Lot closing in 20 min should be < 30 minutes"); + assertTrue(closing31Min.minutesUntilClose() >= 30, + "Lot closing in 31 min should be >= 30 minutes"); + assertTrue(closing40Min.minutesUntilClose() > 30, + "Lot closing in 40 min should be > 30 minutes"); + } + + @Test + @Order(7) + @DisplayName("Should calculate correctly for lots closing soon (boundary cases)") + void testBoundaryCases() { + // Just closed (< 1 minute ago) + var justClosed = createLot(LocalDateTime.now().minusSeconds(30)); + assertTrue(justClosed.minutesUntilClose() <= 0, "Just closed should be <= 0"); + + // Closing very soon (< 1 minute) + var closingVerySoon = createLot(LocalDateTime.now().plusSeconds(30)); + assertTrue(closingVerySoon.minutesUntilClose() < 1, "Closing in 30 sec should be < 1 minute"); + + // Closing in exactly 1 hour + var closing1Hour = createLot(LocalDateTime.now().plusHours(1)); + long minutes1Hour = closing1Hour.minutesUntilClose(); + assertTrue(minutes1Hour >= 59 && minutes1Hour <= 61, + "Closing in 1 hour should be ~60 minutes, was: " + minutes1Hour); + } + + @Test + @Order(8) + @DisplayName("Multiple lots should sort correctly by urgency") + void testSortingByUrgency() { + var lot5Min = createLot(LocalDateTime.now().plusMinutes(5)); + var lot30Min = createLot(LocalDateTime.now().plusMinutes(30)); + var lot1Hour = createLot(LocalDateTime.now().plusHours(1)); + var lot3Hours = createLot(LocalDateTime.now().plusHours(3)); + + var lots = java.util.List.of(lot3Hours, lot30Min, lot5Min, lot1Hour); + var sorted = lots.stream() + .sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose())) + .toList(); + + assertEquals(lot5Min, sorted.get(0), "Most urgent should be first"); + assertEquals(lot30Min, sorted.get(1), "Second most urgent"); + assertEquals(lot1Hour, sorted.get(2), "Third most urgent"); + assertEquals(lot3Hours, sorted.get(3), "Least urgent should be last"); + } + + // Helper method + private Lot createLot(LocalDateTime closingTime) { + return Lot.basic(100, 1001, "Test Item", "", "", "", 0, "General", + 100.0, "EUR", "http://test.com/1001", closingTime, false); + } +} diff --git a/src/test/java/auctiora/DatabaseServiceTest.java b/src/test/java/auctiora/DatabaseServiceTest.java new file mode 100644 index 0000000..96cb997 --- /dev/null +++ b/src/test/java/auctiora/DatabaseServiceTest.java @@ -0,0 +1,390 @@ +package auctiora; + +import org.junit.jupiter.api.*; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Test cases for DatabaseService. + * Tests database operations including schema creation, CRUD operations, and data retrieval. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class DatabaseServiceTest { + + private DatabaseService db; + private String testDbPath; + + @BeforeAll + void setUp() throws SQLException { + // Load SQLite JDBC driver + try { + Class.forName("org.sqlite.JDBC"); + } catch (ClassNotFoundException e) { + throw new SQLException("SQLite JDBC driver not found", e); + } + + testDbPath = "test_database_" + System.currentTimeMillis() + ".db"; + db = new DatabaseService(testDbPath); + db.ensureSchema(); + } + + @AfterAll + void tearDown() throws Exception { + // Clean up test database + Files.deleteIfExists(Paths.get(testDbPath)); + } + + @Test + @DisplayName("Should create database schema successfully") + void testEnsureSchema() { + assertDoesNotThrow(() -> db.ensureSchema()); + } + + @Test + @DisplayName("Should insert and retrieve auction") + void testUpsertAndGetAuction() throws SQLException { + var auction = new AuctionInfo( + 12345, + "Test Auction", + "Amsterdam, NL", + "Amsterdam", + "NL", + "https://example.com/auction/12345", + "A7", + 50, + LocalDateTime.of(2025, 12, 15, 14, 30) + ); + + db.upsertAuction(auction); + + var auctions = db.getAllAuctions(); + assertFalse(auctions.isEmpty()); + + var retrieved = auctions.stream() + .filter(a -> a.auctionId() == 12345) + .findFirst() + .orElse(null); + + assertNotNull(retrieved); + assertEquals("Test Auction", retrieved.title()); + assertEquals("Amsterdam", retrieved.city()); + assertEquals("NL", retrieved.country()); + assertEquals(50, retrieved.lotCount()); + } + + @Test + @DisplayName("Should update existing auction on conflict") + void testUpsertAuctionUpdate() throws SQLException { + var auction1 = new AuctionInfo( + 99999, + "Original Title", + "Rotterdam, NL", + "Rotterdam", + "NL", + "https://example.com/auction/99999", + "A1", + 10, + null + ); + + db.upsertAuction(auction1); + + // Update with same ID + var auction2 = new AuctionInfo( + 99999, + "Updated Title", + "Rotterdam, NL", + "Rotterdam", + "NL", + "https://example.com/auction/99999", + "A1", + 20, + null + ); + + db.upsertAuction(auction2); + + var auctions = db.getAllAuctions(); + var retrieved = auctions.stream() + .filter(a -> a.auctionId() == 99999) + .findFirst() + .orElse(null); + + assertNotNull(retrieved); + assertEquals("Updated Title", retrieved.title()); + assertEquals(20, retrieved.lotCount()); + } + + @Test + @DisplayName("Should retrieve auctions by country code") + void testGetAuctionsByCountry() throws SQLException { + // Insert auctions from different countries + db.upsertAuction(new AuctionInfo( + 10001, "Dutch Auction", "Amsterdam, NL", "Amsterdam", "NL", + "https://example.com/10001", "A1", 10, null + )); + + db.upsertAuction(new AuctionInfo( + 10002, "Romanian Auction", "Cluj, RO", "Cluj", "RO", + "https://example.com/10002", "A2", 15, null + )); + + db.upsertAuction(new AuctionInfo( + 10003, "Another Dutch", "Utrecht, NL", "Utrecht", "NL", + "https://example.com/10003", "A3", 20, null + )); + + var nlAuctions = db.getAuctionsByCountry("NL"); + assertEquals(2, nlAuctions.stream().filter(a -> a.auctionId() >= 10001 && a.auctionId() <= 10003).count()); + + var roAuctions = db.getAuctionsByCountry("RO"); + assertEquals(1, roAuctions.stream().filter(a -> a.auctionId() == 10002).count()); + } + + @Test + @DisplayName("Should insert and retrieve lot") + void testUpsertAndGetLot() throws SQLException { + var lot = Lot.basic( + 12345, // saleId + 67890, // lotId + "Forklift", + "Electric forklift in good condition", + "Toyota", + "Electric", + 2018, + "Machinery", + 1500.00, + "EUR", + "https://example.com/lot/67890", + LocalDateTime.of(2025, 12, 20, 16, 0), + false + ); + + db.upsertLot(lot); + + var lots = db.getAllLots(); + assertFalse(lots.isEmpty()); + + var retrieved = lots.stream() + .filter(l -> l.lotId() == 67890) + .findFirst() + .orElse(null); + + assertNotNull(retrieved); + assertEquals("Forklift", retrieved.title()); + assertEquals("Toyota", retrieved.manufacturer()); + assertEquals(2018, retrieved.year()); + assertEquals(1500.00, retrieved.currentBid(), 0.01); + assertFalse(retrieved.closingNotified()); + } + + @Test + @DisplayName("Should update lot current bid") + void testUpdateLotCurrentBid() throws SQLException { + var lot = Lot.basic( + 11111, 22222, "Test Item", "Description", "", "", 0, "Category", + 100.00, "EUR", "https://example.com/lot/22222", null, false + ); + + db.upsertLot(lot); + + // Update bid + var updatedLot = Lot.basic( + 11111, 22222, "Test Item", "Description", "", "", 0, "Category", + 250.00, "EUR", "https://example.com/lot/22222", null, false + ); + + db.updateLotCurrentBid(updatedLot); + + var lots = db.getAllLots(); + var retrieved = lots.stream() + .filter(l -> l.lotId() == 22222) + .findFirst() + .orElse(null); + + assertNotNull(retrieved); + assertEquals(250.00, retrieved.currentBid(), 0.01); + } + + @Test + @DisplayName("Should update lot notification flags") + void testUpdateLotNotificationFlags() throws SQLException { + var lot = Lot.basic( + 33333, 44444, "Test Item", "Description", "", "", 0, "Category", + 100.00, "EUR", "https://example.com/lot/44444", null, false + ); + + db.upsertLot(lot); + + // Update notification flag + var updatedLot = Lot.basic( + 33333, 44444, "Test Item", "Description", "", "", 0, "Category", + 100.00, "EUR", "https://example.com/lot/44444", null, true + ); + + db.updateLotNotificationFlags(updatedLot); + + var lots = db.getAllLots(); + var retrieved = lots.stream() + .filter(l -> l.lotId() == 44444) + .findFirst() + .orElse(null); + + assertNotNull(retrieved); + assertTrue(retrieved.closingNotified()); + } + + @Test + @DisplayName("Should insert and retrieve image records") + void testInsertAndGetImages() throws SQLException { + // First create a lot + var lot = Lot.basic( + 55555, 66666, "Test Lot", "Description", "", "", 0, "Category", + 100.00, "EUR", "https://example.com/lot/66666", null, false + ); + db.upsertLot(lot); + + // Insert images + db.insertImage(66666, "https://example.com/img1.jpg", + "C:/images/66666/img1.jpg", List.of("car", "vehicle")); + + db.insertImage(66666, "https://example.com/img2.jpg", + "C:/images/66666/img2.jpg", List.of("truck")); + + var images = db.getImagesForLot(66666); + assertEquals(2, images.size()); + + var img1 = images.stream() + .filter(i -> i.url().contains("img1.jpg")) + .findFirst() + .orElse(null); + + assertNotNull(img1); + assertEquals("car,vehicle", img1.labels()); + } + + @Test + @DisplayName("Should count total images") + void testGetImageCount() throws SQLException { + int initialCount = db.getImageCount(); + + // Add a lot and image + var lot = Lot.basic( + 77777, 88888, "Test Lot", "Description", "", "", 0, "Category", + 100.00, "EUR", "https://example.com/lot/88888", null, false + ); + db.upsertLot(lot); + + db.insertImage(88888, "https://example.com/test.jpg", + "C:/images/88888/test.jpg", List.of("object")); + + int newCount = db.getImageCount(); + assertTrue(newCount > initialCount); + } + + @Test + @DisplayName("Should handle empty database gracefully") + void testEmptyDatabase() throws SQLException, IOException { + DatabaseService emptyDb = new DatabaseService("empty_test_" + System.currentTimeMillis() + ".db"); + emptyDb.ensureSchema(); + + var auctions = emptyDb.getAllAuctions(); + var lots = emptyDb.getAllLots(); + int imageCount = emptyDb.getImageCount(); + + assertNotNull(auctions); + assertNotNull(lots); + assertTrue(auctions.isEmpty()); + assertTrue(lots.isEmpty()); + assertEquals(0, imageCount); + + // Clean up + Files.deleteIfExists(Paths.get("empty_test_" + System.currentTimeMillis() + ".db")); + } + + @Test + @DisplayName("Should handle lots with null closing time") + void testLotWithNullClosingTime() throws SQLException { + var lot = Lot.basic( + 98765, 12340, "Test Item", "Description", "", "", 0, "Category", + 100.00, "EUR", "https://example.com/lot/12340", null, false + ); + + assertDoesNotThrow(() -> db.upsertLot(lot)); + + var retrieved = db.getAllLots().stream() + .filter(l -> l.lotId() == 12340) + .findFirst() + .orElse(null); + + assertNotNull(retrieved); + assertNull(retrieved.closingTime()); + } + + @Test + @DisplayName("Should retrieve active lots only") + void testGetActiveLots() throws SQLException { + var activeLot = Lot.basic( + 11111, 55551, "Active Lot", "Description", "", "", 0, "Category", + 100.00, "EUR", "https://example.com/lot/55551", + LocalDateTime.now().plusDays(1), false + ); + + db.upsertLot(activeLot); + + var activeLots = db.getActiveLots(); + assertFalse(activeLots.isEmpty()); + + var found = activeLots.stream() + .anyMatch(l -> l.lotId() == 55551); + assertTrue(found); + } + + @Test + @DisplayName("Should handle concurrent upserts") + void testConcurrentUpserts() throws InterruptedException, SQLException { + Thread t1 = new Thread(() -> { + try { + for (int i = 0; i < 10; i++) { + db.upsertLot(Lot.basic( + 99000 + i, 99100 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat", + 100.0, "EUR", "https://example.com/" + i, null, false + )); + } + } catch (Exception e) { + fail("Thread 1 failed: " + e.getMessage()); + } + }); + + Thread t2 = new Thread(() -> { + try { + for (int i = 10; i < 20; i++) { + db.upsertLot(Lot.basic( + 99000 + i, 99100 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat", + 200.0, "EUR", "https://example.com/" + i, null, false + )); + } + } catch (Exception e) { + fail("Thread 2 failed: " + e.getMessage()); + } + }); + + t1.start(); + t2.start(); + t1.join(); + t2.join(); + + var lots = db.getAllLots(); + long concurrentLots = lots.stream() + .filter(l -> l.lotId() >= 99100 && l.lotId() < 99120) + .count(); + + assertTrue(concurrentLots >= 20); + } +} diff --git a/src/test/java/auctiora/ImageProcessingServiceTest.java b/src/test/java/auctiora/ImageProcessingServiceTest.java new file mode 100644 index 0000000..4d5a47a --- /dev/null +++ b/src/test/java/auctiora/ImageProcessingServiceTest.java @@ -0,0 +1,186 @@ +package auctiora; + +import org.junit.jupiter.api.*; + +import java.sql.SQLException; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +/** + * Test cases for ImageProcessingService. + * Tests object detection integration and database label updates. + * + * NOTE: Image downloading is now handled by the scraper, so these tests + * focus only on object detection and label storage. + */ +class ImageProcessingServiceTest { + + private DatabaseService mockDb; + private ObjectDetectionService mockDetector; + private ImageProcessingService service; + private java.io.File testImage; + + @BeforeEach + void setUp() throws Exception { + mockDb = mock(DatabaseService.class); + mockDetector = mock(ObjectDetectionService.class); + service = new ImageProcessingService(mockDb, mockDetector); + + // Create a temporary test image file + testImage = java.io.File.createTempFile("test_image_", ".jpg"); + testImage.deleteOnExit(); + // Write minimal JPEG header to make it a valid file + try (var out = new java.io.FileOutputStream(testImage)) { + out.write(new byte[]{(byte)0xFF, (byte)0xD8, (byte)0xFF, (byte)0xE0}); + } + } + + @Test + @DisplayName("Should process single image and update labels") + void testProcessImage() throws SQLException { + // Normalize path (convert backslashes to forward slashes) + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + + // Mock object detection with normalized path + when(mockDetector.detectObjects(normalizedPath)) + .thenReturn(List.of("car", "vehicle")); + + // Process image + boolean result = service.processImage(1, testImage.getAbsolutePath(), 12345); + + // Verify success + assertTrue(result); + verify(mockDetector).detectObjects(normalizedPath); + verify(mockDb).updateImageLabels(1, List.of("car", "vehicle")); + } + + @Test + @DisplayName("Should handle empty detection results") + void testProcessImageWithNoDetections() throws SQLException { + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + + when(mockDetector.detectObjects(normalizedPath)) + .thenReturn(List.of()); + + boolean result = service.processImage(2, testImage.getAbsolutePath(), 12346); + + assertTrue(result); + verify(mockDb).updateImageLabels(2, List.of()); + } + + @Test + @DisplayName("Should handle database error gracefully") + void testProcessImageDatabaseError() { + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + + when(mockDetector.detectObjects(normalizedPath)) + .thenReturn(List.of("object")); + + doThrow(new RuntimeException("Database error")) + .when(mockDb).updateImageLabels(anyInt(), anyList()); + + // Should return false on error + boolean result = service.processImage(3, testImage.getAbsolutePath(), 12347); + assertFalse(result); + } + + @Test + @DisplayName("Should handle object detection error gracefully") + void testProcessImageDetectionError() { + when(mockDetector.detectObjects(anyString())) + .thenThrow(new RuntimeException("Detection failed")); + + // Should return false on error + boolean result = service.processImage(4, "/path/to/image4.jpg", 12348); + assertFalse(result); + } + + @Test + @DisplayName("Should process pending images batch") + void testProcessPendingImages() throws SQLException { + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + + // Mock pending images from database - use real test image path + when(mockDb.getImagesNeedingDetection()).thenReturn(List.of( + new DatabaseService.ImageDetectionRecord(1, 100L, testImage.getAbsolutePath()), + new DatabaseService.ImageDetectionRecord(2, 101L, testImage.getAbsolutePath()) + )); + + when(mockDetector.detectObjects(normalizedPath)) + .thenReturn(List.of("item1")) + .thenReturn(List.of("item2")); + + when(mockDb.getImageLabels(anyInt())) + .thenReturn(List.of("item1")) + .thenReturn(List.of("item2")); + + // Process batch + service.processPendingImages(); + + // Verify all images were processed + verify(mockDb).getImagesNeedingDetection(); + verify(mockDetector, times(2)).detectObjects(normalizedPath); + verify(mockDb, times(2)).updateImageLabels(anyInt(), anyList()); + } + + @Test + @DisplayName("Should handle empty pending images list") + void testProcessPendingImagesEmpty() throws SQLException { + when(mockDb.getImagesNeedingDetection()).thenReturn(List.of()); + + service.processPendingImages(); + + verify(mockDb).getImagesNeedingDetection(); + verify(mockDetector, never()).detectObjects(anyString()); + } + + @Test + @DisplayName("Should continue processing after single image failure") + void testProcessPendingImagesWithFailure() throws SQLException { + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + + when(mockDb.getImagesNeedingDetection()).thenReturn(List.of( + new DatabaseService.ImageDetectionRecord(1, 100L, testImage.getAbsolutePath()), + new DatabaseService.ImageDetectionRecord(2, 101L, testImage.getAbsolutePath()) + )); + + // First image fails, second succeeds + when(mockDetector.detectObjects(normalizedPath)) + .thenThrow(new RuntimeException("Detection error")) + .thenReturn(List.of("item")); + + when(mockDb.getImageLabels(2)) + .thenReturn(List.of("item")); + + service.processPendingImages(); + + // Verify second image was still processed + verify(mockDetector, times(2)).detectObjects(normalizedPath); + } + + @Test + @DisplayName("Should handle database query error in batch processing") + void testProcessPendingImagesDatabaseError() { + when(mockDb.getImagesNeedingDetection()) + .thenThrow(new RuntimeException("Database connection failed")); + + // Should not throw exception + assertDoesNotThrow(() -> service.processPendingImages()); + } + + @Test + @DisplayName("Should process images with multiple detected objects") + void testProcessImageMultipleDetections() throws SQLException { + String normalizedPath = testImage.getAbsolutePath().replace('\\', '/'); + + when(mockDetector.detectObjects(normalizedPath)) + .thenReturn(List.of("car", "truck", "vehicle", "road")); + + boolean result = service.processImage(5, testImage.getAbsolutePath(), 12349); + + assertTrue(result); + verify(mockDb).updateImageLabels(5, List.of("car", "truck", "vehicle", "road")); + } +} diff --git a/src/test/java/auctiora/IntegrationTest.java b/src/test/java/auctiora/IntegrationTest.java new file mode 100644 index 0000000..4cf1704 --- /dev/null +++ b/src/test/java/auctiora/IntegrationTest.java @@ -0,0 +1,461 @@ +package auctiora; + +import org.junit.jupiter.api.*; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Integration test for complete workflow. + * Tests end-to-end scenarios including: + * 1. Scraper data import + * 2. Data transformation + * 3. Image processing + * 4. Object detection + * 5. Bid monitoring + * 6. Notifications + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class IntegrationTest { + + private String testDbPath; + private DatabaseService db; + private NotificationService notifier; + private ObjectDetectionService detector; + private ImageProcessingService imageProcessor; + private TroostwijkMonitor monitor; + + @BeforeAll + void setUp() throws SQLException, IOException { + testDbPath = "test_integration_" + System.currentTimeMillis() + ".db"; + + // Initialize all services + db = new DatabaseService(testDbPath); + db.ensureSchema(); + + notifier = new NotificationService("desktop"); + + detector = new ObjectDetectionService( + "non_existent.cfg", + "non_existent.weights", + "non_existent.txt" + ); + + imageProcessor = new ImageProcessingService(db, detector); + + monitor = new TroostwijkMonitor( + testDbPath, + "desktop", + "non_existent.cfg", + "non_existent.weights", + "non_existent.txt" + ); + } + + @AfterAll + void tearDown() throws Exception { + Files.deleteIfExists(Paths.get(testDbPath)); + } + + @Test + @Order(1) + @DisplayName("Integration: Complete scraper data import workflow") + void testCompleteScraperImportWorkflow() throws SQLException { + // Step 1: Import auction from scraper format + var auction = new AuctionInfo( + 12345, + "Industrial Equipment Auction", + "Rotterdam, NL", + "Rotterdam", + "NL", + "https://example.com/auction/12345", + "A7", + 25, + LocalDateTime.now().plusDays(3) + ); + + db.upsertAuction(auction); + + // Step 2: Import lots for this auction + var lot1 = Lot.basic( + 12345, 10001, + "Toyota Forklift 2.5T", + "Electric forklift in excellent condition", + "Toyota", + "Electric", + 2018, + "Machinery", + 1500.00, + "EUR", + "https://example.com/lot/10001", + LocalDateTime.now().plusDays(3), + false + ); + + var lot2 = Lot.basic( + 12345, 10002, + "Office Furniture Set", + "Desks, chairs, and cabinets", + "", + "", + 0, + "Furniture", + 500.00, + "EUR", + "https://example.com/lot/10002", + LocalDateTime.now().plusDays(3), + false + ); + + db.upsertLot(lot1); + db.upsertLot(lot2); + + // Verify import + var auctions = db.getAllAuctions(); + var lots = db.getAllLots(); + + assertTrue(auctions.stream().anyMatch(a -> a.auctionId() == 12345)); + assertEquals(2, lots.stream().filter(l -> l.saleId() == 12345).count()); + } + + @Test + @Order(2) + @DisplayName("Integration: Image processing and detection workflow") + void testImageProcessingWorkflow() throws SQLException { + // Add images for a lot + db.insertImage(10001, "https://example.com/img1.jpg", + "C:/images/10001/img1.jpg", List.of("truck", "vehicle")); + + db.insertImage(10001, "https://example.com/img2.jpg", + "C:/images/10001/img2.jpg", List.of("forklift", "machinery")); + + // Verify images were saved + var images = db.getImagesForLot(10001); + assertEquals(2, images.size()); + + var labels = images.stream() + .flatMap(img -> List.of(img.labels().split(",")).stream()) + .distinct() + .toList(); + + assertTrue(labels.contains("truck") || labels.contains("forklift")); + } + + @Test + @Order(3) + @DisplayName("Integration: Bid monitoring and notification workflow") + void testBidMonitoringWorkflow() throws SQLException { + // Simulate bid change + var lot = db.getAllLots().stream() + .filter(l -> l.lotId() == 10001) + .findFirst() + .orElseThrow(); + + // Update bid + var updatedLot = Lot.basic( + lot.saleId(), lot.lotId(), lot.title(), lot.description(), + lot.manufacturer(), lot.type(), lot.year(), lot.category(), + 2000.00, // Increased from 1500.00 + lot.currency(), lot.url(), lot.closingTime(), lot.closingNotified() + ); + + db.updateLotCurrentBid(updatedLot); + + // Send notification + var message = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)", + lot.lotId(), 2000.00, 1500.00); + + assertDoesNotThrow(() -> + notifier.sendNotification(message, "Kavel bieding update", 0) + ); + + // Verify bid was updated + var refreshed = db.getAllLots().stream() + .filter(l -> l.lotId() == 10001) + .findFirst() + .orElseThrow(); + + assertEquals(2000.00, refreshed.currentBid(), 0.01); + } + + @Test + @Order(4) + @DisplayName("Integration: Closing alert workflow") + void testClosingAlertWorkflow() throws SQLException { + // Create lot closing soon + var closingSoon = Lot.basic( + 12345, 20001, + "Closing Soon Item", + "Description", + "", + "", + 0, + "Category", + 750.00, + "EUR", + "https://example.com/lot/20001", + LocalDateTime.now().plusMinutes(4), + false + ); + + db.upsertLot(closingSoon); + + // Check if lot is closing soon + assertTrue(closingSoon.minutesUntilClose() < 5); + + // Send high-priority notification + var message = "Kavel " + closingSoon.lotId() + " sluit binnen 5 min."; + assertDoesNotThrow(() -> + notifier.sendNotification(message, "Lot nearing closure", 1) + ); + + // Mark as notified + var notified = Lot.basic( + closingSoon.saleId(), closingSoon.lotId(), closingSoon.title(), + closingSoon.description(), closingSoon.manufacturer(), closingSoon.type(), + closingSoon.year(), closingSoon.category(), closingSoon.currentBid(), + closingSoon.currency(), closingSoon.url(), closingSoon.closingTime(), + true + ); + + db.updateLotNotificationFlags(notified); + + // Verify notification flag + var updated = db.getAllLots().stream() + .filter(l -> l.lotId() == 20001) + .findFirst() + .orElseThrow(); + + assertTrue(updated.closingNotified()); + } + + @Test + @Order(5) + @DisplayName("Integration: Multi-country auction filtering") + void testMultiCountryFiltering() throws SQLException { + // Add auctions from different countries + db.upsertAuction(new AuctionInfo( + 30001, "Dutch Auction", "Amsterdam, NL", "Amsterdam", "NL", + "https://example.com/30001", "A1", 10, null + )); + + db.upsertAuction(new AuctionInfo( + 30002, "Romanian Auction", "Cluj, RO", "Cluj", "RO", + "https://example.com/30002", "A2", 15, null + )); + + db.upsertAuction(new AuctionInfo( + 30003, "Belgian Auction", "Brussels, BE", "Brussels", "BE", + "https://example.com/30003", "A3", 20, null + )); + + // Filter by country + var nlAuctions = db.getAuctionsByCountry("NL"); + var roAuctions = db.getAuctionsByCountry("RO"); + var beAuctions = db.getAuctionsByCountry("BE"); + + assertTrue(nlAuctions.stream().anyMatch(a -> a.auctionId() == 30001)); + assertTrue(roAuctions.stream().anyMatch(a -> a.auctionId() == 30002)); + assertTrue(beAuctions.stream().anyMatch(a -> a.auctionId() == 30003)); + } + + @Test + @Order(6) + @DisplayName("Integration: Complete monitoring cycle") + void testCompleteMonitoringCycle() throws SQLException { + // Monitor should handle all lots + monitor.printDatabaseStats(); + + var activeLots = db.getActiveLots(); + assertFalse(activeLots.isEmpty()); + + // Process pending images + assertDoesNotThrow(() -> monitor.processPendingImages()); + + // Verify database integrity + var imageCount = db.getImageCount(); + assertTrue(imageCount >= 0); + } + + @Test + @Order(7) + @DisplayName("Integration: Data consistency across services") + void testDataConsistency() throws SQLException { + // Verify all auctions have valid data + var auctions = db.getAllAuctions(); + for (var auction : auctions) { + assertNotNull(auction.auctionId()); + assertNotNull(auction.title()); + assertNotNull(auction.url()); + } + + // Verify all lots have valid data + var lots = db.getAllLots(); + for (var lot : lots) { + assertNotNull(lot.lotId()); + assertNotNull(lot.title()); + assertTrue(lot.currentBid() >= 0); + } + } + + @Test + @Order(8) + @DisplayName("Integration: Object detection value estimation workflow") + void testValueEstimationWorkflow() throws SQLException { + // Create lot with detected objects + var lot = Lot.basic( + 40000, 50000, + "Construction Equipment", + "Heavy machinery for construction", + "Caterpillar", + "Excavator", + 2015, + "Machinery", + 25000.00, + "EUR", + "https://example.com/lot/50000", + LocalDateTime.now().plusDays(5), + false + ); + + db.upsertLot(lot); + + // Add images with detected objects + db.insertImage(50000, "https://example.com/excavator1.jpg", + "C:/images/50000/1.jpg", List.of("truck", "excavator", "machinery")); + + db.insertImage(50000, "https://example.com/excavator2.jpg", + "C:/images/50000/2.jpg", List.of("excavator", "construction")); + + // Retrieve and analyze + var images = db.getImagesForLot(50000); + assertFalse(images.isEmpty()); + + // Count unique objects + var allLabels = images.stream() + .flatMap(img -> List.of(img.labels().split(",")).stream()) + .distinct() + .toList(); + + assertTrue(allLabels.contains("excavator") || allLabels.contains("machinery")); + + // Simulate value estimation notification + var message = String.format( + "Lot contains: %s\nEstimated value: €%,.2f", + String.join(", ", allLabels), + lot.currentBid() + ); + + assertDoesNotThrow(() -> + notifier.sendNotification(message, "Object Detected", 0) + ); + } + + @Test + @Order(9) + @DisplayName("Integration: Handle rapid concurrent updates") + void testConcurrentOperations() throws InterruptedException, SQLException { + var auctionThread = new Thread(() -> { + try { + for (var i = 0; i < 10; i++) { + db.upsertAuction(new AuctionInfo( + 60000 + i, "Concurrent Auction " + i, "Test, NL", "Test", "NL", + "https://example.com/60" + i, "A1", 5, null + )); + } + } catch (Exception e) { + fail("Auction thread failed: " + e.getMessage()); + } + }); + + var lotThread = new Thread(() -> { + try { + for (var i = 0; i < 10; i++) { + db.upsertLot(Lot.basic( + 60000 + i, 70000 + i, "Concurrent Lot " + i, "Desc", "", "", 0, "Cat", + 100.0 * i, "EUR", "https://example.com/70" + i, null, false + )); + } + } catch (Exception e) { + fail("Lot thread failed: " + e.getMessage()); + } + }); + + auctionThread.start(); + lotThread.start(); + auctionThread.join(); + lotThread.join(); + + // Verify all were inserted + var auctions = db.getAllAuctions(); + var lots = db.getAllLots(); + + var auctionCount = auctions.stream() + .filter(a -> a.auctionId() >= 60000 && a.auctionId() < 60010) + .count(); + + var lotCount = lots.stream() + .filter(l -> l.lotId() >= 70000 && l.lotId() < 70010) + .count(); + + assertEquals(10, auctionCount); + assertEquals(10, lotCount); + } + + @Test + @Order(10) + @DisplayName("Integration: End-to-end notification scenarios") + void testAllNotificationScenarios() { + // 1. Bid change notification + assertDoesNotThrow(() -> + notifier.sendNotification( + "Nieuw bod op kavel 12345: €150.00 (was €125.00)", + "Kavel bieding update", + 0 + ) + ); + + // 2. Closing alert + assertDoesNotThrow(() -> + notifier.sendNotification( + "Kavel 67890 sluit binnen 5 min.", + "Lot nearing closure", + 1 + ) + ); + + // 3. Object detection + assertDoesNotThrow(() -> + notifier.sendNotification( + "Detected: car, truck, machinery", + "Object Detected", + 0 + ) + ); + + // 4. Value estimate + assertDoesNotThrow(() -> + notifier.sendNotification( + "Geschatte waarde: €5,000 - €7,500", + "Value Estimate", + 0 + ) + ); + + // 5. Viewing day reminder + assertDoesNotThrow(() -> + notifier.sendNotification( + "Bezichtiging op 15-12-2025 om 14:00", + "Viewing Day Reminder", + 0 + ) + ); + } +} diff --git a/src/test/java/auctiora/NotificationServiceTest.java b/src/test/java/auctiora/NotificationServiceTest.java new file mode 100644 index 0000000..d279e9e --- /dev/null +++ b/src/test/java/auctiora/NotificationServiceTest.java @@ -0,0 +1,243 @@ +package auctiora; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Test cases for NotificationService. + * Tests desktop and email notification configuration and delivery. + */ +class NotificationServiceTest { + + @Test + @DisplayName("Should initialize with desktop-only configuration") + void testDesktopOnlyConfiguration() { + var service = new NotificationService("desktop"); + assertNotNull(service); + } + + @Test + @DisplayName("Should initialize with SMTP configuration") + void testSMTPConfiguration() { + var service = new NotificationService( + "smtp:test@gmail.com:app_password:recipient@example.com" + ); + assertNotNull(service); + } + + @Test + @DisplayName("Should reject invalid SMTP configuration format") + void testInvalidSMTPConfiguration() { + // Missing parts (only 2 parts total) + assertThrows(IllegalArgumentException.class, () -> + new NotificationService("smtp:incomplete") + ); + + // Wrong format (only 3 parts total, needs 4) + assertThrows(IllegalArgumentException.class, () -> + new NotificationService("smtp:only:two") + ); + } + + @Test + @DisplayName("Should reject unknown configuration type") + void testUnknownConfiguration() { + assertThrows(IllegalArgumentException.class, () -> + new NotificationService("unknown_type") + ); + } + + @Test + @DisplayName("Should send desktop notification without error") + void testDesktopNotification() { + var service = new NotificationService("desktop"); + + // Should not throw exception even if system tray not available + assertDoesNotThrow(() -> + service.sendNotification("Test message", "Test title", 0) + ); + } + + @Test + @DisplayName("Should send high priority notification") + void testHighPriorityNotification() { + var service = new NotificationService("desktop"); + + assertDoesNotThrow(() -> + service.sendNotification("Urgent message", "High Priority", 1) + ); + } + + @Test + @DisplayName("Should send normal priority notification") + void testNormalPriorityNotification() { + var service = new NotificationService("desktop"); + + assertDoesNotThrow(() -> + service.sendNotification("Regular message", "Normal Priority", 0) + ); + } + + @Test + @DisplayName("Should handle notification when system tray not supported") + void testNoSystemTraySupport() { + var service = new NotificationService("desktop"); + + // Should gracefully handle missing system tray + assertDoesNotThrow(() -> + service.sendNotification("Test", "Test", 0) + ); + } + + @Test + @DisplayName("Should send email notification with valid SMTP config") + void testEmailNotificationWithValidConfig() { + // Note: This won't actually send email without valid credentials + // But it should initialize properly + var service = new NotificationService( + "smtp:test@gmail.com:fake_password:test@example.com" + ); + + // Should not throw during initialization + assertNotNull(service); + + // Sending will fail with fake credentials, but shouldn't crash + assertDoesNotThrow(() -> + service.sendNotification("Test email", "Email Test", 0) + ); + } + + @Test + @DisplayName("Should include both desktop and email when SMTP configured") + void testBothNotificationChannels() { + var service = new NotificationService( + "smtp:michael.bakker1986@gmail.com:agrepolhlnvhipkv:michael.bakker1986@gmail.com" + ); + + // Both desktop and email should be attempted + assertDoesNotThrow(() -> + service.sendNotification("Dual channel test", "Test", 0) + ); + } + + @Test + @DisplayName("Should handle empty message gracefully") + void testEmptyMessage() { + var service = new NotificationService("desktop"); + + assertDoesNotThrow(() -> + service.sendNotification("", "", 0) + ); + } + + @Test + @DisplayName("Should handle very long message") + void testLongMessage() { + var service = new NotificationService("desktop"); + + var longMessage = "A".repeat(1000); + assertDoesNotThrow(() -> + service.sendNotification(longMessage, "Long Message Test", 0) + ); + } + + @Test + @DisplayName("Should handle special characters in message") + void testSpecialCharactersInMessage() { + var service = new NotificationService("desktop"); + + assertDoesNotThrow(() -> + service.sendNotification( + "€123.45 - Kavel sluit binnen 5 min! ⚠️", + "Special Chars Test", + 1 + ) + ); + } + + @Test + @DisplayName("Should accept case-insensitive desktop config") + void testCaseInsensitiveDesktopConfig() { + assertDoesNotThrow(() -> { + new NotificationService("DESKTOP"); + new NotificationService("Desktop"); + new NotificationService("desktop"); + }); + } + + @Test + @DisplayName("Should validate SMTP config parts count") + void testSMTPConfigPartsValidation() { + // Too few parts + assertThrows(IllegalArgumentException.class, () -> + new NotificationService("smtp:user:pass") + ); + + // Too many parts should work (extras ignored in split) + assertDoesNotThrow(() -> + new NotificationService("smtp:user:pass:email:extra") + ); + } + + @Test + @DisplayName("Should handle multiple rapid notifications") + void testRapidNotifications() { + var service = new NotificationService("desktop"); + + assertDoesNotThrow(() -> { + for (var i = 0; i < 5; i++) { + service.sendNotification("Notification " + i, "Rapid Test", 0); + } + }); + } + + @Test + @DisplayName("Should handle notification with null config parameter") + void testNullConfigParameter() { + // Second parameter can be empty string (kept for compatibility) + assertDoesNotThrow(() -> + new NotificationService("desktop") + ); + } + + @Test + @DisplayName("Should send bid change notification format") + void testBidChangeNotificationFormat() { + var service = new NotificationService("desktop"); + + var message = "Nieuw bod op kavel 12345: €150.00 (was €125.00)"; + var title = "Kavel bieding update"; + + assertDoesNotThrow(() -> + service.sendNotification(message, title, 0) + ); + } + + @Test + @DisplayName("Should send closing alert notification format") + void testClosingAlertNotificationFormat() { + var service = new NotificationService("desktop"); + + var message = "Kavel 12345 sluit binnen 5 min."; + var title = "Lot nearing closure"; + + assertDoesNotThrow(() -> + service.sendNotification(message, title, 1) + ); + } + + @Test + @DisplayName("Should send object detection notification format") + void testObjectDetectionNotificationFormat() { + var service = new NotificationService("desktop"); + + var message = "Lot contains: car, truck, machinery\nEstimated value: €5000"; + var title = "Object Detected"; + + assertDoesNotThrow(() -> + service.sendNotification(message, title, 0) + ); + } +} diff --git a/src/test/java/auctiora/ObjectDetectionServiceTest.java b/src/test/java/auctiora/ObjectDetectionServiceTest.java new file mode 100644 index 0000000..355015b --- /dev/null +++ b/src/test/java/auctiora/ObjectDetectionServiceTest.java @@ -0,0 +1,185 @@ +package auctiora; + +import org.junit.jupiter.api.*; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Test cases for ObjectDetectionService. + * Tests YOLO model loading and object detection functionality. + */ +class ObjectDetectionServiceTest { + + private static final String TEST_CFG = "test_yolo.cfg"; + private static final String TEST_WEIGHTS = "test_yolo.weights"; + private static final String TEST_CLASSES = "test_classes.txt"; + + @Test + @DisplayName("Should initialize with missing YOLO models (disabled mode)") + void testInitializeWithoutModels() throws IOException { + ObjectDetectionService service = new ObjectDetectionService( + "non_existent.cfg", + "non_existent.weights", + "non_existent.txt" + ); + + assertNotNull(service); + } + + @Test + @DisplayName("Should return empty list when detection is disabled") + void testDetectObjectsWhenDisabled() throws IOException { + ObjectDetectionService service = new ObjectDetectionService( + "non_existent.cfg", + "non_existent.weights", + "non_existent.txt" + ); + + var result = service.detectObjects("any_image.jpg"); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + @DisplayName("Should handle invalid image path gracefully") + void testInvalidImagePath() throws IOException { + ObjectDetectionService service = new ObjectDetectionService( + "non_existent.cfg", + "non_existent.weights", + "non_existent.txt" + ); + + var result = service.detectObjects("completely_invalid_path.jpg"); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + @DisplayName("Should handle empty image file") + void testEmptyImageFile() throws IOException { + ObjectDetectionService service = new ObjectDetectionService( + "non_existent.cfg", + "non_existent.weights", + "non_existent.txt" + ); + + // Create empty test file + var tempFile = Files.createTempFile("test_image", ".jpg"); + try { + var result = service.detectObjects(tempFile.toString()); + assertNotNull(result); + assertTrue(result.isEmpty()); + } finally { + Files.deleteIfExists(tempFile); + } + } + + @Test + @DisplayName("Should gracefully handle when model files exist but OpenCV fails to load") + void testInitializeWithValidModels() throws IOException { + var cfgPath = Paths.get(TEST_CFG); + var weightsPath = Paths.get(TEST_WEIGHTS); + var classesPath = Paths.get(TEST_CLASSES); + + try { + Files.writeString(cfgPath, "[net]\nwidth=416\nheight=416\n"); + Files.write(weightsPath, new byte[]{0, 1, 2, 3}); + Files.writeString(classesPath, "person\ncar\ntruck\n"); + + // When files exist but OpenCV native library isn't loaded, + // service should construct successfully but be disabled (handled in @PostConstruct) + var service = new ObjectDetectionService(TEST_CFG, TEST_WEIGHTS, TEST_CLASSES); + // Service is created, but init() handles failures gracefully + // detectObjects should return empty list when disabled + assertNotNull(service); + } finally { + Files.deleteIfExists(cfgPath); + Files.deleteIfExists(weightsPath); + Files.deleteIfExists(classesPath); + } + } + + @Test + @DisplayName("Should handle missing class names file") + void testMissingClassNamesFile() throws IOException { + // When model files don't exist, service initializes in disabled mode (no exception) + ObjectDetectionService service = new ObjectDetectionService( + "non_existent.cfg", + "non_existent.weights", + "non_existent.txt" + ); + assertNotNull(service); + // Verify it returns empty results when disabled + assertTrue(service.detectObjects("test.jpg").isEmpty()); + } + + @Test + @DisplayName("Should detect when model files are missing") + void testDetectMissingModelFiles() throws IOException { + // Should initialize in disabled mode + ObjectDetectionService service = new ObjectDetectionService( + "missing.cfg", + "missing.weights", + "missing.names" + ); + + // Should return empty results when disabled + var results = service.detectObjects("test.jpg"); + assertTrue(results.isEmpty()); + } + + @Test + @DisplayName("Should return unique labels only") + void testUniqueLabels() throws IOException { + // When disabled, returns empty list (unique by default) + ObjectDetectionService service = new ObjectDetectionService( + "non_existent.cfg", + "non_existent.weights", + "non_existent.txt" + ); + + var result = service.detectObjects("test.jpg"); + assertNotNull(result); + assertEquals(0, result.size()); + } + + @Test + @DisplayName("Should handle multiple detections in same image") + void testMultipleDetections() throws IOException { + // Test structure for when detection works + // With actual YOLO models, this would return multiple objects + ObjectDetectionService service = new ObjectDetectionService( + "non_existent.cfg", + "non_existent.weights", + "non_existent.txt" + ); + + var result = service.detectObjects("test_image.jpg"); + assertNotNull(result); + // When disabled, returns empty list + assertTrue(result.isEmpty()); + } + + @Test + @DisplayName("Should respect confidence threshold") + void testConfidenceThreshold() throws IOException { + // The service uses 0.5 confidence threshold + // This test documents that behavior + ObjectDetectionService service = new ObjectDetectionService( + "non_existent.cfg", + "non_existent.weights", + "non_existent.txt" + ); + + // Low confidence detections should be filtered out + // (when detection is working) + var result = service.detectObjects("test.jpg"); + assertNotNull(result); + } +} diff --git a/src/test/java/auctiora/ParserTest.java b/src/test/java/auctiora/ParserTest.java new file mode 100644 index 0000000..0fa1009 --- /dev/null +++ b/src/test/java/auctiora/ParserTest.java @@ -0,0 +1,62 @@ +package auctiora; + +import com.vladsch.flexmark.html2md.converter.FlexmarkHtmlConverter; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; +import org.junit.jupiter.api.Test; +@Slf4j +public class ParserTest { + + public record AuctionItem( + String title, + String link, + int lotCount, + String location, + String closingTime + ) { } + + public static AuctionItem parseItem(String html, String baseUrl) { + var doc = Jsoup.parse(html, baseUrl); + + var li = doc.selectFirst("li.grid"); + if (li == null) return null; + + var linkEl = li.selectFirst("a[data-cy=item-link]"); + var link = linkEl != null ? linkEl.absUrl("href") : null; + + var title = text(li, "div.heading-6"); + + var closingTime = text(li, "[data-cy=end-time-text]"); + + var lotCountStr = text(li, "[data-cy=lot-count-text]").trim(); + var lotCount = lotCountStr.isEmpty() ? 0 : Integer.parseInt(lotCountStr); + + // Tweede span in de location grid + var location = li.select("[data-cy=location-text] span").size() >= 2 + ? li.select("[data-cy=location-text] span").get(1).text() + : null; + + return new AuctionItem(title, link, lotCount, location, closingTime); + } + + private static String text(org.jsoup.nodes.Element root, String css) { + var el = root.selectFirst(css); + return el != null ? el.text() : ""; + } + + @Test + void testbla() { + var html = "
  • 03:17:00
    \"\"
    \"\"
    \"\"
    \"\"
    115
    Sluiting van een metaalbewerkingsfabriek – CNC-bewerkingscentra, draadvonkmachine, gereedschapsmachines en meer
    Vahingen, DE
    • "; + var doc = Jsoup.parse(html, "https://www.troostwijkauctions.com"); + var markdown = FlexmarkHtmlConverter.builder().build().convert(html); + + log.info(String.valueOf(doc.body())); + var item = ParserTest.parseItem(html, "https://www.troostwijkauctions.com"); + + log.info(item.title()); + log.info(item.link()); + log.info(String.valueOf(item.lotCount())); + log.info(item.location()); + log.info(item.closingTime()); + } +} \ No newline at end of file diff --git a/src/test/java/auctiora/ScraperDataAdapterTest.java b/src/test/java/auctiora/ScraperDataAdapterTest.java new file mode 100644 index 0000000..1dcfabf --- /dev/null +++ b/src/test/java/auctiora/ScraperDataAdapterTest.java @@ -0,0 +1,275 @@ +package auctiora; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.DisplayName; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.Instant; +import java.time.ZoneId; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +/** + * Test cases for ScraperDataAdapter. + * Tests conversion from external scraper schema to monitor schema. + */ +class ScraperDataAdapterTest { + + @Test + @DisplayName("Should extract numeric ID from text format auction ID") + void testExtractNumericIdFromAuctionId() { + assertEquals(39813, ScraperDataAdapter.extractNumericId("A7-39813")); + assertEquals(12345, ScraperDataAdapter.extractNumericId("A1-12345")); + assertEquals(0, ScraperDataAdapter.extractNumericId(null)); + assertEquals(0, ScraperDataAdapter.extractNumericId("")); + assertEquals(0, ScraperDataAdapter.extractNumericId("ABC")); + } + + @Test + @DisplayName("Should extract numeric ID from text format lot ID") + void testExtractNumericIdFromLotId() { + // "A1-28505-5" β†’ 285055 (concatenates all digits) + assertEquals(285055, ScraperDataAdapter.extractNumericId("A1-28505-5")); + assertEquals(123456, ScraperDataAdapter.extractNumericId("A7-1234-56")); + } + + @Test + @DisplayName("Should return 0 for IDs that exceed Long.MAX_VALUE") + void testExtractNumericIdTooLarge() { + // These IDs are too large for a long (> 19 digits or > Long.MAX_VALUE) + assertEquals(0, ScraperDataAdapter.extractNumericId("856462986966260305674")); + assertEquals(0, ScraperDataAdapter.extractNumericId("28492384530402679688")); + assertEquals(0, ScraperDataAdapter.extractNumericId("A7-856462986966260305674")); + } + + @Test + @DisplayName("Should convert scraper auction format to AuctionInfo") + void testFromScraperAuction() throws SQLException { + // Mock ResultSet with scraper format data + ResultSet rs = mock(ResultSet.class); + when(rs.getString("auction_id")).thenReturn("A7-39813"); + when(rs.getString("title")).thenReturn("Industrial Equipment Auction"); + when(rs.getString("location")).thenReturn("Cluj-Napoca, RO"); + when(rs.getString("url")).thenReturn("https://example.com/auction/A7-39813"); + when(rs.getInt("lots_count")).thenReturn(150); + when(rs.getString("first_lot_closing_time")).thenReturn("2025-12-15T14:30:00"); + + AuctionInfo result = ScraperDataAdapter.fromScraperAuction(rs); + + assertNotNull(result); + assertEquals(39813, result.auctionId()); + assertEquals("Industrial Equipment Auction", result.title()); + assertEquals("Cluj-Napoca, RO", result.location()); + assertEquals("Cluj-Napoca", result.city()); + assertEquals("RO", result.country()); + assertEquals("https://example.com/auction/A7-39813", result.url()); + assertEquals("A7", result.typePrefix()); + assertEquals(150, result.lotCount()); + assertNotNull(result.firstLotClosingTime()); + } + + @Test + @DisplayName("Should handle auction with simple location without country") + void testFromScraperAuctionSimpleLocation() throws SQLException { + ResultSet rs = mock(ResultSet.class); + when(rs.getString("auction_id")).thenReturn("A1-12345"); + when(rs.getString("title")).thenReturn("Test Auction"); + when(rs.getString("location")).thenReturn("Amsterdam"); + when(rs.getString("url")).thenReturn("https://example.com/auction/A1-12345"); + when(rs.getInt("lots_count")).thenReturn(50); + when(rs.getString("first_lot_closing_time")).thenReturn(null); + + AuctionInfo result = ScraperDataAdapter.fromScraperAuction(rs); + + assertEquals("Amsterdam", result.city()); + assertEquals("", result.country()); + assertNull(result.firstLotClosingTime()); + } + + @Test + @DisplayName("Should convert scraper lot format to Lot") + void testFromScraperLot() throws SQLException { + ResultSet rs = mock(ResultSet.class); + when(rs.getString("lot_id")).thenReturn("A1-28505-5"); + when(rs.getString("auction_id")).thenReturn("A7-39813"); + when(rs.getString("title")).thenReturn("Forklift Toyota"); + when(rs.getString("description")).thenReturn("Electric forklift in good condition"); + when(rs.getString("category")).thenReturn("Machinery"); + when(rs.getString("current_bid")).thenReturn("€1250.50"); + when(rs.getString("closing_time")).thenReturn("2025-12-15T14:30:00"); + when(rs.getString("url")).thenReturn("https://example.com/lot/A1-28505-5"); + + Lot result = ScraperDataAdapter.fromScraperLot(rs); + + assertNotNull(result); + assertEquals(285055, result.lotId()); + assertEquals(39813, result.saleId()); + assertEquals("Forklift Toyota", result.title()); + assertEquals("Electric forklift in good condition", result.description()); + assertEquals("Machinery", result.category()); + assertEquals(1250.50, result.currentBid(), 0.01); + assertEquals("EUR", result.currency()); + assertEquals("https://example.com/lot/A1-28505-5", result.url()); + assertNotNull(result.closingTime()); + assertFalse(result.closingNotified()); + } + + @Test + @DisplayName("Should parse bid amount from various formats") + void testParseBidAmount() throws SQLException { + // Test €123.45 format + ResultSet rs1 = createLotResultSet("€123.45"); + Lot lot1 = ScraperDataAdapter.fromScraperLot(rs1); + assertEquals(123.45, lot1.currentBid(), 0.01); + assertEquals("EUR", lot1.currency()); + + // Test $50.00 format + ResultSet rs2 = createLotResultSet("$50.00"); + Lot lot2 = ScraperDataAdapter.fromScraperLot(rs2); + assertEquals(50.00, lot2.currentBid(), 0.01); + assertEquals("USD", lot2.currency()); + + // Test "No bids" format + ResultSet rs3 = createLotResultSet("No bids"); + Lot lot3 = ScraperDataAdapter.fromScraperLot(rs3); + assertEquals(0.0, lot3.currentBid(), 0.01); + + // Test plain number + ResultSet rs4 = createLotResultSet("999.99"); + Lot lot4 = ScraperDataAdapter.fromScraperLot(rs4); + assertEquals(999.99, lot4.currentBid(), 0.01); + } + + @Test + @DisplayName("Should handle missing or null fields gracefully") + void testHandleNullFields() throws SQLException { + ResultSet rs = mock(ResultSet.class); + when(rs.getString("lot_id")).thenReturn("A1-12345-1"); + when(rs.getString("auction_id")).thenReturn("A7-99999"); + when(rs.getString("title")).thenReturn("Test Lot"); + when(rs.getString("description")).thenReturn(null); + when(rs.getString("category")).thenReturn(null); + when(rs.getString("current_bid")).thenReturn(null); + when(rs.getString("closing_time")).thenReturn(null); + when(rs.getString("url")).thenReturn("https://example.com/lot"); + + Lot result = ScraperDataAdapter.fromScraperLot(rs); + + assertNotNull(result); + assertEquals("", result.description()); + assertEquals("", result.category()); + assertEquals(0.0, result.currentBid()); + assertNull(result.closingTime()); + } + + @Test + @DisplayName("Should parse various timestamp formats") + void testTimestampParsing() throws SQLException { + // ISO local date time + ResultSet rs1 = mock(ResultSet.class); + setupBasicLotMock(rs1); + when(rs1.getString("closing_time")).thenReturn("2025-12-15T14:30:00"); + Lot lot1 = ScraperDataAdapter.fromScraperLot(rs1); + assertNotNull(lot1.closingTime()); + assertEquals(LocalDateTime.of(2025, 12, 15, 14, 30, 0), lot1.closingTime()); + + // SQL timestamp format + ResultSet rs2 = mock(ResultSet.class); + setupBasicLotMock(rs2); + when(rs2.getString("closing_time")).thenReturn("2025-12-15 14:30:00"); + Lot lot2 = ScraperDataAdapter.fromScraperLot(rs2); + assertNotNull(lot2.closingTime()); + } + + @Test + @DisplayName("Should handle invalid timestamp gracefully") + void testInvalidTimestamp() throws SQLException { + ResultSet rs = mock(ResultSet.class); + setupBasicLotMock(rs); + when(rs.getString("closing_time")).thenReturn("invalid-date"); + + Lot result = ScraperDataAdapter.fromScraperLot(rs); + assertNull(result.closingTime()); + } + + @Test + @DisplayName("Should extract type prefix from auction ID") + void testTypeExtraction() throws SQLException { + ResultSet rs1 = mock(ResultSet.class); + when(rs1.getString("auction_id")).thenReturn("A7-39813"); + when(rs1.getString("title")).thenReturn("Test"); + when(rs1.getString("location")).thenReturn("Test, NL"); + when(rs1.getString("url")).thenReturn("http://test.com"); + when(rs1.getInt("lots_count")).thenReturn(10); + when(rs1.getString("first_lot_closing_time")).thenReturn(null); + + AuctionInfo auction1 = ScraperDataAdapter.fromScraperAuction(rs1); + assertEquals("A7", auction1.typePrefix()); + + ResultSet rs2 = mock(ResultSet.class); + when(rs2.getString("auction_id")).thenReturn("B1-12345"); + when(rs2.getString("title")).thenReturn("Test"); + when(rs2.getString("location")).thenReturn("Test, NL"); + when(rs2.getString("url")).thenReturn("http://test.com"); + when(rs2.getInt("lots_count")).thenReturn(10); + when(rs2.getString("first_lot_closing_time")).thenReturn(null); + + AuctionInfo auction2 = ScraperDataAdapter.fromScraperAuction(rs2); + assertEquals("B1", auction2.typePrefix()); + } + + @Test + @DisplayName("Should handle GBP currency symbol") + void testGBPCurrency() throws SQLException { + ResultSet rs = createLotResultSet("Β£75.00"); + Lot lot = ScraperDataAdapter.fromScraperLot(rs); + assertEquals(75.00, lot.currentBid(), 0.01); + assertEquals("GBP", lot.currency()); + } + + @Test + @DisplayName("Should parse epoch seconds timestamp") + void testParseEpochSeconds() { + long seconds = 1765994400L; + LocalDateTime expected = LocalDateTime.ofInstant(Instant.ofEpochSecond(seconds), ZoneId.systemDefault()); + LocalDateTime parsed = ScraperDataAdapter.parseTimestamp(String.valueOf(seconds)); + assertEquals(expected, parsed); + } + + @Test + @DisplayName("Should parse epoch milliseconds timestamp") + void testParseEpochMillis() { + long millis = 1765994400000L; + LocalDateTime expected = LocalDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneId.systemDefault()); + LocalDateTime parsed = ScraperDataAdapter.parseTimestamp(String.valueOf(millis)); + assertEquals(expected, parsed); + } + + // Helper methods + + private ResultSet createLotResultSet(String bidAmount) throws SQLException { + ResultSet rs = mock(ResultSet.class); + when(rs.getString("lot_id")).thenReturn("A1-12345-1"); + when(rs.getString("auction_id")).thenReturn("A7-99999"); + when(rs.getString("title")).thenReturn("Test Lot"); + when(rs.getString("description")).thenReturn("Test description"); + when(rs.getString("category")).thenReturn("Test"); + when(rs.getString("current_bid")).thenReturn(bidAmount); + when(rs.getString("closing_time")).thenReturn("2025-12-15T14:30:00"); + when(rs.getString("url")).thenReturn("https://example.com/lot"); + return rs; + } + + private void setupBasicLotMock(ResultSet rs) throws SQLException { + when(rs.getString("lot_id")).thenReturn("A1-12345-1"); + when(rs.getString("auction_id")).thenReturn("A7-99999"); + when(rs.getString("title")).thenReturn("Test Lot"); + when(rs.getString("description")).thenReturn("Test"); + when(rs.getString("category")).thenReturn("Test"); + when(rs.getString("current_bid")).thenReturn("€100.00"); + when(rs.getString("url")).thenReturn("https://example.com/lot"); + } +} diff --git a/src/test/java/auctiora/TroostwijkMonitorTest.java b/src/test/java/auctiora/TroostwijkMonitorTest.java new file mode 100644 index 0000000..5b6e1b5 --- /dev/null +++ b/src/test/java/auctiora/TroostwijkMonitorTest.java @@ -0,0 +1,380 @@ +package auctiora; + +import org.junit.jupiter.api.*; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.SQLException; +import java.time.LocalDateTime; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Test cases for TroostwijkMonitor. + * Tests monitoring orchestration, bid tracking, and notification triggers. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TroostwijkMonitorTest { + + private String testDbPath; + private TroostwijkMonitor monitor; + + @BeforeAll + void setUp() throws SQLException, IOException { + testDbPath = "test_monitor_" + System.currentTimeMillis() + ".db"; + + monitor = new TroostwijkMonitor( + testDbPath, + "desktop", + "non_existent.cfg", + "non_existent.weights", + "non_existent.txt" + ); + } + + @AfterAll + void tearDown() throws Exception { + Files.deleteIfExists(Paths.get(testDbPath)); + } + + @Test + @DisplayName("Should initialize monitor successfully") + void testMonitorInitialization() { + assertNotNull(monitor); + assertNotNull(monitor.getDb()); + } + + @Test + @DisplayName("Should print database stats without error") + void testPrintDatabaseStats() { + assertDoesNotThrow(() -> monitor.printDatabaseStats()); + } + + @Test + @DisplayName("Should process pending images without error") + void testProcessPendingImages() { + assertDoesNotThrow(() -> monitor.processPendingImages()); + } + + @Test + @DisplayName("Should handle empty database gracefully") + void testEmptyDatabaseHandling() throws SQLException { + var auctions = monitor.getDb().getAllAuctions(); + var lots = monitor.getDb().getAllLots(); + + assertNotNull(auctions); + assertNotNull(lots); + assertTrue(auctions.isEmpty() || auctions.size() >= 0); + } + + @Test + @DisplayName("Should track lots in database") + void testLotTracking() throws SQLException { + // Insert test lot + var lot = Lot.basic( + 11111, 22222, + "Test Forklift", + "Electric forklift in good condition", + "Toyota", + "Electric", + 2020, + "Machinery", + 1500.00, + "EUR", + "https://example.com/lot/22222", + LocalDateTime.now().plusDays(1), + false + ); + + monitor.getDb().upsertLot(lot); + + var lots = monitor.getDb().getAllLots(); + assertTrue(lots.stream().anyMatch(l -> l.lotId() == 22222)); + } + + @Test + @DisplayName("Should monitor lots closing soon") + void testClosingSoonMonitoring() throws SQLException { + // Insert lot closing in 4 minutes + var closingSoon = Lot.basic( + 33333, 44444, + "Closing Soon Item", + "Description", + "", + "", + 0, + "Category", + 100.00, + "EUR", + "https://example.com/lot/44444", + LocalDateTime.now().plusMinutes(4), + false + ); + + monitor.getDb().upsertLot(closingSoon); + + var lots = monitor.getDb().getActiveLots(); + var found = lots.stream() + .filter(l -> l.lotId() == 44444) + .findFirst() + .orElse(null); + + assertNotNull(found); + assertTrue(found.minutesUntilClose() < 30); + } + + @Test + @DisplayName("Should identify lots with time remaining") + void testTimeRemainingCalculation() throws SQLException { + var futureLot = Lot.basic( + 55555, 66666, + "Future Lot", + "Description", + "", + "", + 0, + "Category", + 200.00, + "EUR", + "https://example.com/lot/66666", + LocalDateTime.now().plusHours(2), + false + ); + + monitor.getDb().upsertLot(futureLot); + + var lots = monitor.getDb().getActiveLots(); + var found = lots.stream() + .filter(l -> l.lotId() == 66666) + .findFirst() + .orElse(null); + + assertNotNull(found); + assertTrue(found.minutesUntilClose() > 60); + } + + @Test + @DisplayName("Should handle lots without closing time") + void testLotsWithoutClosingTime() throws SQLException { + var noClosing = Lot.basic( + 77777, 88888, + "No Closing Time", + "Description", + "", + "", + 0, + "Category", + 150.00, + "EUR", + "https://example.com/lot/88888", + null, + false + ); + + monitor.getDb().upsertLot(noClosing); + + var lots = monitor.getDb().getActiveLots(); + var found = lots.stream() + .filter(l -> l.lotId() == 88888) + .findFirst() + .orElse(null); + + assertNotNull(found); + assertNull(found.closingTime()); + } + + @Test + @DisplayName("Should track notification status") + void testNotificationStatusTracking() throws SQLException { + var lot = Lot.basic( + 99999, 11110, + "Test Notification", + "Description", + "", + "", + 0, + "Category", + 100.00, + "EUR", + "https://example.com/lot/11110", + LocalDateTime.now().plusMinutes(3), + false + ); + + monitor.getDb().upsertLot(lot); + + // Update notification flag + var notified = Lot.basic( + 99999, 11110, + "Test Notification", + "Description", + "", + "", + 0, + "Category", + 100.00, + "EUR", + "https://example.com/lot/11110", + LocalDateTime.now().plusMinutes(3), + true + ); + + monitor.getDb().updateLotNotificationFlags(notified); + + var lots = monitor.getDb().getActiveLots(); + var found = lots.stream() + .filter(l -> l.lotId() == 11110) + .findFirst() + .orElse(null); + + assertNotNull(found); + assertTrue(found.closingNotified()); + } + + @Test + @DisplayName("Should update bid amounts") + void testBidAmountUpdates() throws SQLException { + var lot = Lot.basic( + 12121, 13131, + "Bid Update Test", + "Description", + "", + "", + 0, + "Category", + 100.00, + "EUR", + "https://example.com/lot/13131", + LocalDateTime.now().plusDays(1), + false + ); + + monitor.getDb().upsertLot(lot); + + // Simulate bid increase + var higherBid = Lot.basic( + 12121, 13131, + "Bid Update Test", + "Description", + "", + "", + 0, + "Category", + 250.00, + "EUR", + "https://example.com/lot/13131", + LocalDateTime.now().plusDays(1), + false + ); + + monitor.getDb().updateLotCurrentBid(higherBid); + + var lots = monitor.getDb().getActiveLots(); + var found = lots.stream() + .filter(l -> l.lotId() == 13131) + .findFirst() + .orElse(null); + + assertNotNull(found); + assertEquals(250.00, found.currentBid(), 0.01); + } + + @Test + @DisplayName("Should handle multiple concurrent lot updates") + void testConcurrentLotUpdates() throws InterruptedException, SQLException { + Thread t1 = new Thread(() -> { + try { + for (int i = 0; i < 5; i++) { + monitor.getDb().upsertLot(Lot.basic( + 20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat", + 100.0, "EUR", "https://example.com/" + i, null, false + )); + } + } catch (Exception e) { + fail("Thread 1 failed: " + e.getMessage()); + } + }); + + Thread t2 = new Thread(() -> { + try { + for (int i = 5; i < 10; i++) { + monitor.getDb().upsertLot(Lot.basic( + 20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat", + 200.0, "EUR", "https://example.com/" + i, null, false + )); + } + } catch (Exception e) { + fail("Thread 2 failed: " + e.getMessage()); + } + }); + + t1.start(); + t2.start(); + t1.join(); + t2.join(); + + var lots = monitor.getDb().getActiveLots(); + long count = lots.stream() + .filter(l -> l.lotId() >= 30000 && l.lotId() < 30010) + .count(); + + assertTrue(count >= 10); + } + + @Test + @DisplayName("Should schedule monitoring without error") + void testScheduleMonitoring() { + // This just tests that scheduling doesn't throw + // Actual monitoring would run in background + assertDoesNotThrow(() -> { + // Don't actually start monitoring in test + // Just verify monitor is ready + assertNotNull(monitor); + }); + } + + @Test + @DisplayName("Should handle database with auctions and lots") + void testDatabaseWithData() throws SQLException { + // Insert auction + var auction = new AuctionInfo( + 40000, + "Test Auction", + "Amsterdam, NL", + "Amsterdam", + "NL", + "https://example.com/auction/40000", + "A7", + 10, + LocalDateTime.now().plusDays(2) + ); + + monitor.getDb().upsertAuction(auction); + + // Insert related lot + var lot = Lot.basic( + 40000, 50000, + "Test Lot", + "Description", + "", + "", + 0, + "Category", + 500.00, + "EUR", + "https://example.com/lot/50000", + LocalDateTime.now().plusDays(2), + false + ); + + monitor.getDb().upsertLot(lot); + + // Verify + var auctions = monitor.getDb().getAllAuctions(); + var lots = monitor.getDb().getAllLots(); + + assertTrue(auctions.stream().anyMatch(a -> a.auctionId() == 40000)); + assertTrue(lots.stream().anyMatch(l -> l.lotId() == 50000)); + } +} diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties new file mode 100644 index 0000000..d531548 --- /dev/null +++ b/src/test/resources/application.properties @@ -0,0 +1,65 @@ +# Application Configuration +# Values will be injected from pom.xml during build +quarkus.application.name=${project.artifactId} +quarkus.application.version=${project.version} +# Custom properties for groupId if needed +application.groupId=${project.groupId} +application.artifactId=${project.artifactId} +application.version=${project.version} + + +# HTTP Configuration +quarkus.http.port=8081 +# ========== DEVELOPMENT (quarkus:dev) ========== +%dev.quarkus.http.host=127.0.0.1 +# ========== PRODUCTION (Docker/JAR) ========== +%prod.quarkus.http.host=0.0.0.0 +# ========== TEST PROFILE ========== +%test.quarkus.http.host=localhost + +# Enable CORS for frontend development +quarkus.http.cors=true +quarkus.http.cors.origins=* +quarkus.http.cors.methods=GET,POST,PUT,DELETE,OPTIONS +quarkus.http.cors.headers=accept,authorization,content-type,x-requested-with + +# Logging Configuration +quarkus.log.console.format=%d{HH:mm:ss} %-5p [%c{2.}] (%t) %s%e%n +quarkus.log.console.level=INFO + +# Development mode settings +%dev.quarkus.log.console.level=DEBUG +%dev.quarkus.live-reload.instrumentation=true + +# JVM Arguments for native access (Jansi, OpenCV, etc.) +quarkus.native.additional-build-args=--enable-native-access=ALL-UNNAMED + +# Production optimizations +%prod.quarkus.package.type=fast-jar +%prod.quarkus.http.enable-compression=true + +# Static resources +quarkus.http.enable-compression=true +quarkus.rest.path=/ +quarkus.http.root-path=/ + +# Auction Monitor Configuration +auction.database.path=/mnt/okcomputer/output/cache.db +auction.images.path=/mnt/okcomputer/output/images +# auction.notification.config=desktop +# Format: smtp:username:password:recipient_email +auction.notification.config=smtp:michael.bakker1986@gmail.com:agrepolhlnvhipkv:michael.bakker1986@gmail.com + +auction.yolo.config=/mnt/okcomputer/output/models/yolov4.cfg +auction.yolo.weights=/mnt/okcomputer/output/models/yolov4.weights +auction.yolo.classes=/mnt/okcomputer/output/models/coco.names + +# HTTP Rate Limiting Configuration +# Prevents overloading external services and getting blocked +auction.http.rate-limit.default-max-rps=2 +auction.http.rate-limit.troostwijk-max-rps=1 +auction.http.timeout-seconds=30 + +# Health Check Configuration +quarkus.smallrye-health.root-path=/health + diff --git a/workflows/maven.yml b/workflows/maven.yml new file mode 100644 index 0000000..59fa2c9 --- /dev/null +++ b/workflows/maven.yml @@ -0,0 +1,20 @@ +name: Publish to Gitea Package Registry +on: + push: + tags: + - 'v*' + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 25 + uses: actions/setup-java@v4 + with: + java-version: '25' + distribution: 'temurin' + - name: Publish with Maven + run: mvn --batch-mode clean deploy + env: + GITEA_TOKEN: ${{ secrets.EA_PUBLISH_TOKEN }} \ No newline at end of file