tour/auctiora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial clean commit

Browse Source
This commit is contained in:
Tour
2025-12-08 09:35:13 +01:00
commit 19a538d27a
79 changed files with 15794 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.aiassistant/rules/rules.md Normal file
View File

@@ -0,0 +1,4 @@
---
apply: always
---

12
.aiignore Normal file
View File

@@ -0,0 +1,12 @@
# An .aiignore file follows the same syntax as a .gitignore file.
# .gitignore documentation: https://git-scm.com/docs/gitignore
# you can ignore files
.DS_Store
*.log
*.tmp
# or folders
dist/
build/
out/

26
.dockerignore Normal file
View File

@@ -0,0 +1,26 @@
# Exclude large model files from Docker build
models/
*.weights
# Exclude build artifacts
target/
*.class
*.jar
# Exclude version control
.git/
.gitignore
# Exclude IDE files
.idea/
*.iml
.vscode/
# Exclude data files
images/
*.db
*.db-journal
# Exclude docs
README.md
*.md

1
.env Normal file
View File

@@ -0,0 +1 @@
GMAIL_APP_PASSWORD=agrepolhlnvhipkv

42
.github/workflows/__deploy.old vendored Normal file
View File

@@ -0,0 +1,42 @@
name: Build and Deploy
on:
push:
branches: ["main"]
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Java
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '21'
- name: Build
run: mvn -B clean package
- name: Upload to JFrog
run: |
curl -u "${{ secrets.JFROG_USER }}:${{ secrets.JFROG_PASS }}" \
-T target/*.jar \
"http://JFROG-SERVER/artifactory/myrepo/app-latest.jar"
deploy:
needs: build
runs-on: ubuntu-latest
steps:
- name: Trigger remote deploy script
uses: appleboy/ssh-action@v0.1.7
with:
host: ${{ secrets.SERVER_IP }}
username: ${{ secrets.SERVER_USER }}
key: ${{ secrets.SERVER_SSH_KEY }}
script: |
/opt/myapp/update.sh

50
.github/workflows/_oldbuild.nothing vendored Normal file
View File

@@ -0,0 +1,50 @@
name: Build and Deploy Auction App
on:
push:
branches:
- main
jobs:
build_and_deploy:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Maven
run: |
apt update
apt install -y maven
- name: Set up Java 17
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: 17
- name: Build with Maven
run: mvn -B clean package
- name: Copy jar to server (no tar)
uses: appleboy/scp-action@v0.1.7
with:
host: ${{ secrets.SERVER_IP }}
username: ${{ secrets.SERVER_USER }}
key: ${{ secrets.SERVER_SSH_KEY }}
source: "target/*.jar"
target: "/opt/auction/"
overwrite: true
strip_components: 1 # Strips the 'target/' directory
timeout: 60s
- name: Restart service
uses: appleboy/ssh-action@v0.1.7
with:
host: ${{ secrets.SERVER_IP }}
username: ${{ secrets.SERVER_USER }}
key: ${{ secrets.SERVER_SSH_KEY }}
script: |
systemctl restart auction
echo "Deploy complete"

34
.gitignore vendored Normal file
View File

@@ -0,0 +1,34 @@
### IntelliJ IDEA ###
out/
!**/src/main/**/out/
!**/src/test/**/out/
.kotlin
### Eclipse ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
bin/
!**/src/main/**/bin/
!**/src/test/**/bin/
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
### VS Code ###
.vscode/
### MacOS ###
.DS_Store
NUL
target/
.idea/

6
.mvn/jvm.config Normal file
View File

@@ -0,0 +1,6 @@
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--enable-native-access=ALL-UNNAMED

117
.mvn/wrapper/MavenWrapperDownloader.java vendored Normal file
View File

@@ -0,0 +1,117 @@
/*
* Copyright 2007-present the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.net.*;
import java.io.*;
import java.nio.channels.*;
import java.util.Properties;
public class MavenWrapperDownloader {
private static final String WRAPPER_VERSION = "0.5.6";
/**
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
*/
private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+ WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
/**
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
* use instead of the default one.
*/
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
".mvn/wrapper/maven-wrapper.properties";
/**
* Path where the maven-wrapper.jar will be saved to.
*/
private static final String MAVEN_WRAPPER_JAR_PATH =
".mvn/wrapper/maven-wrapper.jar";
/**
* Name of the property which should be used to override the default download url for the wrapper.
*/
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
public static void main(String args[]) {
System.out.println("- Downloader started");
File baseDirectory = new File(args[0]);
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
// If the maven-wrapper.properties exists, read it and check if it contains a custom
// wrapperUrl parameter.
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
String url = DEFAULT_DOWNLOAD_URL;
if(mavenWrapperPropertyFile.exists()) {
FileInputStream mavenWrapperPropertyFileInputStream = null;
try {
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
Properties mavenWrapperProperties = new Properties();
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
} catch (IOException e) {
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
} finally {
try {
if(mavenWrapperPropertyFileInputStream != null) {
mavenWrapperPropertyFileInputStream.close();
}
} catch (IOException e) {
// Ignore ...
}
}
}
System.out.println("- Downloading from: " + url);
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
if(!outputFile.getParentFile().exists()) {
if(!outputFile.getParentFile().mkdirs()) {
System.out.println(
"- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
}
}
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
try {
downloadFileFromURL(url, outputFile);
System.out.println("Done");
System.exit(0);
} catch (Throwable e) {
System.out.println("- Error downloading");
e.printStackTrace();
System.exit(1);
}
}
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
String username = System.getenv("MVNW_USERNAME");
char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
Authenticator.setDefault(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(username, password);
}
});
}
URL website = new URL(urlString);
ReadableByteChannel rbc;
rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(destination);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
rbc.close();
}
}

1
.mvn/wrapper/maven-wrapper.config vendored Normal file
View File

@@ -0,0 +1 @@
jvmArguments=-Djava.util.logging.manager=org.jboss.logmanager.LogManager --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED

BIN
.mvn/wrapper/maven-wrapper.jar vendored Normal file
View File

Binary file not shown.

2
.mvn/wrapper/maven-wrapper.properties vendored Normal file
View File

@@ -0,0 +1,2 @@
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip
wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar

BIN
.mvn/wrapper/maven-wrapper_old.jar vendored Normal file
View File

Binary file not shown.

41
Dockerfile Normal file
View File

@@ -0,0 +1,41 @@
# Stage 1: Build
FROM maven:3.9-eclipse-temurin-25-alpine AS builder
WORKDIR /app
# Copy POM first (allows for cached dependency layer)
COPY pom.xml .
RUN mvn dependency:resolve -B
COPY src ./src
# Updated with both properties to avoid the warning
RUN mvn package -DskipTests -Dquarkus.package.jar.type=uber-jar -Dquarkus.package.jar.enabled=true
# Stage 2: Runtime (DEBIAN-based for OpenCV native libs)
FROM eclipse-temurin:25-jre
WORKDIR /app
# Install dependencies + wget for health checks
RUN apt-get update && \
apt-get install -y --no-install-recommends \
libstdc++6 \
libgomp1 \
wget \
&& rm -rf /var/lib/apt/lists/*
# Create user (Debian syntax)
RUN groupadd -r quarkus && useradd -r -g quarkus quarkus
# Create user with explicit UID 1000 (Debian syntax)
# RUN groupadd -r -g 1000 quarkus && useradd -r -u 1000 -g quarkus quarkus
# Create the data directory and set ownership BEFORE switching user
# RUN mkdir -p /mnt/okcomputer/output && chown -R quarkus:quarkus /mnt/okcomputer/output
# Copy the built jar with correct pattern
COPY --from=builder --chown=quarkus:quarkus /app/target/auctiora-*.jar app.jar
USER quarkus
EXPOSE 8081
ENTRYPOINT ["java", \
"-Dio.netty.tryReflectionSetAccessible=true", \
"--enable-native-access=ALL-UNNAMED", \
"--add-opens", "java.base/java.nio=ALL-UNNAMED", \
"-jar", "app.jar"]

563
README.md Normal file
View File

@@ -0,0 +1,563 @@
# Troostwijk Auction Scraper
A Java-based web scraper for Dutch auctions on Troostwijk Auctions with **100% free** desktop/email notifications, SQLite persistence, and AI-powered object detection.
## Features
- **Auction Discovery**: Automatically discovers active Dutch auctions
- **Data Scraping**: Fetches detailed lot information via Troostwijk's JSON API
- **SQLite Storage**: Persists auction data, lots, images, and detected objects
- **Image Processing**: Downloads and analyzes lot images using OpenCV YOLO object detection
- **Free Notifications**: Real-time notifications when:
- Bids change on monitored lots
- Auctions are closing soon (within 5 minutes)
- Via desktop notifications (Windows/macOS/Linux system tray) ✅
- Optionally via email (Gmail SMTP - free) ✅
## Dependencies
All dependencies are managed via Maven (see `pom.xml`):
- **jsoup 1.17.2** - HTML parsing and HTTP client
- **Jackson 2.17.0** - JSON processing
- **SQLite JDBC 3.45.1.0** - Database operations
- **JavaMail 1.6.2** - Email notifications (free)
- **OpenCV 4.9.0** - Image processing and object detection
## Quick Start
### Development: Sync Production Data
To work with real production data locally:
```powershell
# Linux/Mac (Bash)
./scripts/sync-production-data.sh --db-only
```
See [scripts/README.md](scripts/README.md) for full documentation.
## Setup
### 1. Notification Options (Choose One)
#### Option A: Desktop Notifications Only ⭐ (Recommended - Zero Setup)
Desktop notifications work out of the box on:
- **Windows**: System tray notifications
- **macOS**: Notification Center
- **Linux**: Desktop environment notifications (GNOME, KDE, etc.)
**No configuration required!** Just run with default settings:
```bash
export NOTIFICATION_CONFIG="desktop"
# Or simply don't set it - desktop is the default
```
#### Option B: Desktop + Email Notifications 📧 (Free Gmail)
1. Enable 2-Factor Authentication in your Google Account
2. Go to: **Google Account → Security → 2-Step Verification → App passwords**
3. Generate an app password for "Mail"
4. Set environment variable:
```bash
export NOTIFICATION_CONFIG="smtp:your.email@gmail.com:your_app_password:recipient@example.com"
```
**Format**: `smtp:username:app_password:recipient_email`
**Example**:
```bash
export NOTIFICATION_CONFIG="smtp:john.doe@gmail.com:abcd1234efgh5678:john.doe@gmail.com"
```
**Note**: This is completely free using Gmail's SMTP server. No paid services required!
### 2. OpenCV Native Libraries
Download and install OpenCV native libraries for your platform:
**Windows:**
```bash
# Download from https://opencv.org/releases/
# Extract and add to PATH or use:
java -Djava.library.path="C:\opencv\build\java\x64" -jar scraper.jar
```
**Linux:**
```bash
sudo apt-get install libopencv-dev
```
**macOS:**
```bash
brew install opencv
```
### 3. YOLO Model Files
Download YOLO model files for object detection:
```bash
mkdir /mnt/okcomputer/output/models
cd /mnt/okcomputer/output/models
# Download YOLOv4 config
wget https://raw.githubusercontent.com/AlexeyAB/darknet/master/cfg/yolov4.cfg
# Download YOLOv4 weights (245 MB)
wget https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v3_optimal/yolov4.weights
# Download COCO class names
wget https://raw.githubusercontent.com/AlexeyAB/darknet/master/data/coco.names
```
## Building
```bash
mvn clean package
```
This creates:
- `target/troostwijk-scraper-1.0-SNAPSHOT.jar` - Regular JAR
- `target/troostwijk-scraper-1.0-SNAPSHOT-jar-with-dependencies.jar` - Executable JAR with all dependencies
## Running
### Quick Start (Desktop Notifications Only)
```bash
java -Djava.library.path="/path/to/opencv/lib" \
-jar target/troostwijk-scraper-1.0-SNAPSHOT-jar-with-dependencies.jar
```
### With Email Notifications
```bash
export NOTIFICATION_CONFIG="smtp:your@gmail.com:app_password:your@gmail.com"
java -Djava.library.path="/path/to/opencv/lib" \
-jar target/troostwijk-scraper-1.0-SNAPSHOT-jar-with-dependencies.jar
```
### Using Maven
```bash
mvn exec:java -Dexec.mainClass="com.auction.scraper.TroostwijkScraper"
```
## System Architecture & Integration Flow
> **📊 Complete Integration Flowchart**: See [docs/INTEGRATION_FLOWCHART.md](docs/INTEGRATION_FLOWCHART.md) for the detailed intelligence integration diagram with GraphQL API fields, analytics, and dashboard features.
### Quick Overview
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ COMPLETE SYSTEM INTEGRATION DIAGRAM │
└─────────────────────────────────────────────────────────────────────────────┘
┌──────────────────────────────────────────────────────────────────────────────┐
│ PHASE 1: EXTERNAL SCRAPER (Python/Playwright) - ARCHITECTURE-TROOSTWIJK │
└──────────────────────────────────────────────────────────────────────────────┘
│
┌─────────────────────────────┼─────────────────────────────┐
▼ ▼ ▼
[Listing Pages] [Auction Pages] [Lot Pages]
/auctions?page=N /a/auction-id /l/lot-id
│ │ │
│ Extract URLs │ Parse __NEXT_DATA__ │ Parse __NEXT_DATA__
├────────────────────────────▶│ JSON │ JSON
│ │ │
│ ▼ ▼
│ ┌────────────────┐ ┌────────────────┐
│ │ INSERT auctions│ │ INSERT lots │
│ │ to SQLite │ │ INSERT images │
│ └────────────────┘ │ (URLs only) │
│ │ └────────────────┘
│ │ │
└─────────────────────────────┴────────────────────────────┘
▼
┌──────────────────┐
│ SQLITE DATABASE │
│ output/cache.db │
└──────────────────┘
│
┌─────────────────┼─────────────────┐
▼ ▼ ▼
[auctions table] [lots table] [images table]
- auction_id - lot_id - id
- title - auction_id - lot_id
- location - title - url
- lots_count - current_bid - local_path
- closing_time - bid_count - downloaded=0
- closing_time
│
┌─────────────────────────────────────┴─────────────────────────────────────┐
│ PHASE 2: MONITORING & PROCESSING (Java) - THIS PROJECT │
└────────────────────────────────────────────────────────────────────────────┘
│
┌─────────────────┼─────────────────┐
▼ ▼ ▼
[TroostwijkMonitor] [DatabaseService] [ScraperDataAdapter]
│ │ │
│ Read lots │ Query lots │ Transform data
│ every hour │ Import images │ TEXT → INTEGER
│ │ │ "€123" → 123.0
└─────────────────┴─────────────────┘
│
┌─────────────────────────┼─────────────────────────┐
▼ ▼ ▼
[Bid Monitoring] [Image Processing] [Closing Alerts]
Check API every 1h Download images Check < 5 min
│ │ │
│ New bid? │ Process via │ Time critical?
├─[YES]──────────┐ │ ObjectDetection ├─[YES]────┐
│ │ │ │ │
▼ │ ▼ │ │
[Update current_bid] │ ┌──────────────────┐ │ │
in database │ │ YOLO Detection │ │ │
│ │ OpenCV DNN │ │ │
│ └──────────────────┘ │ │
│ │ │ │
│ │ Detect objects │ │
│ ├─[vehicle] │ │
│ ├─[furniture] │ │
│ ├─[machinery] │ │
│ │ │ │
│ ▼ │ │
│ [Save labels to DB] │ │
│ [Estimate value] │ │
│ │ │ │
│ │ │ │
└─────────┴───────────────────────┴──────────┘
│
┌───────────────────────────────────────────────┴────────────────────────────┐
│ PHASE 3: NOTIFICATION SYSTEM - USER INTERACTION TRIGGERS │
└────────────────────────────────────────────────────────────────────────────┘
│
┌─────────────────┴─────────────────┐
▼ ▼
[NotificationService] [User Decision Points]
│ │
┌───────────────────┼───────────────────┐ │
▼ ▼ ▼ │
[Desktop Notify] [Email Notify] [Priority Level] │
Windows/macOS/ Gmail SMTP 0=Normal │
Linux system (FREE) 1=High │
tray │
│ │ │ │
└───────────────────┴───────────────────┘ │
│ │
▼ ▼
┌──────────────────┐ ┌──────────────────┐
│ USER INTERACTION │ │ TRIGGER EVENTS: │
│ NOTIFICATIONS │ │ │
└──────────────────┘ └──────────────────┘
│ │
┌───────────────────┼───────────────────┐ │
▼ ▼ ▼ │
┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ │
│ 1. BID CHANGE │ │ 2. OBJECT │ │ 3. CLOSING │ │
│ │ │ DETECTED │ │ ALERT │ │
│ "Nieuw bod op │ │ │ │ │ │
│ kavel 12345: │ │ "Lot contains: │ │ "Kavel 12345 │ │
│ €150 (was €125)"│ │ - Vehicle │ │ sluit binnen │ │
│ │ │ - Machinery │ │ 5 min." │ │
│ Priority: NORMAL │ │ Est: €5000" │ │ Priority: HIGH │ │
│ │ │ │ │ │ │
│ Action needed: │ │ Action needed: │ │ Action needed: │ │
│ ▸ Place bid? │ │ ▸ Review item? │ │ ▸ Place final │ │
│ ▸ Monitor? │ │ ▸ Confirm value? │ │ bid? │ │
│ ▸ Ignore? │ │ ▸ Add to watch? │ │ ▸ Let expire? │ │
└──────────────────┘ └──────────────────┘ └──────────────────┘ │
│ │ │ │
└───────────────────┴───────────────────┴─────────────┘
│
▼
┌─────────────────────────────────────────────────────────────────────────────┐
│ USER ACTIONS & EXCEPTIONS │
├─────────────────────────────────────────────────────────────────────────────┤
│ Additional interaction points: │
│ │
│ 4. VIEWING DAY QUESTIONS │
│ "Bezichtiging op [date] - kunt u aanwezig zijn?" │
│ Action: ▸ Confirm attendance ▸ Request alternative ▸ Decline │
│ │
│ 5. ITEM RECOGNITION CONFIRMATION │
│ "Detected: [object] - Is deze correcte identificatie?" │
│ Action: ▸ Confirm ▸ Correct label ▸ Add notes │
│ │
│ 6. VALUE ESTIMATE APPROVAL │
│ "Geschatte waarde: €X - Akkoord?" │
│ Action: ▸ Accept ▸ Adjust ▸ Request manual review │
│ │
│ 7. EXCEPTION HANDLING │
│ "Afwijkende sluitingstijd / locatiewijziging / special terms" │
│ Action: ▸ Acknowledge ▸ Update preferences ▸ Withdraw interest │
└─────────────────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────────────────┐
│ OBJECT DETECTION & VALUE ESTIMATION PIPELINE │
└─────────────────────────────────────────────────────────────────────────────┘
[Downloaded Image] → [ImageProcessingService]
│ │
│ ▼
│ [ObjectDetectionService]
│ │
│ ├─ Load YOLO model
│ ├─ Run inference (416x416)
│ ├─ Post-process detections
│ │ (confidence > 0.5)
│ │
│ ▼
│ ┌──────────────────────┐
│ │ Detected Objects: │
│ │ - person │
│ │ - car │
│ │ - truck │
│ │ - furniture │
│ │ - machinery │
│ │ - electronics │
│ │ (80 COCO classes) │
│ └──────────────────────┘
│ │
│ ▼
│ [Value Estimation Logic]
│ (Future enhancement)
│ │
│ ├─ Match objects to auction categories
│ ├─ Historical price analysis
│ ├─ Condition assessment
│ ├─ Market trends
│ │
│ ▼
│ ┌──────────────────────┐
│ │ Estimated Value: │
│ │ €X - €Y range │
│ │ Confidence: 75% │
│ └──────────────────────┘
│ │
└──────────────────────┴─ [Save to DB]
│
▼
[Trigger notification if
value > threshold]
```
```mermaid
flowchart TD
subgraph P1["PHASE 1: EXTERNAL SCRAPER (Python/Playwright)"]
direction LR
A1[Listing Pages<br/>/auctions?page=N] --> A2[Extract URLs]
B1[Auction Pages<br/>/a/auction-id] --> B2[Parse __NEXT_DATA__ JSON]
C1[Lot Pages<br/>/l/lot-id] --> C2[Parse __NEXT_DATA__ JSON]
A2 --> D1[INSERT auctions to SQLite]
B2 --> D1
C2 --> D2[INSERT lots & image URLs]
D1 --> DB[(SQLite Database<br/>output/cache.db)]
D2 --> DB
end
DB --> P2_Entry
subgraph P2["PHASE 2: MONITORING & PROCESSING (Java)"]
direction TB
P2_Entry[Data Ready] --> Monitor[TroostwijkMonitor<br/>Read lots every hour]
P2_Entry --> DBService[DatabaseService<br/>Query & Import]
P2_Entry --> Adapter[ScraperDataAdapter<br/>Transform TEXT → INTEGER]
Monitor --> BM[Bid Monitoring<br/>Check API every 1h]
DBService --> IP[Image Processing<br/>Download & Analyze]
Adapter --> DataForNotify[Formatted Data]
BM --> BidUpdate{New bid?}
BidUpdate -->|Yes| UpdateDB[Update current_bid in DB]
UpdateDB --> NotifyTrigger1
IP --> Detection[Object Detection<br/>YOLO/OpenCV DNN]
Detection --> ObjectCheck{Detect objects?}
ObjectCheck -->|Vehicle| Save1[Save labels & estimate value]
ObjectCheck -->|Furniture| Save2[Save labels & estimate value]
ObjectCheck -->|Machinery| Save3[Save labels & estimate value]
Save1 --> NotifyTrigger2
Save2 --> NotifyTrigger2
Save3 --> NotifyTrigger2
CA[Closing Alerts<br/>Check < 5 min] --> TimeCheck{Time critical?}
TimeCheck -->|Yes| NotifyTrigger3
end
NotifyTrigger1 --> NS
NotifyTrigger2 --> NS
NotifyTrigger3 --> NS
subgraph P3["PHASE 3: NOTIFICATION SYSTEM"]
NS[NotificationService] --> DN[Desktop Notify<br/>Windows/macOS/Linux]
NS --> EN[Email Notify<br/>Gmail SMTP]
NS --> PL[Set Priority Level<br/>0=Normal, 1=High]
end
DN --> UI[User Interaction & Decisions]
EN --> UI
PL --> UI
subgraph UI_Details[User Decision Points / Trigger Events]
direction LR
E1["1. BID CHANGE<br/>'Nieuw bod op kavel 12345...'<br/>Actions: Place bid? Monitor? Ignore?"]
E2["2. OBJECT DETECTED<br/>'Lot contains: Vehicle...'<br/>Actions: Review? Confirm value?"]
E3["3. CLOSING ALERT<br/>'Kavel 12345 sluit binnen 5 min.'<br/>Actions: Place final bid? Let expire?"]
E4["4. VIEWING DAY QUESTIONS<br/>'Bezichtiging op [date]...'"]
E5["5. ITEM RECOGNITION CONFIRMATION<br/>'Detected: [object]...'"]
E6["6. VALUE ESTIMATE APPROVAL<br/>'Geschatte waarde: €X...'"]
E7["7. EXCEPTION HANDLING<br/>'Afwijkende sluitingstijd...'"]
end
UI --> UI_Details
%% Object Detection Sub-Flow Detail
subgraph P2_Detail["Object Detection & Value Estimation Pipeline"]
direction LR
DI[Downloaded Image] --> IPS[ImageProcessingService]
IPS --> ODS[ObjectDetectionService]
ODS --> Load[Load YOLO model]
ODS --> Run[Run inference]
ODS --> Post[Post-process detections<br/>confidence > 0.5]
Post --> ObjList["Detected Objects List<br/>(80 COCO classes)"]
ObjList --> VEL[Value Estimation Logic<br/>Future enhancement]
VEL --> Match[Match to categories]
VEL --> History[Historical price analysis]
VEL --> Condition[Condition assessment]
VEL --> Market[Market trends]
Market --> ValueEst["Estimated Value Range<br/>Confidence: 75%"]
ValueEst --> SaveToDB[Save to Database]
SaveToDB --> TriggerNotify{Value > threshold?}
end
IP -.-> P2_Detail
TriggerNotify -.-> NotifyTrigger2
```
## Integration Hooks & Timing
| Event | Frequency | Trigger | Notification Type | User Action Required |
|--------------------------------|-------------------|----------------------------|----------------------------|------------------------|
| **New auction discovered** | On scrape | Scraper finds new auction | Desktop + Email (optional) | Review auction |
| **Bid change detected** | Every 1 hour | Monitor detects higher bid | Desktop + Email | Place counter-bid? |
| **Closing soon (< 30 min)** | When detected | Time-based check | Desktop + Email | Review lot |
| **Closing imminent (< 5 min)** | When detected | Time-based check | Desktop + Email (HIGH) | Final bid decision |
| **Object detected** | On image process | YOLO finds objects | Desktop + Email | Confirm identification |
| **Value estimated** | After detection | Estimation complete | Desktop + Email | Approve estimate |
| **Viewing day scheduled** | From lot metadata | Scraper extracts date | Desktop + Email | Confirm attendance |
| **Exception/Change** | On update | Scraper detects change | Desktop + Email (HIGH) | Acknowledge |
## Project Structure
```
src/main/java/com/auction/
├── Main.java # Entry point
├── TroostwijkMonitor.java # Monitoring & orchestration
├── DatabaseService.java # SQLite operations
├── ScraperDataAdapter.java # Schema translation (TEXT→INT, €→float)
├── ImageProcessingService.java # Downloads & processes images
├── ObjectDetectionService.java # OpenCV YOLO detection
├── NotificationService.java # Desktop + Email notifications (FREE)
├── Lot.java # Domain model for auction lots
├── AuctionInfo.java # Domain model for auctions
└── Console.java # Logging utility
```
## Configuration
Edit `TroostwijkScraper.main()` to customize:
- **Database file**: `troostwijk.db` (SQLite database location)
- **YOLO paths**: Model configuration and weights files
- **Monitoring frequency**: Default is every 1 hour
- **Closing alerts**: Default is 5 minutes before closing
## Database Schema
The scraper creates three tables:
**sales**
- `sale_id` (PRIMARY KEY)
- `title`, `location`, `closing_time`
**lots**
- `lot_id` (PRIMARY KEY)
- `sale_id`, `title`, `description`, `manufacturer`, `type`, `year`
- `category`, `current_bid`, `currency`, `url`
- `closing_time`, `closing_notified`
**images**
- `id` (PRIMARY KEY)
- `lot_id`, `url`, `local_path`, `labels` (detected objects)
## Notification Examples
### Desktop Notification
![System Tray Notification]
```
🔔 Kavel bieding update
Nieuw bod op kavel 12345: €150.00 (was €125.00)
```
### Email Notification
```
From: your.email@gmail.com
To: your.email@gmail.com
Subject: [Troostwijk] Kavel bieding update
Nieuw bod op kavel 12345: €150.00 (was €125.00)
```
**High Priority Alerts** (closing soon):
```
⚠️ Lot nearing closure
Kavel 12345 sluit binnen 5 min.
```
## Why This Approach?
✅ **100% Free** - No paid services (Twilio, Pushover, etc.)
✅ **No External Dependencies** - Desktop notifications built into Java
✅ **Works Offline** - Desktop notifications don't need internet
✅ **Privacy First** - Your data stays on your machine
✅ **Cross-Platform** - Windows, macOS, Linux supported
✅ **Optional Email** - Add Gmail notifications if you want
## Troubleshooting
### Desktop Notifications Not Showing
- **Windows**: Check if Java has notification permissions
- **Linux**: Ensure you have a desktop environment running (not headless)
- **macOS**: Check System Preferences → Notifications
### Email Not Sending
1. Verify 2FA is enabled in Google Account
2. Confirm you're using an **App Password** (not your regular Gmail password)
3. Check that "Less secure app access" is NOT needed (app passwords work with 2FA)
4. Verify the SMTP format: `smtp:username:app_password:recipient`
## Notes
- Desktop notifications require a graphical environment (not headless servers)
- For headless servers, use email-only notifications
- Gmail SMTP is free and has generous limits (500 emails/day)
- OpenCV native libraries must match your platform architecture
- YOLO weights file is ~245 MB
```shell
ssh tour@athena.lan "docker run --rm -v shared-auction-data:/data -v /tmp:/tmp alpine cp /data/cache.db /tmp/cache.db" && scp tour@athena.lan:/tmp/cache.db c:/mnt/okcomputer/cache.db
```
## License
This is example code for educational purposes.

56
docker-compose.yml Normal file
View File

@@ -0,0 +1,56 @@
services:
auctiora:
user: "1000:1000"
build:
context: /opt/apps/auctiora
dockerfile: Dockerfile
container_name: auctiora
restart: unless-stopped
networks:
- traefik_net
environment:
# Database configuration
- AUCTION_DATABASE_PATH=/mnt/okcomputer/output/cache.db
- AUCTION_IMAGES_PATH=/mnt/okcomputer/output/images
# Notification configuration
# - AUCTION_NOTIFICATION_CONFIG=desktop
- AUCTION_NOTIFICATION_CONFIG=smtp:michael.bakker1986@gmail.com:agrepolhlnvhipkv:michael.bakker1986@gmail.com
# Quarkus configuration
- QUARKUS_HTTP_PORT=8081
- QUARKUS_HTTP_HOST=0.0.0.0
- QUARKUS_LOG_CONSOLE_LEVEL=INFO
# Scheduler configuration (cron expressions)
- AUCTION_WORKFLOW_SCRAPER_IMPORT_CRON=0 */30 * * * ?
- AUCTION_WORKFLOW_IMAGE_PROCESSING_CRON=0 0 * * * ?
- AUCTION_WORKFLOW_BID_MONITORING_CRON=0 */15 * * * ?
- AUCTION_WORKFLOW_CLOSING_ALERTS_CRON=0 */5 * * * ?
volumes:
# Mount database and images directory1
- shared-auction-data:/mnt/okcomputer/output
labels:
- "traefik.enable=true"
- "traefik.http.routers.auctiora.rule=Host(`auctiora.appmodel.nl`)"
- "traefik.http.routers.auctiora.entrypoints=websecure"
- "traefik.http.routers.auctiora.tls=true"
- "traefik.http.routers.auctiora.tls.certresolver=letsencrypt"
- "traefik.http.services.auctiora.loadbalancer.server.port=8081"
#healthcheck:
# test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8081/q/health/live"]
# interval: 30s
# timeout: 3s
# retries: 3
# start_period: 10s
networks:
traefik_net:
external: true
name: traefik_net
volumes:
shared-auction-data:
external: true

326
docs/ARCHITECTURE-TROOSTWIJK-SCRAPER.md Normal file
View File

@@ -0,0 +1,326 @@
# Troostwijk Scraper - Architecture & Data Flow
## System Overview
The scraper follows a **3-phase hierarchical crawling pattern** to extract auction and lot data from Troostwijk Auctions website.
## Architecture Diagram
```
┌─────────────────────────────────────────────────────────────────┐
│ TROOSTWIJK SCRAPER │
└─────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 1: COLLECT AUCTION URLs │
│ ┌──────────────┐ ┌──────────────┐ │
│ │ Listing Page │────────▶│ Extract /a/ │ │
│ │ /auctions? │ │ auction URLs │ │
│ │ page=1..N │ └──────────────┘ │
│ └──────────────┘ │ │
│ ▼ │
│ [ List of Auction URLs ] │
└─────────────────────────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 2: EXTRACT LOT URLs FROM AUCTIONS │
│ ┌──────────────┐ ┌──────────────┐ │
│ │ Auction Page │────────▶│ Parse │ │
│ │ /a/... │ │ __NEXT_DATA__│ │
│ └──────────────┘ │ JSON │ │
│ │ └──────────────┘ │
│ │ │ │
│ ▼ ▼ │
│ ┌──────────────┐ ┌──────────────┐ │
│ │ Save Auction │ │ Extract /l/ │ │
│ │ Metadata │ │ lot URLs │ │
│ │ to DB │ └──────────────┘ │
│ └──────────────┘ │ │
│ ▼ │
│ [ List of Lot URLs ] │
└─────────────────────────────────────────────────────────────────┘
│
▼
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 3: SCRAPE LOT DETAILS │
│ ┌──────────────┐ ┌──────────────┐ │
│ │ Lot Page │────────▶│ Parse │ │
│ │ /l/... │ │ __NEXT_DATA__│ │
│ └──────────────┘ │ JSON │ │
│ └──────────────┘ │
│ │ │
│ ┌─────────────────────────┴─────────────────┐ │
│ ▼ ▼ │
│ ┌──────────────┐ ┌──────────────┐ │
│ │ Save Lot │ │ Save Images │ │
│ │ Details │ │ URLs to DB │ │
│ │ to DB │ └──────────────┘ │
│ └──────────────┘ │ │
│ ▼ │
│ [Optional Download] │
└─────────────────────────────────────────────────────────────────┘
```
## Database Schema
```sql
┌──────────────────────────────────────────────────────────────────┐
│ CACHE TABLE (HTML Storage with Compression) │
├──────────────────────────────────────────────────────────────────┤
│ cache │
│ ├── url (TEXT, PRIMARY KEY) │
│ ├── content (BLOB) -- Compressed HTML (zlib) │
│ ├── timestamp (REAL) │
│ ├── status_code (INTEGER) │
│ └── compressed (INTEGER) -- 1=compressed, 0=plain │
└──────────────────────────────────────────────────────────────────┘
┌──────────────────────────────────────────────────────────────────┐
│ AUCTIONS TABLE │
├──────────────────────────────────────────────────────────────────┤
│ auctions │
│ ├── auction_id (TEXT, PRIMARY KEY) -- e.g. "A7-39813" │
│ ├── url (TEXT, UNIQUE) │
│ ├── title (TEXT) │
│ ├── location (TEXT) -- e.g. "Cluj-Napoca, RO" │
│ ├── lots_count (INTEGER) │
│ ├── first_lot_closing_time (TEXT) │
│ └── scraped_at (TEXT) │
└──────────────────────────────────────────────────────────────────┘
┌──────────────────────────────────────────────────────────────────┐
│ LOTS TABLE │
├──────────────────────────────────────────────────────────────────┤
│ lots │
│ ├── lot_id (TEXT, PRIMARY KEY) -- e.g. "A1-28505-5" │
│ ├── auction_id (TEXT) -- FK to auctions │
│ ├── url (TEXT, UNIQUE) │
│ ├── title (TEXT) │
│ ├── current_bid (TEXT) -- "€123.45" or "No bids" │
│ ├── bid_count (INTEGER) │
│ ├── closing_time (TEXT) │
│ ├── viewing_time (TEXT) │
│ ├── pickup_date (TEXT) │
│ ├── location (TEXT) -- e.g. "Dongen, NL" │
│ ├── description (TEXT) │
│ ├── category (TEXT) │
│ └── scraped_at (TEXT) │
│ FOREIGN KEY (auction_id) → auctions(auction_id) │
└──────────────────────────────────────────────────────────────────┘
┌──────────────────────────────────────────────────────────────────┐
│ IMAGES TABLE (Image URLs & Download Status) │
├──────────────────────────────────────────────────────────────────┤
│ images ◀── THIS TABLE HOLDS IMAGE LINKS│
│ ├── id (INTEGER, PRIMARY KEY AUTOINCREMENT) │
│ ├── lot_id (TEXT) -- FK to lots │
│ ├── url (TEXT) -- Image URL │
│ ├── local_path (TEXT) -- Path after download │
│ └── downloaded (INTEGER) -- 0=pending, 1=downloaded │
│ FOREIGN KEY (lot_id) → lots(lot_id) │
└──────────────────────────────────────────────────────────────────┘
```
## Sequence Diagram
```
User Scraper Playwright Cache DB Data Tables
│ │ │ │ │
│ Run │ │ │ │
├──────────────▶│ │ │ │
│ │ │ │ │
│ │ Phase 1: Listing Pages │ │
│ ├───────────────▶│ │ │
│ │ goto() │ │ │
│ │◀───────────────┤ │ │
│ │ HTML │ │ │
│ ├───────────────────────────────▶│ │
│ │ compress & cache │ │
│ │ │ │ │
│ │ Phase 2: Auction Pages │ │
│ ├───────────────▶│ │ │
│ │◀───────────────┤ │ │
│ │ HTML │ │ │
│ │ │ │ │
│ │ Parse __NEXT_DATA__ JSON │ │
│ │────────────────────────────────────────────────▶│
│ │ │ │ INSERT auctions
│ │ │ │ │
│ │ Phase 3: Lot Pages │ │
│ ├───────────────▶│ │ │
│ │◀───────────────┤ │ │
│ │ HTML │ │ │
│ │ │ │ │
│ │ Parse __NEXT_DATA__ JSON │ │
│ │────────────────────────────────────────────────▶│
│ │ │ │ INSERT lots │
│ │────────────────────────────────────────────────▶│
│ │ │ │ INSERT images│
│ │ │ │ │
│ │ Export to CSV/JSON │ │
│ │◀────────────────────────────────────────────────┤
│ │ Query all data │ │
│◀──────────────┤ │ │ │
│ Results │ │ │ │
```
## Data Flow Details
### 1. **Page Retrieval & Caching**
```
Request URL
│
├──▶ Check cache DB (with timestamp validation)
│ │
│ ├─[HIT]──▶ Decompress (if compressed=1)
│ │ └──▶ Return HTML
│ │
│ └─[MISS]─▶ Fetch via Playwright
│ │
│ ├──▶ Compress HTML (zlib level 9)
│ │ ~70-90% size reduction
│ │
│ └──▶ Store in cache DB (compressed=1)
│
└──▶ Return HTML for parsing
```
### 2. **JSON Parsing Strategy**
```
HTML Content
│
└──▶ Extract <script id="__NEXT_DATA__">
│
├──▶ Parse JSON
│ │
│ ├─[has pageProps.lot]──▶ Individual LOT
│ │ └──▶ Extract: title, bid, location, images, etc.
│ │
│ └─[has pageProps.auction]──▶ AUCTION
│ │
│ ├─[has lots[] array]──▶ Auction with lots
│ │ └──▶ Extract: title, location, lots_count
│ │
│ └─[no lots[] array]──▶ Old format lot
│ └──▶ Parse as lot
│
└──▶ Fallback to HTML regex parsing (if JSON fails)
```
### 3. **Image Handling**
```
Lot Page Parsed
│
├──▶ Extract images[] from JSON
│ │
│ └──▶ INSERT INTO images (lot_id, url, downloaded=0)
│
└──▶ [If DOWNLOAD_IMAGES=True]
│
├──▶ Download each image
│ │
│ ├──▶ Save to: /images/{lot_id}/001.jpg
│ │
│ └──▶ UPDATE images SET local_path=?, downloaded=1
│
└──▶ Rate limit between downloads (0.5s)
```
## Key Configuration
| Setting | Value | Purpose |
|---------|-------|---------|
| `CACHE_DB` | `/mnt/okcomputer/output/cache.db` | SQLite database path |
| `IMAGES_DIR` | `/mnt/okcomputer/output/images` | Downloaded images storage |
| `RATE_LIMIT_SECONDS` | `0.5` | Delay between requests |
| `DOWNLOAD_IMAGES` | `False` | Toggle image downloading |
| `MAX_PAGES` | `50` | Number of listing pages to crawl |
## Output Files
```
/mnt/okcomputer/output/
├── cache.db # SQLite database (compressed HTML + data)
├── auctions_{timestamp}.json # Exported auctions
├── auctions_{timestamp}.csv # Exported auctions
├── lots_{timestamp}.json # Exported lots
├── lots_{timestamp}.csv # Exported lots
└── images/ # Downloaded images (if enabled)
├── A1-28505-5/
│ ├── 001.jpg
│ └── 002.jpg
└── A1-28505-6/
└── 001.jpg
```
## Extension Points for Integration
### 1. **Downstream Processing Pipeline**
```python
# Query lots without downloaded images
SELECT lot_id, url FROM images WHERE downloaded = 0
# Process images: OCR, classification, etc.
# Update status when complete
UPDATE images SET downloaded = 1, local_path = ? WHERE id = ?
```
### 2. **Real-time Monitoring**
```python
# Check for new lots every N minutes
SELECT COUNT(*) FROM lots WHERE scraped_at > datetime('now', '-1 hour')
# Monitor bid changes
SELECT lot_id, current_bid, bid_count FROM lots WHERE bid_count > 0
```
### 3. **Analytics & Reporting**
```python
# Top locations
SELECT location, COUNT(*) as lot_count FROM lots GROUP BY location
# Auction statistics
SELECT
a.auction_id,
a.title,
COUNT(l.lot_id) as actual_lots,
SUM(CASE WHEN l.bid_count > 0 THEN 1 ELSE 0 END) as lots_with_bids
FROM auctions a
LEFT JOIN lots l ON a.auction_id = l.auction_id
GROUP BY a.auction_id
```
### 4. **Image Processing Integration**
```python
# Get all images for a lot
SELECT url, local_path FROM images WHERE lot_id = 'A1-28505-5'
# Batch process unprocessed images
SELECT i.id, i.lot_id, i.local_path, l.title, l.category
FROM images i
JOIN lots l ON i.lot_id = l.lot_id
WHERE i.downloaded = 1 AND i.local_path IS NOT NULL
```
## Performance Characteristics
- **Compression**: ~70-90% HTML size reduction (1GB → ~100-300MB)
- **Rate Limiting**: Exactly 0.5s between requests (respectful scraping)
- **Caching**: 24-hour default cache validity (configurable)
- **Throughput**: ~7,200 pages/hour (with 0.5s rate limit)
- **Scalability**: SQLite handles millions of rows efficiently
## Error Handling
- **Network failures**: Cached as status_code=500, retry after cache expiry
- **Parse failures**: Falls back to HTML regex patterns
- **Compression errors**: Auto-detects and handles uncompressed legacy data
- **Missing fields**: Defaults to "No bids", empty string, or 0
## Rate Limiting & Ethics
- **REQUIRED**: 0.5 second delay between ALL requests
- **Respects cache**: Avoids unnecessary re-fetching
- **User-Agent**: Identifies as standard browser
- **No parallelization**: Single-threaded sequential crawling

258
docs/DATABASE_ARCHITECTURE.md Normal file
View File

@@ -0,0 +1,258 @@
# Database Architecture
## Overview
The Auctiora auction monitoring system uses **SQLite** as its database engine, shared between the scraper process and the monitor application for simplicity and performance.
## Current State (Dec 2025)
- **Database**: `C:\mnt\okcomputer\output\cache.db`
- **Size**: 1.6 GB
- **Records**: 16,006 lots, 536,502 images
- **Concurrent Processes**: 2 (scraper + monitor)
- **Access Pattern**: Scraper writes, Monitor reads + occasional updates
## Why SQLite?
### ✅ Advantages for This Use Case
1. **Embedded Architecture**
- No separate database server to manage
- Zero network latency (local file access)
- Perfect for single-machine scraping + monitoring
2. **Excellent Read Performance**
- Monitor performs mostly SELECT queries
- Well-indexed access by `lot_id`, `url`, `auction_id`
- Sub-millisecond query times for simple lookups
3. **Simplicity**
- Single file database
- Automatic backup via file copy
- No connection pooling or authentication overhead
4. **Proven Scalability**
- Tested up to 281 TB database size
- 1.6 GB is only 0.0006% of capacity
- Handles billions of rows efficiently
5. **WAL Mode for Concurrency**
- Multiple readers don't block each other
- Readers don't block writers
- Writers don't block readers
- Perfect for scraper + monitor workload
## Configuration
### Connection String (DatabaseService.java:28)
```java
jdbc:sqlite:C:\mnt\okcomputer\output\cache.db?journal_mode=WAL&busy_timeout=10000
```
### Key PRAGMAs (DatabaseService.java:38-40)
```sql
PRAGMA journal_mode=WAL; -- Write-Ahead Logging for concurrency
PRAGMA busy_timeout=10000; -- 10s retry on lock contention
PRAGMA synchronous=NORMAL; -- Balance safety and performance
```
### What These Settings Do
| Setting | Purpose | Impact |
|---------|---------|--------|
| `journal_mode=WAL` | Write-Ahead Logging | Enables concurrent read/write access |
| `busy_timeout=10000` | Wait 10s on lock | Prevents immediate `SQLITE_BUSY` errors |
| `synchronous=NORMAL` | Balanced sync mode | Faster writes, still crash-safe |
## Schema Integration
### Scraper Schema (Read-Only for Monitor)
```sql
CREATE TABLE lots (
lot_id TEXT PRIMARY KEY,
auction_id TEXT,
url TEXT UNIQUE, -- ⚠️ Enforced by scraper
title TEXT,
current_bid TEXT,
closing_time TEXT,
manufacturer TEXT,
type TEXT,
year INTEGER,
currency TEXT DEFAULT 'EUR',
closing_notified INTEGER DEFAULT 0,
...
)
```
### Monitor Schema (Tables Created by Monitor)
```sql
CREATE TABLE images (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id INTEGER,
url TEXT,
local_path TEXT,
labels TEXT, -- Object detection results
processed_at INTEGER,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)
```
### Handling Schema Conflicts
**Problem**: Scraper has `UNIQUE` constraint on `lots.url`
**Solution** (DatabaseService.java:361-424):
```java
// Try UPDATE first
UPDATE lots SET ... WHERE lot_id = ?
// If no rows updated, INSERT OR IGNORE
INSERT OR IGNORE INTO lots (...) VALUES (...)
```
This approach:
- ✅ Updates existing lots by `lot_id`
- ✅ Skips inserts that violate UNIQUE constraints
- ✅ No crashes on re-imports or duplicate URLs
## Performance Characteristics
### Current Performance
- Simple SELECT by ID: <1ms
- Full table scan (16K lots): ~50ms
- Image INSERT: <5ms
- Concurrent operations: No blocking observed
### Scalability Projections
| Metric | Current | 1 Year | 3 Years | SQLite Limit |
|--------|---------|--------|---------|--------------|
| Lots | 16K | 365K | 1M | 1B+ rows |
| Images | 536K | 19M | 54M | 1B+ rows |
| DB Size | 1.6GB | 36GB | 100GB | 281TB |
| Queries | <1ms | <5ms | <20ms | Depends on indexes |
## When to Migrate to PostgreSQL/MySQL
### 🚨 Migration Triggers
Consider migrating if you encounter **any** of these:
1. **Concurrency Limits**
- >5 concurrent writers needed
- Frequent `SQLITE_BUSY` errors despite WAL mode
- Need for distributed access across multiple servers
2. **Performance Degradation**
- Database >50GB AND queries >1s for simple SELECTs
- Complex JOIN queries become bottleneck
- Index sizes exceed available RAM
3. **Operational Requirements**
- Need for replication (master/slave)
- Geographic distribution required
- High availability / failover needed
- Remote access from multiple locations
4. **Advanced Features**
- Full-text search on large text fields
- Complex analytical queries (window functions, CTEs)
- User management and fine-grained permissions
- Connection pooling for web applications
### Migration Path (If Needed)
1. **Choose Database**: PostgreSQL (recommended) or MySQL
2. **Schema Export**: Use SQLite `.schema` command
3. **Data Migration**: Use `sqlite3-to-postgres` or custom scripts
4. **Update Connection**: Change JDBC URL in `application.properties`
5. **Update Queries**: Fix SQL dialect differences
6. **Performance Tuning**: Create appropriate indexes
Example PostgreSQL configuration:
```properties
# application.properties
auction.database.url=jdbc:postgresql://localhost:5432/auctiora
auction.database.username=monitor
auction.database.password=${DB_PASSWORD}
```
## Current Recommendation: ✅ **Stick with SQLite**
### Rationale
1. **Sufficient Capacity**: 1.6GB is 0.0006% of SQLite's limit
2. **Excellent Performance**: Sub-millisecond queries
3. **Simple Operations**: No complex transactions or analytics
4. **Low Concurrency**: Only 2 processes (scraper + monitor)
5. **Local Architecture**: No need for network DB access
6. **Zero Maintenance**: No DB server to manage or monitor
### Monitoring Dashboard Metrics
Track these to know when to reconsider:
```sql
-- Add to praetium.html dashboard
SELECT
(SELECT COUNT(*) FROM lots) as lot_count,
(SELECT COUNT(*) FROM images) as image_count,
(SELECT page_count * page_size FROM pragma_page_count(), pragma_page_size()) as db_size_bytes,
(SELECT (page_count - freelist_count) * 100.0 / page_count FROM pragma_page_count(), pragma_freelist_count()) as db_utilization
```
**Review decision when**:
- Database >20GB
- Query times >500ms for simple lookups
- More than 3 concurrent processes needed
## Backup Strategy
### Recommended Approach
```bash
# Nightly backup via Windows Task Scheduler
sqlite3 C:\mnt\okcomputer\output\cache.db ".backup C:\backups\cache_$(date +%Y%m%d).db"
# Keep last 30 days
forfiles /P C:\backups /M cache_*.db /D -30 /C "cmd /c del @path"
```
### WAL File Management
SQLite creates additional files in WAL mode:
- `cache.db` - Main database
- `cache.db-wal` - Write-Ahead Log
- `cache.db-shm` - Shared memory
**Important**: Backup all three files together for consistency.
## Integration Points
### Scraper Process
- **Writes**: INSERT new lots, auctions, images
- **Schema Owner**: Creates tables, enforces constraints
- **Frequency**: Continuous (every 30 minutes)
### Monitor Process (Auctiora)
- **Reads**: SELECT lots, auctions for monitoring
- **Writes**: UPDATE bid amounts, notification flags; INSERT image processing results
- **Schema**: Adds `images` table for object detection
- **Frequency**: Every 15 seconds (dashboard refresh)
### Conflict Resolution
| Conflict | Strategy | Implementation |
|----------|----------|----------------|
| Duplicate lot_id | UPDATE instead of INSERT | DatabaseService.upsertLot() |
| Duplicate URL | INSERT OR IGNORE | Silent skip |
| Oversized IDs (>Long.MAX_VALUE) | Return 0L, skip import | ScraperDataAdapter.extractNumericId() |
| Invalid timestamps | Try-catch, log, continue | DatabaseService.getAllAuctions() |
| Database locked | 10s busy_timeout + WAL | Connection string |
## References
- [SQLite Documentation](https://www.sqlite.org/docs.html)
- [WAL Mode](https://www.sqlite.org/wal.html)
- [SQLite Limits](https://www.sqlite.org/limits.html)
- [When to Use SQLite](https://www.sqlite.org/whentouse.html)

109
docs/DATA_SYNC_SETUP.md Normal file
View File

@@ -0,0 +1,109 @@
# Production Data Sync Setup
Quick reference for syncing production data from `athena.lan` to your local development environment.
## 🚀 One-Command Setup
### Linux/Mac
```bash
./scripts/sync-production-data.sh
```
## 📋 Complete Usage
### Bash (Linux/Mac/Git Bash)
```bash
# Database only
./scripts/sync-production-data.sh --db-only
# Everything
./scripts/sync-production-data.sh --all
# Images only
./scripts/sync-production-data.sh --images-only
```
## 🔧 What It Does
1. **Connects to athena.lan** via SSH
2. **Copies database** from Docker volume to /tmp
3. **Downloads to local** machine (c:\mnt\okcomputer\cache.db)
4. **Backs up** existing local database automatically
5. **Shows statistics** (auction count, lot count, etc.)
6. **Cleans up** temporary files on remote server
### With Images
- Also syncs the `/data/images/` directory
- Uses rsync for incremental sync (if available)
- Can be large (several GB)
## 📊 What You Get
### Database (`cache.db`)
- **~8.9 GB** of production data
- 16,000+ lots
- 536,000+ images metadata
- Full auction history
- HTTP cache from scraper
### Images (`images/`)
- Downloaded lot images
- Organized by lot ID
- Variable size (can be large)
## ⚡ Quick Workflow
### Daily Development
```powershell
# Morning: Get fresh data
.\scripts\Sync-ProductionData.sh -Force
# Develop & test
mvn quarkus:dev
# View dashboard
start http://localhost:8080
```
## 🔒 Safety Features
- ✅ **Automatic backups** before overwriting
- ✅ **Confirmation prompts** (unless `-Force`)
- ✅ **Error handling** with clear messages
- ✅ **Cleanup** of temporary files
- ✅ **Non-destructive** - production data is never modified
## 🐛 Troubleshooting
### "Permission denied" or SSH errors
```bash
# Test SSH connection
ssh tour@athena.lan "echo OK"
# If fails, check your SSH key
ssh-add -l
```
### Database already exists
- Script automatically backs up existing database
- Backup format: `cache.db.backup-YYYYMMDD-HHMMSS`
### Slow image transfer
- Install rsync for 10x faster incremental sync
- Or sync database only: `.\scripts\Sync-ProductionData.sh` (default)
## 📚 Full Documentation
See [scripts/README.md](../scripts/README.md) for:
- Prerequisites
- Performance tips
- Automation setup
- Detailed troubleshooting
## 🎯 Common Use Cases
**Quick Links**:
- [Main README](../README.md)
- [Scripts Documentation](../scripts/README.md)
- [Integration Flowchart](INTEGRATION_FLOWCHART.md)
- [Intelligence Features](INTELLIGENCE_FEATURES_SUMMARY.md)

226
docs/EMAIL_CONFIGURATION.md Normal file
View File

@@ -0,0 +1,226 @@
# Email Notification Configuration Guide
## Overview
The application uses Gmail SMTP to send email notifications for auction alerts and lot updates.
## Gmail App Password Setup (Required for michael@appmodel.nl)
### Why App Passwords?
Google requires **App Passwords** instead of your regular Gmail password when using SMTP with 2-factor authentication enabled.
### Steps to Generate Gmail App Password:
1. **Enable 2-Factor Authentication** (if not already enabled)
- Go to https://myaccount.google.com/security
- Under "Signing in to Google", enable "2-Step Verification"
2. **Generate App Password**
- Go to https://myaccount.google.com/apppasswords
- Or navigate: Google Account → Security → 2-Step Verification → App passwords
- Select app: "Mail"
- Select device: "Other (Custom name)" → Enter "Auctiora Monitor"
- Click "Generate"
- Google will display a 16-character password (e.g., `abcd efgh ijkl mnop`)
- **Copy this password immediately** (you won't see it again)
3. **Use the App Password**
- Use this 16-character password (without spaces) in your configuration
- Format: `abcdefghijklmnop`
## Configuration
### Method 1: Environment Variable (Recommended for Production)
Set the `auction.notification.config` property in your `application.properties` or via environment variable:
```properties
# Format: smtp:username:password:recipient_email
auction.notification.config=smtp:michael@appmodel.nl:YOUR_APP_PASSWORD:michael@appmodel.nl
```
**Example with Docker:**
```bash
docker run -e AUCTION_NOTIFICATION_CONFIG="smtp:michael@appmodel.nl:abcdefghijklmnop:michael@appmodel.nl" ...
```
### Method 2: application.properties (Development)
Edit `src/main/resources/application.properties`:
```properties
# BEFORE (desktop only):
auction.notification.config=desktop
# AFTER (desktop + email):
auction.notification.config=smtp:michael@appmodel.nl:YOUR_APP_PASSWORD_HERE:michael@appmodel.nl
```
### Format Breakdown
The configuration string format is:
```
smtp:<SMTP_USERNAME>:<APP_PASSWORD>:<RECIPIENT_EMAIL>
```
Where:
- `SMTP_USERNAME`: Your Gmail address (michael@appmodel.nl)
- `APP_PASSWORD`: The 16-character app password from Google (no spaces)
- `RECIPIENT_EMAIL`: Email address to receive notifications (can be same as sender)
## Configuration Examples
### Desktop Notifications Only
```properties
auction.notification.config=desktop
```
### Email Notifications Only
```properties
auction.notification.config=smtp:michael@appmodel.nl:abcdefghijklmnop:michael@appmodel.nl
```
### Both Desktop and Email (Recommended)
The SMTP configuration automatically enables both:
```properties
auction.notification.config=smtp:michael@appmodel.nl:abcdefghijklmnop:michael@appmodel.nl
```
### Send to Multiple Recipients
To send to multiple recipients, you can modify the code or set up Gmail forwarding rules.
## SMTP Configuration Details
The application uses these Gmail SMTP settings (hardcoded):
- **Host**: smtp.gmail.com
- **Port**: 587
- **Security**: STARTTLS
- **Authentication**: Required
## Testing Configuration
After configuration, restart the application and check logs:
**Success:**
```
✓ OpenCV loaded successfully
Email notification: Test Alert
```
**Failure (wrong password):**
```
WARN NotificationService - Email failed: 535-5.7.8 Username and Password not accepted
```
## Troubleshooting
### Error: "Username and Password not accepted"
- **Cause**: Invalid App Password or 2FA not enabled
- **Solution**:
1. Verify 2-Factor Authentication is enabled
2. Generate a new App Password
3. Ensure no spaces in the password
4. Check for typos in email address
### Error: "AuthenticationFailedException"
- **Cause**: Incorrect credentials format
- **Solution**: Verify the format: `smtp:user:pass:recipient`
### Gmail Blocks Sign-in
- **Cause**: "Less secure app access" is disabled (deprecated by Google)
- **Solution**: Use App Passwords (as described above)
### Configuration Not Taking Effect
- **Cause**: Application not restarted or environment variable not set
- **Solution**:
1. Restart the application/container
2. Verify with: `docker logs auctiora | grep notification`
### SMTP Connection Timeout
- **Error**: `Couldn't connect to host, port: smtp.gmail.com, 587; timeout -1`
- **Causes**:
1. **Firewall/Network blocking port 587**
2. **Corporate network blocking SMTP**
3. **Antivirus/security software blocking connections**
4. **No internet access in test/container environment**
- **Solutions**:
1. **Test connectivity**:
```bash
# On Linux/Mac
telnet smtp.gmail.com 587
# On Windows
Test-NetConnection -ComputerName smtp.gmail.com -Port 587
```
2. **Check firewall rules**: Allow outbound connections to port 587
3. **Docker network**: Ensure container has internet access
```bash
docker exec auctiora ping -c 3 smtp.gmail.com
```
4. **Try alternative port 465** (SSL/TLS):
- Requires code change to use `mail.smtp.socketFactory`
5. **Corporate networks**: May require VPN or proxy configuration
6. **Windows Firewall**: Add Java/application to allowed programs
### Connection Succeeds but Authentication Fails
- **Error**: `Email authentication failed - check Gmail App Password`
- **Solution**: Verify App Password is correct and has no spaces
## Security Best Practices
1. **Never commit passwords to git**
- Use environment variables in production
- Add `application-local.properties` to `.gitignore`
2. **Rotate App Passwords periodically**
- Generate new App Password every 90 days
- Revoke old passwords at https://myaccount.google.com/apppasswords
3. **Use separate App Passwords per application**
- Creates "Auctiora Monitor" specific password
- Easy to revoke if compromised
4. **Monitor Gmail Activity**
- Check https://myaccount.google.com/notifications
- Review "Recent security activity"
## Example Docker Compose Configuration
```yaml
services:
auctiora:
image: auctiora:latest
environment:
- AUCTION_NOTIFICATION_CONFIG=smtp:michael@appmodel.nl:${GMAIL_APP_PASSWORD}:michael@appmodel.nl
- AUCTION_DATABASE_PATH=/mnt/okcomputer/output/cache.db
volumes:
- shared-auction-data:/mnt/okcomputer/output
```
Then set the password in `.env` file (not committed):
```bash
GMAIL_APP_PASSWORD=abcdefghijklmnop
```
## Notification Types
The application sends these email notifications:
1. **Lot Closing Soon** (Priority: High)
- Sent when a lot closes within 5 minutes
- Subject: `[Troostwijk] Lot nearing closure`
2. **Bid Updated** (Priority: Normal)
- Sent when current bid increases
- Subject: `[Troostwijk] Bid update`
3. **Critical Alerts** (Priority: High)
- System errors or important events
- Subject: `[Troostwijk] Critical Alert`
## Alternative: Desktop Notifications Only
If you don't want email notifications, use:
```properties
auction.notification.config=desktop
```
This will only show system tray notifications (Linux/Windows/Mac).

153
docs/EXPERT_ANALITICS.sql Normal file
View File

@@ -0,0 +1,153 @@
-- Extend 'lots' table
ALTER TABLE lots
ADD COLUMN starting_bid DECIMAL(12, 2);
ALTER TABLE lots
ADD COLUMN estimated_min DECIMAL(12, 2);
ALTER TABLE lots
ADD COLUMN estimated_max DECIMAL(12, 2);
ALTER TABLE lots
ADD COLUMN reserve_price DECIMAL(12, 2);
ALTER TABLE lots
ADD COLUMN reserve_met BOOLEAN DEFAULT FALSE;
ALTER TABLE lots
ADD COLUMN bid_increment DECIMAL(12, 2);
ALTER TABLE lots
ADD COLUMN watch_count INTEGER DEFAULT 0;
ALTER TABLE lots
ADD COLUMN view_count INTEGER DEFAULT 0;
ALTER TABLE lots
ADD COLUMN first_bid_time TEXT;
ALTER TABLE lots
ADD COLUMN last_bid_time TEXT;
ALTER TABLE lots
ADD COLUMN bid_velocity DECIMAL(5, 2);
-- bids per hour
-- New table: bid history (CRITICAL)
CREATE TABLE bid_history
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT REFERENCES lots (lot_id),
bid_amount DECIMAL(12, 2) NOT NULL,
bid_time TEXT NOT NULL,
is_winning BOOLEAN DEFAULT FALSE,
is_autobid BOOLEAN DEFAULT FALSE,
bidder_id TEXT, -- anonymized
created_at TEXT DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX idx_bid_history_lot_time ON bid_history (lot_id, bid_time);
-- Extend 'lots' table
ALTER TABLE lots
ADD COLUMN condition_score DECIMAL(3, 2); -- 0.00-10.00
ALTER TABLE lots
ADD COLUMN condition_description TEXT;
ALTER TABLE lots
ADD COLUMN year_manufactured INTEGER;
ALTER TABLE lots
ADD COLUMN serial_number TEXT;
ALTER TABLE lots
ADD COLUMN originality_score DECIMAL(3, 2); -- % original parts
ALTER TABLE lots
ADD COLUMN provenance TEXT;
ALTER TABLE lots
ADD COLUMN comparable_lot_ids TEXT;
-- JSON array
-- New table: comparable sales
CREATE TABLE comparable_sales
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT REFERENCES lots (lot_id),
comparable_lot_id TEXT,
similarity_score DECIMAL(3, 2), -- 0.00-1.00
price_difference_percent DECIMAL(5, 2),
created_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- New table: market indices
CREATE TABLE market_indices
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
category TEXT NOT NULL,
manufacturer TEXT,
avg_price DECIMAL(12, 2),
median_price DECIMAL(12, 2),
price_change_30d DECIMAL(5, 2),
volume_change_30d DECIMAL(5, 2),
calculated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- Extend 'auctions' table
ALTER TABLE auctions
ADD COLUMN auction_house TEXT;
ALTER TABLE auctions
ADD COLUMN auction_house_rating DECIMAL(3, 2);
ALTER TABLE auctions
ADD COLUMN buyers_premium_percent DECIMAL(5, 2);
ALTER TABLE auctions
ADD COLUMN payment_methods TEXT; -- JSON
ALTER TABLE auctions
ADD COLUMN shipping_cost_min DECIMAL(12, 2);
ALTER TABLE auctions
ADD COLUMN shipping_cost_max DECIMAL(12, 2);
ALTER TABLE auctions
ADD COLUMN seller_verified BOOLEAN DEFAULT FALSE;
-- New table: auction performance metrics
CREATE TABLE auction_metrics
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
auction_id TEXT REFERENCES auctions (auction_id),
sell_through_rate DECIMAL(5, 2),
avg_hammer_vs_estimate DECIMAL(5, 2),
total_hammer_price DECIMAL(15, 2),
total_starting_price DECIMAL(15, 2),
calculated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- New table: seasonal trends
CREATE TABLE seasonal_trends
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
category TEXT NOT NULL,
month INTEGER NOT NULL,
avg_price_multiplier DECIMAL(4, 2), -- vs annual avg
volume_multiplier DECIMAL(4, 2),
PRIMARY KEY (category, month)
);
-- New table: external market data
CREATE TABLE external_market_data
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
category TEXT NOT NULL,
manufacturer TEXT,
model TEXT,
dealer_avg_price DECIMAL(12, 2),
retail_avg_price DECIMAL(12, 2),
wholesale_avg_price DECIMAL(12, 2),
source TEXT,
fetched_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- New table: image analysis results
CREATE TABLE image_analysis
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
image_id INTEGER REFERENCES images (id),
damage_detected BOOLEAN,
damage_severity DECIMAL(3, 2),
wear_level TEXT CHECK (wear_level IN ('EXCELLENT', 'GOOD', 'FAIR', 'POOR')),
estimated_hours_used INTEGER,
ai_confidence DECIMAL(3, 2)
);
-- New table: economic indicators
CREATE TABLE economic_indicators
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
indicator_date TEXT NOT NULL,
currency TEXT NOT NULL,
exchange_rate DECIMAL(10, 4),
inflation_rate DECIMAL(5, 2),
market_volatility DECIMAL(5, 2)
);

38
docs/EXPERT_ANALITICS_PRIORITY.md Normal file
View File

@@ -0,0 +1,38 @@
```mermaid
graph TD
A[Add bid_history table] --> B[Add watch_count + estimates]
B --> C[Create market_indices]
C --> D[Add condition + year fields]
D --> E[Build comparable matching]
E --> F[Enrich with auction house data]
F --> G[Add AI image analysis]
```
| Current Practice | New Requirement | Why |
|-----------------------|---------------------------------|---------------------------|
| Scrape once per hour | **Scrape every bid update** | Capture velocity & timing |
| Save only current bid | **Save full bid history** | Detect patterns & sniping |
| Ignore watchers | **Track watch\_count** | Predict competition |
| Skip auction metadata | **Capture house estimates** | Anchor valuations |
| No historical data | **Store sold prices** | Train prediction models |
| Basic text scraping | **Parse condition/serial/year** | Enable comparables |
```bazaar
Week 1-2: Foundation
Implement bid_history scraping (most critical)
Add watch_count, starting_bid, estimated_min/max fields
Calculate basic bid_velocity
Week 3-4: Valuation
Extract year_manufactured, manufacturer, condition_description
Create market_indices (manually or via external API)
Build comparable lot matching logic
Week 5-6: Intelligence Layer
Add auction house performance tracking
Implement undervaluation detection algorithm
Create price alert system
Week 7-8: Automation
Integrate image analysis API
Add economic indicator tracking
Refine ML-based price predictions
```

126
docs/GraphQL.md Normal file
View File

@@ -0,0 +1,126 @@
# GraphQL Auction Schema Explorer
A Python script for exploring and testing GraphQL queries against the TBAuctions storefront API. This tool helps understand the auction schema by testing different query structures and viewing the responses.
## Features
- Three pre-configured GraphQL queries with varying levels of detail
- Asynchronous HTTP requests using aiohttp for efficient testing
- Error handling and formatted JSON output
- Configurable auction ID, locale, and platform parameters
## Prerequisites
- Python 3.7 or higher
- Required packages: `aiohttp`
## Installation
1. Clone or download this script
2. Install dependencies:
```bash
pip install aiohttp
```
## Usage
Run the script directly:
```bash
python auction_explorer.py
```
Or make it executable and run:
```bash
chmod +x auction_explorer.py
./auction_explorer.py
```
## Queries Included
The script tests three different query structures:
### 1. `viewingDays_simple`
Basic query that retrieves city and country code for viewing days.
### 2. `viewingDays_with_times`
Extended query that includes date ranges (`from` and `to`) along with city information.
### 3. `full_auction`
Comprehensive query that fetches:
- Auction ID and display ID
- Bidding status
- Buyer's premium
- Viewing days with location and timing
- Collection days with location and timing
## Configuration
Modify these variables in the script as needed:
```python
GRAPHQL_ENDPOINT = "https://storefront.tbauctions.com/storefront/graphql"
auction_id = "9d5d9d6b-94de-4147-b523-dfa512d85dfa" # Replace with your auction ID
variables = {
"auctionId": auction_id,
"locale": "nl", # Change locale as needed
"platform": "TWK" # Change platform as needed
}
```
## Output Format
The script outputs:
- Query name and separator
- Success status with formatted JSON response
- Or error messages if the query fails
Example output:
```
============================================================
QUERY: viewingDays_simple
============================================================
SUCCESS:
{
"data": {
"auction": {
"viewingDays": [
{
"city": "Amsterdam",
"countryCode": "NL"
}
]
}
}
}
```
## Customization
To add new queries, extend the `QUERIES` dictionary:
```python
QUERIES = {
"your_query_name": """
query YourQuery($auctionId: TbaUuid!, $locale: String!, $platform: Platform!) {
auction(id: $auctionId, locale: $locale, platform: $platform) {
# Your fields here
}
}
""",
# ... existing queries
}
```
## Notes
- The script includes a 500ms delay between queries to avoid rate limiting
- Timeout is set to 30 seconds per request
- All queries use the same GraphQL endpoint and variables
- Error responses are displayed in a readable format
## License
This script is provided for educational and exploratory purposes.

393
docs/INTEGRATION_FLOWCHART.md Normal file
View File

@@ -0,0 +1,393 @@
# Auctiora Intelligence Integration Flowchart
## Complete System Integration Diagram
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ COMPLETE SYSTEM INTEGRATION DIAGRAM │
└─────────────────────────────────────────────────────────────────────────────┘
┌──────────────────────────────────────────────────────────────────────────────┐
│ PHASE 1: EXTERNAL SCRAPER (Python/Playwright) - ARCHITECTURE-TROOSTWIJK │
└──────────────────────────────────────────────────────────────────────────────┘
│
┌─────────────────────────────┼─────────────────────────────┐
▼ ▼ ▼
[Listing Pages] [Auction Pages] [Lot Pages]
/auctions?page=N /a/auction-id /l/lot-id
│ │ │
│ Extract URLs │ Parse __NEXT_DATA__ │ Parse __NEXT_DATA__
├────────────────────────────▶│ JSON (GraphQL) │ JSON (GraphQL)
│ │ │
│ ▼ ▼
│ ┌────────────────┐ ┌────────────────┐
│ │ INSERT auctions│ │ INSERT lots │
│ │ to SQLite │ │ INSERT images │
│ └────────────────┘ │ (URLs only) │
│ │ └────────────────┘
│ │ │
└─────────────────────────────┴────────────────────────────┘
▼
┌──────────────────┐
│ SQLITE DATABASE │
│ output/cache.db │
└──────────────────┘
│
┌─────────────────┼─────────────────┐
▼ ▼ ▼
[auctions table] [lots table] [images table]
- auction_id - lot_id - id
- title - auction_id - lot_id
- location - title - url
- lots_count - current_bid - local_path
- closing_time - bid_count - downloaded=0
- closing_time
- followersCount ⭐ NEW
- estimatedMin ⭐ NEW
- estimatedMax ⭐ NEW
- nextBidStepInCents ⭐ NEW
- condition ⭐ NEW
- vat ⭐ NEW
- buyerPremiumPercentage ⭐ NEW
- quantity ⭐ NEW
- biddingStatus ⭐ NEW
- remarks ⭐ NEW
│
┌─────────────────────────────────────┴─────────────────────────────────────┐
│ PHASE 2: MONITORING & PROCESSING (Java) - THIS PROJECT │
└────────────────────────────────────────────────────────────────────────────┘
│
┌─────────────────┼─────────────────┐
▼ ▼ ▼
[TroostwijkMonitor] [DatabaseService] [ScraperDataAdapter]
│ │ │
│ Read lots │ Query lots │ Transform data
│ every hour │ Import images │ TEXT → INTEGER
│ │ │ "€123" → 123.0
└─────────────────┴─────────────────┘
│
┌─────────────────────────┼─────────────────────────┐
▼ ▼ ▼
[Bid Monitoring] [Image Processing] [Closing Alerts]
Check API every 1h Download images Check < 5 min
│ │ │
│ New bid? │ Process via │ Time critical?
├─[YES]──────────┐ │ ObjectDetection ├─[YES]────┐
│ │ │ │ │
▼ │ ▼ │ │
[Update current_bid] │ ┌──────────────────┐ │ │
in database │ │ YOLO Detection │ │ │
│ │ OpenCV DNN │ │ │
│ └──────────────────┘ │ │
│ │ │ │
│ │ Detect objects │ │
│ ├─[vehicle] │ │
│ ├─[furniture] │ │
│ ├─[machinery] │ │
│ │ │ │
│ ▼ │ │
│ [Save labels to DB] │ │
│ [Estimate value] │ │
│ │ │ │
│ │ │ │
└─────────┴───────────────────────┴──────────┘
│
┌───────────────────────────────────────────────┴────────────────────────────┐
│ PHASE 3: INTELLIGENCE LAYER ⭐ NEW - PREDICTIVE ANALYTICS │
└────────────────────────────────────────────────────────────────────────────┘
│
┌─────────────────┴─────────────────┐
▼ ▼
[Intelligence Engine] [Analytics Calculations]
│ │
┌───────────────────┼──────────────┐ │
▼ ▼ ▼ │
[Sleeper Detection] [Bargain Finder] [Popularity Tracker] │
High followers Price < estimate Watch count analysis │
Low current bid Opportunity Competition level │
│ │ │ │
│ │ │ │
└───────────────────┴──────────────┴───────────────────┘
│
┌─────────────────┴─────────────────┐
▼ ▼
[Total Cost Calculator] [Next Bid Calculator]
Current bid × (1 + VAT/100) Current bid + increment
× (1 + premium/100) (from API or calculated)
│ │
└─────────────────┬─────────────────┘
│
┌───────────────────────────────────────────────┴────────────────────────────┐
│ PHASE 4: NOTIFICATION SYSTEM - USER INTERACTION TRIGGERS │
└────────────────────────────────────────────────────────────────────────────┘
│
┌─────────────────┴─────────────────┐
▼ ▼
[NotificationService] [User Decision Points]
│ │
┌───────────────────┼───────────────────┐ │
▼ ▼ ▼ │
[Desktop Notify] [Email Notify] [Priority Level] │
Windows/macOS/ Gmail SMTP 0=Normal │
Linux system (FREE) 1=High │
tray │
│ │ │ │
└───────────────────┴───────────────────┘ │
│ │
▼ ▼
┌──────────────────┐ ┌──────────────────┐
│ USER INTERACTION │ │ TRIGGER EVENTS: │
│ NOTIFICATIONS │ │ │
└──────────────────┘ └──────────────────┘
│ │
┌───────────────────┼───────────────────┐ │
▼ ▼ ▼ │
┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ │
│ 1. BID CHANGE │ │ 2. OBJECT │ │ 3. CLOSING │ │
│ │ │ DETECTED │ │ ALERT │ │
│ "Nieuw bod op │ │ │ │ │ │
│ kavel 12345: │ │ "Lot contains: │ │ "Kavel 12345 │ │
│ €150 (was €125)"│ │ - Vehicle │ │ sluit binnen │ │
│ │ │ - Machinery │ │ 5 min." │ │
│ Priority: NORMAL │ │ Est: €5000" │ │ Priority: HIGH │ │
│ │ │ │ │ │ │
│ Action needed: │ │ Action needed: │ │ Action needed: │ │
│ ▸ Place bid? │ │ ▸ Review item? │ │ ▸ Place final │ │
│ ▸ Monitor? │ │ ▸ Confirm value? │ │ bid? │ │
│ ▸ Ignore? │ │ ▸ Add to watch? │ │ ▸ Let expire? │ │
└──────────────────┘ └──────────────────┘ └──────────────────┘ │
│ │ │ │
└───────────────────┴───────────────────┴─────────────┘
│
▼
┌─────────────────────────────────────────────────────────────────────────────┐
│ INTELLIGENCE NOTIFICATIONS ⭐ NEW │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ 4. SLEEPER LOT ALERT │
│ "Lot 12345: 25 watchers, only €50 bid - Opportunity!" │
│ Action: ▸ Place strategic bid ▸ Monitor competition ▸ Set alert │
│ │
│ 5. BARGAIN DETECTED │
│ "Lot 67890: Current €200, Estimate €400-€600 - Below estimate!" │
│ Action: ▸ Bid now ▸ Research comparable ▸ Add to watchlist │
│ │
│ 6. HIGH COMPETITION WARNING │
│ "Lot 11111: 75 watchers, bid velocity 5/hr - Strong competition" │
│ Action: ▸ Review strategy ▸ Set max bid ▸ Find alternatives │
│ │
│ 7. TOTAL COST NOTIFICATION │
│ "True cost: €500 bid + €105 VAT (21%) + €50 premium (10%) = €655" │
│ Action: ▸ Confirm budget ▸ Adjust bid ▸ Calculate logistics │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
## Intelligence Dashboard Flow
```mermaid
flowchart TD
subgraph P1["PHASE 1: DATA COLLECTION"]
A1[GraphQL API] --> A2[Scraper Extracts 15+ New Fields]
A2 --> A3[followersCount]
A2 --> A4[estimatedMin/Max]
A2 --> A5[nextBidStepInCents]
A2 --> A6[vat + buyerPremiumPercentage]
A2 --> A7[condition + biddingStatus]
A3 & A4 & A5 & A6 & A7 --> DB[(SQLite Database)]
end
DB --> P2_Entry
subgraph P2["PHASE 2: INTELLIGENCE PROCESSING"]
P2_Entry[Lot.java Model] --> Intelligence[Intelligence Methods]
Intelligence --> Sleeper[isSleeperLot<br/>High followers, low bid]
Intelligence --> Bargain[isBelowEstimate<br/>Price < estimate]
Intelligence --> Popular[getPopularityLevel<br/>Watch count tiers]
Intelligence --> Cost[calculateTotalCost<br/>Bid + VAT + Premium]
Intelligence --> NextBid[calculateNextBid<br/>API increment]
end
P2_Entry --> API_Layer
subgraph API["PHASE 3: REST API ENDPOINTS"]
API_Layer[AuctionMonitorResource] --> E1[/intelligence/sleepers]
API_Layer --> E2[/intelligence/bargains]
API_Layer --> E3[/intelligence/popular]
API_Layer --> E4[/intelligence/price-analysis]
API_Layer --> E5[/lots/:id/intelligence]
API_Layer --> E6[/charts/watch-distribution]
end
E1 & E2 & E3 & E4 & E5 & E6 --> Dashboard
subgraph UI["PHASE 4: INTELLIGENCE DASHBOARD"]
Dashboard[index.html] --> Widget1[Sleeper Lots Widget<br/>Opportunities]
Dashboard --> Widget2[Bargain Lots Widget<br/>Below Estimate]
Dashboard --> Widget3[Popular Lots Widget<br/>High Competition]
Dashboard --> Table[Enhanced Table<br/>Watchers | Est. Range | Total Cost]
Table --> Badges[Smart Badges:<br/>DEAL | Watch Count | Time Left]
end
Widget1 --> UserAction
Widget2 --> UserAction
Widget3 --> UserAction
Table --> UserAction
subgraph Actions["PHASE 5: USER ACTIONS"]
UserAction[User Decision] --> Bid[Place Strategic Bid]
UserAction --> Monitor[Add to Watchlist]
UserAction --> Research[Research Comparables]
UserAction --> Calculate[Budget Calculator]
end
```
## Key Intelligence Features
### 1. Follower/Watch Count Analytics
- **Data Source**: `followersCount` from GraphQL API
- **Intelligence Value**:
- Predict lot popularity before bidding wars
- Calculate interest-to-bid conversion rates
- Identify "sleeper" lots (high followers, low bids)
- Alert on sudden interest spikes
### 2. Price vs Estimate Analysis
- **Data Source**: `estimatedMin`, `estimatedMax` from GraphQL API
- **Intelligence Value**:
- Identify bargains: `currentBid < estimatedMin`
- Identify overvalued: `currentBid > estimatedMax`
- Build pricing models per category
- Track auction house estimate accuracy
### 3. True Cost Calculator
- **Data Source**: `vat`, `buyerPremiumPercentage` from GraphQL API
- **Intelligence Value**:
- Calculate total cost: `bid × (1 + VAT/100) × (1 + premium/100)`
- Budget planning with accurate all-in costs
- Compare true costs across lots
- Prevent bidding surprises
### 4. Exact Bid Increment
- **Data Source**: `nextBidStepInCents` from GraphQL API
- **Intelligence Value**:
- Show exact next bid amount
- No calculation errors
- Better UX for bidding recommendations
- Strategic bid placement
### 5. Structured Location & Category
- **Data Source**: `cityLocation`, `countryCode`, `categoryPath` from GraphQL API
- **Intelligence Value**:
- Filter by distance from user
- Calculate pickup logistics costs
- Category-based analytics
- Regional pricing trends
## Integration Hooks & Timing
| Event | Frequency | Trigger | Notification Type | User Action Required |
|--------------------------------|-------------------|----------------------------|----------------------------|------------------------|
| **Sleeper lot detected** | On data refresh | followers > 10, bid < €100 | Desktop + Email | Review opportunity |
| **Bargain detected** | On data refresh | bid < estimatedMin | Desktop + Email | Consider bidding |
| **High competition** | On data refresh | followers > 50 | Desktop | Review strategy |
| **Bid change detected** | Every 1 hour | Monitor detects higher bid | Desktop + Email | Place counter-bid? |
| **Closing soon (< 30 min)** | When detected | Time-based check | Desktop + Email | Review lot |
| **Closing imminent (< 5 min)** | When detected | Time-based check | Desktop + Email (HIGH) | Final bid decision |
| **Object detected** | On image process | YOLO finds objects | Desktop + Email | Confirm identification |
| **True cost calculated** | On page load | User views lot | Dashboard display | Budget confirmation |
## API Endpoints Reference
### Intelligence Endpoints
- `GET /api/monitor/intelligence/sleepers` - Returns high-interest, low-bid lots
- `GET /api/monitor/intelligence/bargains` - Returns lots priced below estimate
- `GET /api/monitor/intelligence/popular?level={HIGH|MEDIUM|LOW}` - Returns lots by popularity
- `GET /api/monitor/intelligence/price-analysis` - Returns price vs estimate statistics
- `GET /api/monitor/lots/{lotId}/intelligence` - Returns detailed intelligence for specific lot
### Chart Endpoints
- `GET /api/monitor/charts/watch-distribution` - Returns follower count distribution
- `GET /api/monitor/charts/country-distribution` - Returns geographic distribution
- `GET /api/monitor/charts/category-distribution` - Returns category distribution
- `GET /api/monitor/charts/bidding-trend?hours=24` - Returns time series data
## Dashboard Intelligence Widgets
### Sleeper Lots Widget
- **Color**: Purple gradient
- **Icon**: Eye (fa-eye)
- **Metric**: Count of lots with followers > 10 and bid < €100
- **Action**: Click to filter table to sleeper lots only
### Bargain Lots Widget
- **Color**: Green gradient
- **Icon**: Tag (fa-tag)
- **Metric**: Count of lots where current bid < estimated minimum
- **Action**: Click to filter table to bargain lots only
### Popular/Hot Lots Widget
- **Color**: Orange gradient
- **Icon**: Fire (fa-fire)
- **Metric**: Count of lots with followers > 20
- **Action**: Click to filter table to popular lots only
## Enhanced Table Features
### New Columns
1. **Watchers** - Shows follower count with color-coded badges:
- 50+ followers: Red (high competition)
- 21-50 followers: Orange (medium competition)
- 6-20 followers: Blue (some interest)
- 0-5 followers: Gray (minimal interest)
2. **Est. Range** - Shows auction house estimate: `€min-€max`
- Displays "DEAL" badge if current bid < estimate
3. **Total Cost** - Shows true cost including VAT and buyer premium:
- Hover tooltip shows breakdown: `Including VAT (21%) + Premium (10%)`
### Smart Indicators
- **DEAL Badge**: Green badge when `currentBid < estimatedMin`
- **Watch Count Badge**: Color-coded by competition level
- **Urgency Badge**: Time-based coloring (< 10 min = red)
## Technical Implementation
### Backend (Java)
- **File**: `src/main/java/auctiora/Lot.java`
- Added 24 new fields from GraphQL API
- Added 9 intelligence calculation methods
- Immutable record with Lombok `@With` annotation
- **File**: `src/main/java/auctiora/AuctionMonitorResource.java`
- Added 6 new REST API endpoints
- Enhanced insights with sleeper/bargain/popular detection
- Added watch distribution chart endpoint
### Frontend (HTML/JavaScript)
- **File**: `src/main/resources/META-INF/resources/index.html`
- Added 3 intelligence widgets with click handlers
- Enhanced closing soon table with 3 new columns
- Added `fetchIntelligenceData()` function
- Added smart badges and color coding
- Added total cost calculator display
## Future Enhancements
1. **Bid History Table** - Track bid changes over time
2. **Comparative Analytics** - Compare similar lots across auctions
3. **Machine Learning** - Predict final hammer price based on patterns
4. **Geographic Filtering** - Distance-based sorting and filtering
5. **Email Alerts** - Custom alerts for sleepers, bargains, etc.
6. **Mobile App** - Push notifications for time-critical events
7. **Bid Automation** - Auto-bid up to maximum with increment logic
---
**Last Updated**: December 2025
**Version**: 2.1
**Author**: Auctiora Intelligence Team

650
docs/QUARKUS_GUIDE.md Normal file
View File

@@ -0,0 +1,650 @@
# Quarkus Auction Monitor - Complete Guide
## 🚀 Overview
The Troostwijk Auction Monitor now runs on **Quarkus**, a Kubernetes-native Java framework optimized for fast startup and low memory footprint.
### Key Features
✅ **Quarkus Scheduler** - Built-in cron-based scheduling
✅ **REST API** - Control and monitor via HTTP endpoints
✅ **Health Checks** - Kubernetes-ready liveness/readiness probes
✅ **CDI/Dependency Injection** - Type-safe service management
✅ **Fast Startup** - 0.5s startup time
✅ **Low Memory** - ~50MB RSS memory footprint
✅ **Hot Reload** - Development mode with live coding
---
## 📦 Quick Start
### Option 1: Run with Maven (Development)
```bash
# Start in dev mode with live reload
mvn quarkus:dev
# Access application
# API: http://localhost:8081/api/monitor/status
# Health: http://localhost:8081/health
```
### Option 2: Build and Run JAR
```bash
# Build
mvn clean package
# Run
java -jar target/quarkus-app/quarkus-run.jar
# Or use fast-jar (recommended for production)
mvn clean package -Dquarkus.package.jar.type=fast-jar
java -jar target/quarkus-app/quarkus-run.jar
```
### Option 3: Docker
```bash
# Build image
docker build -t auction-monitor:latest .
# Run container
docker run -p 8081:8081 \
-v $(pwd)/data:/mnt/okcomputer/output \
auction-monitor:latest
```
### Option 4: Docker Compose (Recommended)
```bash
# Start services
docker-compose up -d
# View logs
docker-compose logs -f
# Stop services
docker-compose down
```
---
## 🔧 Configuration
### application.properties
All configuration is in `src/main/resources/application.properties`:
```properties
# Database
auction.database.path=C:\\mnt\\okcomputer\\output\\cache.db
auction.images.path=C:\\mnt\\okcomputer\\output\\images
# Notifications
auction.notification.config=desktop
# Or for email: smtp:your@gmail.com:app_password:recipient@example.com
# YOLO Models (optional)
auction.yolo.config=models/yolov4.cfg
auction.yolo.weights=models/yolov4.weights
auction.yolo.classes=models/coco.names
# Workflow Schedules (cron expressions)
auction.workflow.scraper-import.cron=0 */30 * * * ? # Every 30 min
auction.workflow.image-processing.cron=0 0 * * * ? # Every 1 hour
auction.workflow.bid-monitoring.cron=0 */15 * * * ? # Every 15 min
auction.workflow.closing-alerts.cron=0 */5 * * * ? # Every 5 min
# HTTP Server
quarkus.http.port=8081
quarkus.http.host=0.0.0.0
```
### Environment Variables
Override configuration with environment variables:
```bash
export AUCTION_DATABASE_PATH=/path/to/cache.db
export AUCTION_NOTIFICATION_CONFIG=desktop
export QUARKUS_HTTP_PORT=8081
```
---
## 📅 Scheduled Workflows
Quarkus automatically runs these workflows based on cron expressions:
| Workflow | Schedule | Cron Expression | Description |
|----------|----------|-----------------|-------------|
| **Scraper Import** | Every 30 min | `0 */30 * * * ?` | Import auctions/lots from external scraper |
| **Image Processing** | Every 1 hour | `0 0 * * * ?` | Download images & run object detection |
| **Bid Monitoring** | Every 15 min | `0 */15 * * * ?` | Check for bid changes |
| **Closing Alerts** | Every 5 min | `0 */5 * * * ?` | Send alerts for lots closing soon |
### Cron Expression Format
```
┌───────────── second (0-59)
│ ┌───────────── minute (0-59)
│ │ ┌───────────── hour (0-23)
│ │ │ ┌───────────── day of month (1-31)
│ │ │ │ ┌───────────── month (1-12)
│ │ │ │ │ ┌───────────── day of week (0-6, Sunday=0)
│ │ │ │ │ │
0 */30 * * * ? = Every 30 minutes
0 0 * * * ? = Every hour at minute 0
0 0 0 * * ? = Every day at midnight
```
---
## 🌐 REST API
### Base URL
```
http://localhost:8081/api/monitor
```
### Endpoints
#### 1. Get Status
```bash
GET /api/monitor/status
# Example
curl http://localhost:8081/api/monitor/status
# Response
{
"running": true,
"auctions": 25,
"lots": 150,
"images": 300,
"closingSoon": 5
}
```
#### 2. Get Statistics
```bash
GET /api/monitor/statistics
# Example
curl http://localhost:8081/api/monitor/statistics
# Response
{
"totalAuctions": 25,
"totalLots": 150,
"totalImages": 300,
"activeLots": 120,
"lotsWithBids": 80,
"totalBidValue": "€125,450.00",
"averageBid": "€1,568.13"
}
```
#### 3. Trigger Workflows Manually
```bash
# Scraper Import
POST /api/monitor/trigger/scraper-import
curl -X POST http://localhost:8081/api/monitor/trigger/scraper-import
# Image Processing
POST /api/monitor/trigger/image-processing
curl -X POST http://localhost:8081/api/monitor/trigger/image-processing
# Bid Monitoring
POST /api/monitor/trigger/bid-monitoring
curl -X POST http://localhost:8081/api/monitor/trigger/bid-monitoring
# Closing Alerts
POST /api/monitor/trigger/closing-alerts
curl -X POST http://localhost:8081/api/monitor/trigger/closing-alerts
```
#### 4. Get Auctions
```bash
# All auctions
GET /api/monitor/auctions
curl http://localhost:8081/api/monitor/auctions
# Filter by country
GET /api/monitor/auctions?country=NL
curl http://localhost:8081/api/monitor/auctions?country=NL
```
#### 5. Get Lots
```bash
# Active lots
GET /api/monitor/lots
curl http://localhost:8081/api/monitor/lots
# Lots closing soon (within 30 minutes by default)
GET /api/monitor/lots/closing-soon
curl http://localhost:8081/api/monitor/lots/closing-soon
# Custom minutes threshold
GET /api/monitor/lots/closing-soon?minutes=60
curl http://localhost:8081/api/monitor/lots/closing-soon?minutes=60
```
#### 6. Get Lot Images
```bash
GET /api/monitor/lots/{lotId}/images
# Example
curl http://localhost:8081/api/monitor/lots/12345/images
```
#### 7. Test Notification
```bash
POST /api/monitor/test-notification
Content-Type: application/json
{
"message": "Test message",
"title": "Test Title",
"priority": "0"
}
# Example
curl -X POST http://localhost:8081/api/monitor/test-notification \
-H "Content-Type: application/json" \
-d '{"message":"Test notification","title":"Test","priority":"0"}'
```
---
## 🏥 Health Checks
Quarkus provides built-in health checks for Kubernetes/Docker:
### Liveness Probe
```bash
GET /health/live
# Example
curl http://localhost:8081/health/live
# Response
{
"status": "UP",
"checks": [
{
"name": "Auction Monitor is alive",
"status": "UP"
}
]
}
```
### Readiness Probe
```bash
GET /health/ready
# Example
curl http://localhost:8081/health/ready
# Response
{
"status": "UP",
"checks": [
{
"name": "database",
"status": "UP",
"data": {
"auctions": 25
}
}
]
}
```
### Startup Probe
```bash
GET /health/started
# Example
curl http://localhost:8081/health/started
```
### Combined Health
```bash
GET /health
# Returns all health checks
curl http://localhost:8081/health
```
---
## 🐳 Docker Deployment
### Build Image
```bash
docker build -t auction-monitor:1.0 .
```
### Run Container
```bash
docker run -d \
--name auction-monitor \
-p 8081:8081 \
-v $(pwd)/data:/mnt/okcomputer/output \
-e AUCTION_NOTIFICATION_CONFIG=desktop \
auction-monitor:1.0
```
### Docker Compose
```yaml
version: '3.8'
services:
auction-monitor:
image: auction-monitor:1.0
ports:
- "8081:8081"
volumes:
- ./data:/mnt/okcomputer/output
environment:
- AUCTION_DATABASE_PATH=/mnt/okcomputer/output/cache.db
- AUCTION_NOTIFICATION_CONFIG=desktop
healthcheck:
test: ["CMD", "wget", "--spider", "http://localhost:8081/health/live"]
interval: 30s
timeout: 3s
retries: 3
```
---
## ☸️ Kubernetes Deployment
### deployment.yaml
```yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: auction-monitor
spec:
replicas: 1
selector:
matchLabels:
app: auction-monitor
template:
metadata:
labels:
app: auction-monitor
spec:
containers:
- name: auction-monitor
image: auction-monitor:1.0
ports:
- containerPort: 8081
env:
- name: AUCTION_DATABASE_PATH
value: /data/cache.db
- name: QUARKUS_HTTP_PORT
value: "8081"
volumeMounts:
- name: data
mountPath: /mnt/okcomputer/output
livenessProbe:
httpGet:
path: /health/live
port: 8081
initialDelaySeconds: 10
periodSeconds: 30
readinessProbe:
httpGet:
path: /health/ready
port: 8081
initialDelaySeconds: 5
periodSeconds: 10
startupProbe:
httpGet:
path: /health/started
port: 8081
failureThreshold: 30
periodSeconds: 10
volumes:
- name: data
persistentVolumeClaim:
claimName: auction-data-pvc
---
apiVersion: v1
kind: Service
metadata:
name: auction-monitor
spec:
selector:
app: auction-monitor
ports:
- port: 8081
targetPort: 8081
type: LoadBalancer
```
---
## 🔄 Development Mode
Quarkus dev mode provides live reload for rapid development:
```bash
# Start dev mode
mvn quarkus:dev
# Features available:
# - Live reload (no restart needed)
# - Dev UI: http://localhost:8081/q/dev/
# - Continuous testing
# - Debug on port 5005
```
### Dev UI
Access at: `http://localhost:8081/q/dev/`
Features:
- Configuration editor
- Scheduler dashboard
- Health checks
- REST endpoints explorer
- Continuous testing
---
## 🧪 Testing
### Run All Tests
```bash
mvn test
```
### Run Quarkus Tests
```bash
mvn test -Dtest=*QuarkusTest
```
### Integration Test with Running Application
```bash
# Terminal 1: Start application
mvn quarkus:dev
# Terminal 2: Run integration tests
curl http://localhost:8081/api/monitor/status
curl http://localhost:8081/health/live
curl -X POST http://localhost:8081/api/monitor/trigger/scraper-import
```
---
## 📊 Monitoring & Logging
### View Logs
```bash
# Docker
docker logs -f auction-monitor
# Docker Compose
docker-compose logs -f
# Kubernetes
kubectl logs -f deployment/auction-monitor
```
### Log Levels
Configure in `application.properties`:
```properties
# Production
quarkus.log.console.level=INFO
# Development
%dev.quarkus.log.console.level=DEBUG
# Specific logger
quarkus.log.category."com.auction".level=DEBUG
```
### Scheduled Job Logs
```
14:30:00 INFO [com.auc.Qua] (executor-thread-1) 📥 [WORKFLOW 1] Importing scraper data...
14:30:00 INFO [com.auc.Qua] (executor-thread-1) → Imported 5 auctions
14:30:00 INFO [com.auc.Qua] (executor-thread-1) → Imported 25 lots
14:30:00 INFO [com.auc.Qua] (executor-thread-1) ✓ Scraper import completed in 1250ms
```
---
## ⚙️ Performance
### Startup Time
- **JVM Mode**: ~0.5 seconds
- **Native Image**: ~0.014 seconds
### Memory Footprint
- **JVM Mode**: ~50MB RSS
- **Native Image**: ~15MB RSS
### Build Native Image (Optional)
```bash
# Requires GraalVM
mvn package -Pnative
# Run native executable
./target/troostwijk-scraper-1.0-SNAPSHOT-runner
```
---
## 🔐 Security
### Environment Variables for Secrets
```bash
# Don't commit credentials!
export AUCTION_NOTIFICATION_CONFIG=smtp:user@gmail.com:SECRET_PASSWORD:recipient@example.com
# Or use Kubernetes secrets
kubectl create secret generic auction-secrets \
--from-literal=notification-config='smtp:user@gmail.com:password:recipient@example.com'
```
### Kubernetes Secret
```yaml
apiVersion: v1
kind: Secret
metadata:
name: auction-secrets
type: Opaque
stringData:
notification-config: smtp:user@gmail.com:app_password:recipient@example.com
```
---
## 🛠️ Troubleshooting
### Issue: Schedulers not running
**Check scheduler status:**
```bash
curl http://localhost:8081/health/ready
```
**Enable debug logging:**
```properties
quarkus.log.category."io.quarkus.scheduler".level=DEBUG
```
### Issue: Database not found
**Check file permissions:**
```bash
ls -la C:/mnt/okcomputer/output/cache.db
```
**Create directory:**
```bash
mkdir -p C:/mnt/okcomputer/output
```
### Issue: Port 8081 already in use
**Change port:**
```bash
mvn quarkus:dev -Dquarkus.http.port=8082
# Or
export QUARKUS_HTTP_PORT=8082
```
### Issue: Health check failing
**Check application logs:**
```bash
docker logs auction-monitor
```
**Verify database connection:**
```bash
curl http://localhost:8081/health/ready
```
---
## 📚 Additional Resources
- [Quarkus Official Guide](https://quarkus.io/guides/)
- [Quarkus Scheduler](https://quarkus.io/guides/scheduler)
- [Quarkus REST](https://quarkus.io/guides/rest)
- [Quarkus Health](https://quarkus.io/guides/smallrye-health)
- [Quarkus Docker](https://quarkus.io/guides/container-image)
---
## Summary
✅ **Quarkus Framework** integrated for modern Java development
✅ **CDI/Dependency Injection** for clean architecture
✅ **@Scheduled** annotations for cron-based workflows
✅ **REST API** for control and monitoring
✅ **Health Checks** for Kubernetes/Docker
✅ **Fast Startup** and low memory footprint
✅ **Docker/Kubernetes** ready
✅ **Production** optimized
**Run and enjoy! 🎉**

209
docs/RATE_LIMITING.md Normal file
View File

@@ -0,0 +1,209 @@
# HTTP Rate Limiting
## Overview
The Troostwijk Scraper implements **per-host HTTP rate limiting** to prevent overloading external services (especially Troostwijk APIs) and avoid getting blocked.
## Features
- ✅ **Per-host rate limiting** - Different limits for different hosts
- ✅ **Token bucket algorithm** - Allows burst traffic while maintaining steady rate
- ✅ **Automatic host detection** - Extracts host from URL automatically
- ✅ **Request statistics** - Tracks success/failure/rate-limited requests
- ✅ **Thread-safe** - Uses semaphores for concurrent request handling
- ✅ **Configurable** - Via `application.properties`
## Configuration
Edit `src/main/resources/application.properties`:
```properties
# Default rate limit for all hosts (requests per second)
auction.http.rate-limit.default-max-rps=2
# Troostwijk-specific rate limit (requests per second)
auction.http.rate-limit.troostwijk-max-rps=1
# HTTP request timeout (seconds)
auction.http.timeout-seconds=30
```
### Recommended Settings
| Service | Max RPS | Reason |
|---------|---------|--------|
| `troostwijkauctions.com` | **1 req/s** | Prevent blocking by Troostwijk |
| Other image hosts | **2 req/s** | Balance speed and politeness |
## Usage
The `RateLimitedHttpClient` is automatically injected into services that make HTTP requests:
```java
@Inject
RateLimitedHttpClient httpClient;
// GET request for text
HttpResponse<String> response = httpClient.sendGet(url);
// GET request for binary data (images)
HttpResponse<byte[]> response = httpClient.sendGetBytes(imageUrl);
```
### Integrated Services
1. **TroostwijkMonitor** - API calls for bid monitoring
2. **ImageProcessingService** - Image downloads
3. **QuarkusWorkflowScheduler** - Scheduled workflows
## Monitoring
### REST API Endpoints
#### Get All Rate Limit Statistics
```bash
GET http://localhost:8081/api/monitor/rate-limit/stats
```
Response:
```json
{
"hosts": 2,
"statistics": {
"api.troostwijkauctions.com": {
"totalRequests": 150,
"successfulRequests": 148,
"failedRequests": 1,
"rateLimitedRequests": 0,
"averageDurationMs": 245
},
"images.troostwijkauctions.com": {
"totalRequests": 320,
"successfulRequests": 315,
"failedRequests": 5,
"rateLimitedRequests": 2,
"averageDurationMs": 892
}
}
}
```
#### Get Statistics for Specific Host
```bash
GET http://localhost:8081/api/monitor/rate-limit/stats/api.troostwijkauctions.com
```
Response:
```json
{
"host": "api.troostwijkauctions.com",
"totalRequests": 150,
"successfulRequests": 148,
"failedRequests": 1,
"rateLimitedRequests": 0,
"averageDurationMs": 245
}
```
## How It Works
### Token Bucket Algorithm
1. **Bucket initialization** - Starts with `maxRequestsPerSecond` tokens
2. **Request consumption** - Each request consumes 1 token
3. **Token refill** - Bucket refills every second
4. **Blocking** - If no tokens available, request waits
### Per-Host Rate Limiting
The client automatically:
1. Extracts hostname from URL (e.g., `api.troostwijkauctions.com`)
2. Creates/retrieves rate limiter for that host
3. Applies configured limit (Troostwijk-specific or default)
4. Tracks statistics per host
### Request Flow
```
Request → Extract Host → Get Rate Limiter → Acquire Token → Send Request → Record Stats
↓
troostwijkauctions.com?
↓
Yes: 1 req/s | No: 2 req/s
```
## Warning Signs
Monitor for these indicators of rate limiting issues:
| Metric | Warning Threshold | Action |
|--------|------------------|--------|
| `rateLimitedRequests` | > 0 | Server is rate limiting you - reduce `max-rps` |
| `failedRequests` | > 5% | Investigate connection issues or increase timeout |
| `averageDurationMs` | > 3000ms | Server may be slow - reduce load |
## Testing
### Manual Test via cURL
```bash
# Test Troostwijk API rate limiting
for i in {1..10}; do
echo "Request $i at $(date +%T)"
curl -s http://localhost:8081/api/monitor/status > /dev/null
sleep 0.5
done
# Check statistics
curl http://localhost:8081/api/monitor/rate-limit/stats | jq
```
### Check Logs
Rate limiting is logged at DEBUG level:
```
03:15:23 DEBUG [RateLimitedHttpClient] HTTP 200 GET api.troostwijkauctions.com (245ms)
03:15:24 DEBUG [RateLimitedHttpClient] HTTP 200 GET api.troostwijkauctions.com (251ms)
03:15:25 WARN [RateLimitedHttpClient] ⚠️ Rate limited by api.troostwijkauctions.com (HTTP 429)
```
## Troubleshooting
### Problem: Getting HTTP 429 (Too Many Requests)
**Solution:** Decrease `max-rps` for that host:
```properties
auction.http.rate-limit.troostwijk-max-rps=0.5
```
### Problem: Requests too slow
**Solution:** Increase `max-rps` (be careful not to get blocked):
```properties
auction.http.rate-limit.default-max-rps=3
```
### Problem: Requests timing out
**Solution:** Increase timeout:
```properties
auction.http.timeout-seconds=60
```
## Best Practices
1. **Start conservative** - Begin with low limits (1 req/s)
2. **Monitor statistics** - Watch `rateLimitedRequests` metric
3. **Respect robots.txt** - Check host's crawling policy
4. **Use off-peak hours** - Run heavy scraping during low-traffic times
5. **Implement exponential backoff** - If receiving 429s, wait longer between retries
## Future Enhancements
Potential improvements:
- [ ] Dynamic rate adjustment based on 429 responses
- [ ] Exponential backoff on failures
- [ ] Per-endpoint rate limiting (not just per-host)
- [ ] Request queue visualization
- [ ] Integration with external rate limit APIs (e.g., Redis)

304
docs/VALUATION.md Normal file
View File

@@ -0,0 +1,304 @@
# Auction Valuation Mathematics - Technical Reference
## 1. Fair Market Value (FMV) - Core Valuation Formula
The baseline valuation is calculated using a **weighted comparable sales approach**:
$$
FMV = \frac{\sum_{i=1}^{n} \left( P_i \cdot \omega_c \cdot \omega_t \cdot \omega_p \cdot \omega_h \right)}{\sum_{i=1}^{n} \left( \omega_c \cdot \omega_t \cdot \omega_p \cdot \omega_h \right)}
$$
**Variables:**
- $P_i$ = Final hammer price of comparable lot *i* (€)
- $\omega_c$ = **Condition weight**: $\exp(-\lambda_c \cdot |C_{target} - C_i|)$
- $\omega_t$ = **Time weight**: $\exp(-\lambda_t \cdot |T_{target} - T_i|)$
- $\omega_p$ = **Provenance weight**: $1 + \delta_p \cdot (P_{target} - P_i)$
- $\omega_h$ = **Historical weight**: $\left( \frac{1}{1 + e^{-kh \cdot (D_i - D_{median})}} \right)$
**Parameter Definitions:**
- $C \in [0, 10]$ = Condition score (10 = perfect)
- $T$ = Manufacturing year
- $P \in \{0,1\}$ = Provenance flag (1 = documented history)
- $D_i$ = Days since comparable sale
- $\lambda_c = 0.693$ = Condition decay constant (50% weight at 1-point difference)
- $\lambda_t = 0.048$ = Time decay constant (50% weight at 15-year difference)
- $\delta_p = 0.15$ = Provenance premium coefficient
- $kh = 0.01$ = Historical relevance coefficient
---
## 2. Condition Adjustment Multiplier
Normalizes prices across condition states:
$$
M_{cond} = \exp\left( \alpha_c \cdot \sqrt{C_{target}} - \beta_c \right)
$$
**Variables:**
- $\alpha_c = 0.15$ = Condition sensitivity parameter
- $\beta_c = 0.40$ = Baseline condition offset
- $C_{target}$ = Target lot condition score
**Interpretation:**
- $C = 10$ (mint): $M_{cond} = 1.48$ (48% premium over poor condition)
- $C = 5$ (average): $M_{cond} = 0.91$
---
## 3. Time-Based Depreciation Model
For equipment/machinery with measurable lifespan:
$$
V_{age} = V_{new} \cdot \left( 1 - \gamma \cdot \ln\left( 1 + \frac{Y_{current} - Y_{manu}}{Y_{expected}} \right) \right)
$$
**Variables:**
- $V_{new}$ = Original market value (€)
- $\gamma = 0.25$ = Depreciation aggressivity factor
- $Y_{current}$ = Current year
- $Y_{manu}$ = Manufacturing year
- $Y_{expected}$ = Expected useful life span (years)
**Example:** 10-year-old machinery with 25-year expected life retains 85% of value.
---
## 4. Provenance Premium Calculation
$$
\Delta_{prov} = V_{base} \cdot \left( \eta_0 + \eta_1 \cdot \ln(1 + N_{docs}) \right)
$$
**Variables:**
- $V_{base}$ = Base valuation without provenance (€)
- $N_{docs}$ = Number of verifiable provenance documents
- $\eta_0 = 0.08$ = Base provenance premium (8%)
- $\eta_1 = 0.035$ = Marginal document premium coefficient
---
## 5. Undervaluation Detection Score
Critical for identifying mispriced opportunities:
$$
U_{score} = \frac{FMV - P_{current}}{FMV} \cdot \sigma_{market} \cdot \left( 1 + \frac{B_{velocity}}{B_{threshold}} \right) \cdot \ln\left( 1 + \frac{W_{watch}}{W_{bid}} \right)
$$
**Variables:**
- $P_{current}$ = Current bid price (€)
- $\sigma_{market} \in [0,1]$ = Market volatility factor (from indices)
- $B_{velocity}$ = Bids per hour (bph)
- $B_{threshold} = 10$ bph = High-velocity threshold
- $W_{watch}$ = Watch count
- $W_{bid}$ = Bid count
**Trigger condition:** $U_{score} &gt; 0.25$ (25% undervaluation) with confidence &gt; 0.70
---
## 6. Bid Velocity Indicator (Competition Heat)
Measures real-time competitive intensity:
$$
\Lambda_b(t) = \frac{dB}{dt} \cdot \exp\left( -\lambda_{cool} \cdot (t - t_{last}) \right)
$$
**Variables:**
- $\frac{dB}{dt}$ = Bid frequency derivative (bids/minute)
- $\lambda_{cool} = 0.1$ = Cool-down decay constant
- $t_{last}$ = Timestamp of last bid (minutes)
**Interpretation:**
- $\Lambda_b &gt; 5$ = **Hot lot** (bidding war likely)
- $\Lambda_b &lt; 0.5$ = **Cold lot** (potential sleeper)
---
## 7. Final Price Prediction Model
Composite machine learning-style formula:
$$
\hat{P}_{final} = FMV \cdot \left( 1 + \epsilon_{bid} + \epsilon_{time} + \epsilon_{comp} \right)
$$
**Error Components:**
- **Bid momentum error**:
$$\epsilon_{bid} = \tanh\left( \phi_1 \cdot \Lambda_b - \phi_2 \cdot \frac{P_{current}}{FMV} \right)$$
- **Time-to-close error**:
$$\epsilon_{time} = \psi \cdot \exp\left( -\frac{t_{close}}{30} \right)$$
- **Competition error**:
$$\epsilon_{comp} = \rho \cdot \ln\left( 1 + \frac{W_{watch}}{50} \right)$$
**Parameters:**
- $\phi_1 = 0.15$, $\phi_2 = 0.10$ = Bid momentum coefficients
- $\psi = 0.20$ = Time pressure coefficient
- $\rho = 0.08$ = Competition coefficient
- $t_{close}$ = Minutes until close
**Confidence interval**:
$$
CI_{95\%} = \hat{P}_{final} \pm 1.96 \cdot \sigma_{residual}
$$
---
## 8. Bidding Strategy Recommendation Engine
Optimal max bid and timing:
$$
S_{max} =
\begin{cases}
FMV \cdot (1 - \theta_{agg}) & \text{if } U_{score} &gt; 0.20 \\
FMV \cdot (1 + \theta_{cons}) & \text{if } \Lambda_b &gt; 3 \\
\hat{P}_{final} - \delta_{margin} & \text{otherwise}
\end{cases}
$$
**Variables:**
- $\theta_{agg} = 0.10$ = Aggressive buyer discount target (10% below FMV)
- $\theta_{cons} = 0.05$ = Conservative buyer overbid tolerance
- $\delta_{margin} = €50$ = Minimum margin below predicted final
**Timing function**:
$$
t_{optimal} = t_{close} - \begin{cases}
5 \text{ min} & \text{if } \Lambda_b &lt; 1 \\
30 \text{ sec} & \text{if } \Lambda_b &gt; 5 \\
10 \text{ min} & \text{otherwise}
\end{cases}
$$
---
## Variable Reference Table
| Symbol | Variable | Unit | Data Source |
|--------|----------|------|-------------|
| $P_i$ | Comparable sale price | € | `bid_history.final` |
| $C$ | Condition score | [0,10] | Image analysis + text parsing |
| $T$ | Manufacturing year | Year | Lot description extraction |
| $W_{watch}$ | Number of watchers | Count | Page metadata |
| $\Lambda_b$ | Bid velocity | bids/min | `bid_history.timestamp` diff |
| $t_{close}$ | Time until close | Minutes | `lots.closing_time` - NOW() |
| $\sigma_{market}$ | Market volatility | [0,1] | `market_indices.price_change_30d` |
| $N_{docs}$ | Provenance documents | Count | PDF link analysis |
| $B_{velocity}$ | Bid acceleration | bph² | Second derivative of $\Lambda_b$ |
---
## Backend Implementation (Quarkus Pseudo-Code)
```java
@Inject
MLModelService mlModel;
public Valuation calculateFairMarketValue(Lot lot) {
List&lt;Comparable&gt; comparables = db.findComparables(lot, minSimilarity=0.75, limit=20);
double weightedSum = 0.0;
double weightSum = 0.0;
for (Comparable comp : comparables) {
double wc = Math.exp(-0.693 * Math.abs(lot.getConditionScore() - comp.getConditionScore()));
double wt = Math.exp(-0.048 * Math.abs(lot.getYear() - comp.getYear()));
double wp = 1 + 0.15 * (lot.hasProvenance() ? 1 : 0 - comp.hasProvenance() ? 1 : 0);
double weight = wc * wt * wp;
weightedSum += comp.getFinalPrice() * weight;
weightSum += weight;
}
double fm v = weightSum &gt; 0 ? weightedSum / weightSum : lot.getEstimatedMin();
// Apply condition multiplier
fm v *= Math.exp(0.15 * Math.sqrt(lot.getConditionScore()) - 0.40);
return new Valuation(fm v, calculateConfidence(comparables.size()));
}
public BiddingStrategy getBiddingStrategy(String lotId) {
var lot = db.getLot(lotId);
var bidHistory = db.getBidHistory(lotId);
var watchers = lot.getWatchCount();
// Analyze patterns
boolean isSnipeTarget = watchers &gt; 50 && bidHistory.size() &lt; 5;
boolean hasReserve = lot.getReservePrice() &gt; 0;
double bidVelocity = calculateBidVelocity(bidHistory);
// Strategy recommendation
String strategy = isSnipeTarget ? "SNIPING_DETECTED" :
(hasReserve && lot.getCurrentBid() &lt; lot.getReservePrice() * 0.9) ? "RESERVE_AVOID" :
bidVelocity &gt; 5.0 ? "AGGRESSIVE_COMPETITION" : "STANDARD";
return new BiddingStrategy(
strategy,
calculateRecommendedMax(lot),
isSnipeTarget ? "FINAL_30_SECONDS" : "FINAL_10_MINUTES",
getCompetitionLevel(watchers, bidHistory.size())
);
}
```
```sqlite
-- Core bidding intelligence
ALTER TABLE lots ADD COLUMN starting_bid DECIMAL(12,2);
ALTER TABLE lots ADD COLUMN estimated_min DECIMAL(12,2);
ALTER TABLE lots ADD COLUMN estimated_max DECIMAL(12,2);
ALTER TABLE lots ADD COLUMN reserve_price DECIMAL(12,2);
ALTER TABLE lots ADD COLUMN watch_count INTEGER DEFAULT 0;
ALTER TABLE lots ADD COLUMN first_bid_time TEXT;
ALTER TABLE lots ADD COLUMN last_bid_time TEXT;
ALTER TABLE lots ADD COLUMN bid_velocity DECIMAL(5,2);
-- Bid history (critical)
CREATE TABLE bid_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT REFERENCES lots(lot_id),
bid_amount DECIMAL(12,2) NOT NULL,
bid_time TEXT NOT NULL,
is_winning BOOLEAN DEFAULT FALSE,
is_autobid BOOLEAN DEFAULT FALSE,
bidder_id TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- Valuation support
ALTER TABLE lots ADD COLUMN condition_score DECIMAL(3,2);
ALTER TABLE lots ADD COLUMN year_manufactured INTEGER;
ALTER TABLE lots ADD COLUMN provenance TEXT;
CREATE TABLE comparable_sales (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT REFERENCES lots(lot_id),
comparable_lot_id TEXT,
similarity_score DECIMAL(3,2),
price_difference_percent DECIMAL(5,2)
);
CREATE TABLE market_indices (
category TEXT NOT NULL,
manufacturer TEXT,
avg_price DECIMAL(12,2),
price_change_30d DECIMAL(5,2),
PRIMARY KEY (category, manufacturer)
);
-- Alert system
CREATE TABLE price_alerts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT REFERENCES lots(lot_id),
alert_type TEXT CHECK(alert_type IN ('UNDervalued', 'ACCELERATING', 'RESERVE_IN_SIGHT')),
trigger_price DECIMAL(12,2),
is_triggered BOOLEAN DEFAULT FALSE
);
```

310
mvnw vendored Normal file
View File

@@ -0,0 +1,310 @@
#!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Maven Start Up Batch script
#
# Required ENV vars:
# ------------------
# JAVA_HOME - location of a JDK home dir
#
# Optional ENV vars
# -----------------
# M2_HOME - location of maven2's installed home dir
# MAVEN_OPTS - parameters passed to the Java VM when running Maven
# e.g. to debug Maven itself, use
# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
# ----------------------------------------------------------------------------
if [ -z "$MAVEN_SKIP_RC" ] ; then
if [ -f /etc/mavenrc ] ; then
. /etc/mavenrc
fi
if [ -f "$HOME/.mavenrc" ] ; then
. "$HOME/.mavenrc"
fi
fi
# OS specific support. $var _must_ be set to either true or false.
cygwin=false;
darwin=false;
mingw=false
case "`uname`" in
CYGWIN*) cygwin=true ;;
MINGW*) mingw=true;;
Darwin*) darwin=true
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html
if [ -z "$JAVA_HOME" ]; then
if [ -x "/usr/libexec/java_home" ]; then
export JAVA_HOME="`/usr/libexec/java_home`"
else
export JAVA_HOME="/Library/Java/Home"
fi
fi
;;
esac
if [ -z "$JAVA_HOME" ] ; then
if [ -r /etc/gentoo-release ] ; then
JAVA_HOME=`java-config --jre-home`
fi
fi
if [ -z "$M2_HOME" ] ; then
## resolve links - $0 may be a link to maven's home
PRG="$0"
# need this for relative symlinks
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG="`dirname "$PRG"`/$link"
fi
done
saveddir=`pwd`
M2_HOME=`dirname "$PRG"`/..
# make it fully qualified
M2_HOME=`cd "$M2_HOME" && pwd`
cd "$saveddir"
# echo Using m2 at $M2_HOME
fi
# For Cygwin, ensure paths are in UNIX format before anything is touched
if $cygwin ; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --unix "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
fi
# For Mingw, ensure paths are in UNIX format before anything is touched
if $mingw ; then
[ -n "$M2_HOME" ] &&
M2_HOME="`(cd "$M2_HOME"; pwd)`"
[ -n "$JAVA_HOME" ] &&
JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
fi
if [ -z "$JAVA_HOME" ]; then
javaExecutable="`which javac`"
if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
# readlink(1) is not available as standard on Solaris 10.
readLink=`which readlink`
if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
if $darwin ; then
javaHome="`dirname \"$javaExecutable\"`"
javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
else
javaExecutable="`readlink -f \"$javaExecutable\"`"
fi
javaHome="`dirname \"$javaExecutable\"`"
javaHome=`expr "$javaHome" : '\(.*\)/bin'`
JAVA_HOME="$javaHome"
export JAVA_HOME
fi
fi
fi
if [ -z "$JAVACMD" ] ; then
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
else
JAVACMD="`which java`"
fi
fi
if [ ! -x "$JAVACMD" ] ; then
echo "Error: JAVA_HOME is not defined correctly." >&2
echo " We cannot execute $JAVACMD" >&2
exit 1
fi
if [ -z "$JAVA_HOME" ] ; then
echo "Warning: JAVA_HOME environment variable is not set."
fi
CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
# traverses directory structure from process work directory to filesystem root
# first directory with .mvn subdirectory is considered project base directory
find_maven_basedir() {
if [ -z "$1" ]
then
echo "Path not specified to find_maven_basedir"
return 1
fi
basedir="$1"
wdir="$1"
while [ "$wdir" != '/' ] ; do
if [ -d "$wdir"/.mvn ] ; then
basedir=$wdir
break
fi
# workaround for JBEAP-8937 (on Solaris 10/Sparc)
if [ -d "${wdir}" ]; then
wdir=`cd "$wdir/.."; pwd`
fi
# end of workaround
done
echo "${basedir}"
}
# concatenates all lines of a file
concat_lines() {
if [ -f "$1" ]; then
echo "$(tr -s '\n' ' ' < "$1")"
fi
}
BASE_DIR=`find_maven_basedir "$(pwd)"`
if [ -z "$BASE_DIR" ]; then
exit 1;
fi
##########################################################################################
# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
# This allows using the maven wrapper in projects that prohibit checking in binary data.
##########################################################################################
if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found .mvn/wrapper/maven-wrapper.jar"
fi
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
fi
if [ -n "$MVNW_REPOURL" ]; then
jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
else
jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
fi
while IFS="=" read key value; do
case "$key" in (wrapperUrl) jarUrl="$value"; break ;;
esac
done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
if [ "$MVNW_VERBOSE" = true ]; then
echo "Downloading from: $jarUrl"
fi
wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
if $cygwin; then
wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"`
fi
if command -v wget > /dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found wget ... using wget"
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
wget "$jarUrl" -O "$wrapperJarPath"
else
wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath"
fi
elif command -v curl > /dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found curl ... using curl"
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
curl -o "$wrapperJarPath" "$jarUrl" -f
else
curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
fi
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Falling back to using Java to download"
fi
javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
# For Cygwin, switch paths to Windows format before running javac
if $cygwin; then
javaClass=`cygpath --path --windows "$javaClass"`
fi
if [ -e "$javaClass" ]; then
if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Compiling MavenWrapperDownloader.java ..."
fi
# Compiling the Java class
("$JAVA_HOME/bin/javac" "$javaClass")
fi
if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
# Running the downloader
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Running MavenWrapperDownloader.java ..."
fi
("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
fi
fi
fi
fi
##########################################################################################
# End of extension
##########################################################################################
export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
if [ "$MVNW_VERBOSE" = true ]; then
echo $MAVEN_PROJECTBASEDIR
fi
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
# For Cygwin, switch paths to Windows format before running java
if $cygwin; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --path --windows "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
[ -n "$MAVEN_PROJECTBASEDIR" ] &&
MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
fi
# Provide a "standardized" way to retrieve the CLI args that will
# work with both Windows and non-Windows executions.
MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
export MAVEN_CMD_LINE_ARGS
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
exec "$JAVACMD" \
$MAVEN_OPTS \
-classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
"-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"

187
mvnw.cmd vendored Normal file
View File

@@ -0,0 +1,187 @@
@REM ----------------------------------------------------------------------------
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing,
@REM software distributed under the License is distributed on an
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@REM KIND, either express or implied. See the License for the
@REM specific language governing permissions and limitations
@REM under the License.
@REM ----------------------------------------------------------------------------
@REM ----------------------------------------------------------------------------
@REM Apache Maven Wrapper startup batch script, version 3.3.4
@REM
@REM Required ENV vars:
@REM JAVA_HOME - location of a JDK home dir
@REM
@REM Optional ENV vars
@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
@REM e.g. to debug Maven itself, use
@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
@REM ----------------------------------------------------------------------------
@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
@echo off
@REM set title of command window
title %0
@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
@REM set %HOME% to equivalent of $HOME
if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
@REM Execute a user defined script before this one
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
@REM check for pre script, once with legacy .bat ending and once with .cmd ending
if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %*
if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %*
:skipRcPre
@setlocal
set ERROR_CODE=0
@REM To isolate internal variables from possible post scripts, we use another setlocal
@setlocal
@REM ==== START VALIDATION ====
if not "%JAVA_HOME%" == "" goto OkJHome
echo.
echo Error: JAVA_HOME not found in your environment. >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
:OkJHome
if exist "%JAVA_HOME%\bin\java.exe" goto init
echo.
echo Error: JAVA_HOME is set to an invalid directory. >&2
echo JAVA_HOME = "%JAVA_HOME%" >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
@REM ==== END VALIDATION ====
:init
@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
@REM Fallback to current working directory if not found.
set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
set EXEC_DIR=%CD%
set WDIR=%EXEC_DIR%
:findBaseDir
IF EXIST "%WDIR%"\.mvn goto baseDirFound
cd ..
IF "%WDIR%"=="%CD%" goto baseDirNotFound
set WDIR=%CD%
goto findBaseDir
:baseDirNotFound
set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
cd "%EXEC_DIR%"
goto endDetectBaseDir
:baseDirFound
set MAVEN_PROJECTBASEDIR=%WDIR%
cd "%EXEC_DIR%"
:endDetectBaseDir
IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
@setlocal EnableExtensions EnableDelayedExpansion
for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG=!JVM_CONFIG! %%a
@endlocal & set JVM_CONFIG=%JVM_CONFIG%
:endReadAdditionalConfig
SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar"
FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B
)
@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
if exist %WRAPPER_JAR% (
if "%MVNW_VERBOSE%" == "true" (
echo Found %WRAPPER_JAR%
)
) else (
if not "%MVNW_REPOURL%" == "" (
SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar"
)
if "%MVNW_VERBOSE%" == "true" (
echo Couldn't find %WRAPPER_JAR%, downloading it ...
echo Downloading from: %WRAPPER_URL%
)
powershell -Command "&{"^
"$webclient = new-object System.Net.WebClient;"^
"if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
"$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
"}"^
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^
"}"
if "%MVNW_VERBOSE%" == "true" (
echo Finished downloading %WRAPPER_JAR%
)
)
@REM End of extension
%MAVEN_JAVA_EXE% ^
%JVM_CONFIG% ^
--add-opens=java.base/java.lang=ALL-UNNAMED ^
--add-opens=java.base/java.util=ALL-UNNAMED ^
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED ^
--add-opens=java.base/java.net=ALL-UNNAMED ^
--add-opens=java.base/java.io=ALL-UNNAMED ^
%MAVEN_OPTS% ^
%MAVEN_DEBUG_OPTS% ^
-classpath %WRAPPER_JAR% ^
"-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^
%WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
if ERRORLEVEL 1 goto error
goto end
:error
set ERROR_CODE=1
:end
@endlocal & set ERROR_CODE=%ERROR_CODE%
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
@REM check for post script, once with legacy .bat ending and once with .cmd ending
if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat"
if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd"
:skipRcPost
@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
if "%MAVEN_BATCH_PAUSE%" == "on" pause
if "%MAVEN_VERBOSE%" == "on" echo %ERROR_CODE%
exit /B %ERROR_CODE%

464
pom.xml Normal file
View File

@@ -0,0 +1,464 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>auctiora</groupId>
<artifactId>auctiora</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Troostwijk Auction Scraper</name>
<description>Web scraper for Troostwijk Auctions with object detection and notifications</description>
<repositories>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
</repositories>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>25</maven.compiler.source>
<maven.compiler.target>25</maven.compiler.target>
<maven.compiler.release>25</maven.compiler.release>
<jackson.version>2.17.0</jackson.version>
<opencv.version>4.9.0-0</opencv.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<quarkus.platform.version>3.17.7</quarkus.platform.version>
<asm.version>9.8</asm.version>
<lombok.version>1.18.40</lombok.version>
<!--this is not a bug, its feature -->
<lombok-version>${lombok.version}</lombok-version>
<lombok-maven-version>1.18.20.0</lombok-maven-version>
<maven-compiler-plugin-version>3.14.0</maven-compiler-plugin-version>
<versions-maven-plugin.version>2.19.0</versions-maven-plugin.version>
<jandex-maven-plugin-version>3.5.0</jandex-maven-plugin-version>
<jdbi.version>3.47.0</jdbi.version>
<maven.compiler.args>
--enable-native-access=ALL-UNNAMED
--add-opens java.base/sun.misc=ALL-UNNAMED
-Xdiags:verbose
-Xlint:all
</maven.compiler.args>
<uberJar>true</uberJar> <!-- Your existing properties... -->
<quarkus.package.jar.type>uber-jar</quarkus.package.jar.type>
<quarkus.package.jar.enabled>true</quarkus.package.jar.enabled>
<maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>io.quarkus.platform</groupId>
<artifactId>quarkus-bom</artifactId>
<version>${quarkus.platform.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- Override ASM to support Java 25 -->
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm-commons</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm-tree</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm-util</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>4.1.124.Final</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- Force consistent versions -->
<dependency>
<groupId>org.opentest4j</groupId>
<artifactId>opentest4j</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.9</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.github.vladimir-bukhtoyarov.bucket4j</groupId>
<artifactId>bucket4j-core</artifactId>
<version>7.6.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.github.bucket4j/bucket4j -->
<!--<dependency>
<groupId>io.github.bucket4j</groupId>
<artifactId>bucket4j</artifactId>
<version>8.9.0</version>
</dependency>-->
<!-- JSoup for HTML parsing and HTTP client -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.17.2</version>
</dependency>
<!-- Jackson for JSON parsing -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<!-- SQLite JDBC driver -->
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
<version>3.45.1.0</version>
</dependency>
<!-- Lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven</artifactId>
<version>${lombok-maven-version}</version>
<type>pom</type>
</dependency>
<!-- JavaMail API for email notifications -->
<dependency>
<groupId>com.sun.mail</groupId>
<artifactId>javax.mail</artifactId>
<version>1.6.2</version>
</dependency>
<dependency>
<groupId>com.microsoft.playwright</groupId>
<artifactId>playwright</artifactId>
<version>1.40.0</version>
</dependency>
<!-- SLF4J API and implementation for logging -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.9</version>
</dependency>
<!-- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>2.0.9</version>
</dependency>-->
<!-- JUnit 5 for testing -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.10.1</version>
<scope>test</scope>
</dependency>
<!-- Mockito for mocking in tests -->
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5-mockito</artifactId>
<version>3.30.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>5.8.0</version>
<scope>test</scope>
</dependency>
<!-- Mockito JUnit Jupiter integration -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<version>5.8.0</version>
<scope>test</scope>
</dependency>
<!-- JDBI3 - Lightweight ORM for SQL -->
<dependency>
<groupId>org.jdbi</groupId>
<artifactId>jdbi3-core</artifactId>
<version>${jdbi.version}</version>
</dependency>
<dependency>
<groupId>org.jdbi</groupId>
<artifactId>jdbi3-sqlobject</artifactId>
<version>${jdbi.version}</version>
</dependency>
<!-- AssertJ for fluent assertions (optional but recommended) -->
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.24.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.vladsch.flexmark</groupId>
<artifactId>flexmark-all</artifactId>
<version>0.64.8</version>
</dependency>
<!-- Quarkus Core Dependencies -->
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-rest-jackson</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-arc</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-scheduler</artifactId>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Explicitly add cron-utils with slf4j excluded to avoid path warning -->
<dependency>
<groupId>com.cronutils</groupId>
<artifactId>cron-utils</artifactId>
<version>9.2.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Force Netty 4.1.124.Final to avoid sun.misc.Unsafe warnings -->
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http2</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver-dns</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-smallrye-health</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-config-yaml</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-undertow</artifactId>
</dependency>
<!-- OSGi annotations -->
<dependency>
<groupId>org.osgi</groupId>
<artifactId>org.osgi.annotation.bundle</artifactId>
<version>2.0.0</version>
<scope>provided</scope>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.rest-assured</groupId>
<artifactId>rest-assured</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.openpnp</groupId>
<artifactId>opencv</artifactId>
<version>4.9.0-0</version>
<!--<classifier>windows-x86_64</classifier>-->
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>io.quarkus.platform</groupId>
<artifactId>quarkus-maven-plugin</artifactId>
<version>${quarkus.platform.version}</version>
<extensions>true</extensions>
<executions>
<execution>
<goals>
<goal>build</goal>
<goal>generate-code</goal>
<goal>generate-code-tests</goal>
</goals>
</execution>
</executions>
<configuration>
<properties>
<build.timestamp>${maven.build.timestamp}</build.timestamp>
</properties>
<jvmArgs>--enable-native-access=ALL-UNNAMED --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true</jvmArgs>
</configuration>
</plugin>
<plugin>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven-plugin</artifactId>
<version>${lombok-maven-version}</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>delombok</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Maven Exec Plugin for running with native access -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin-version}</version>
<configuration>
<release>${maven.compiler.release}</release>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok-version}</version>
</path>
<path>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-extension-processor</artifactId>
<version>${quarkus.platform.version}</version>
</path>
</annotationProcessorPaths>
<compilerArgs>
<arg>-Xdiags:verbose</arg>
<arg>-Xlint:all</arg>
</compilerArgs>
<fork>true</fork>
<excludes>
<exclude>module-info.java</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
<version>${versions-maven-plugin.version}</version>
</plugin>
<!-- Maven Surefire Plugin for tests with native access -->
<plugin>
<groupId>io.smallrye</groupId>
<artifactId>jandex-maven-plugin</artifactId>
<version>${jandex-maven-plugin-version}</version>
<executions>
<execution>
<id>make-index</id>
<goals>
<goal>jandex</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.2.5</version>
<configuration>
<!-- Enable ByteBuddy experimental mode for Java 25 support -->
<!-- Mockito requires this for Java 24+ -->
<argLine>
--enable-native-access=ALL-UNNAMED
--add-opens java.base/sun.misc=ALL-UNNAMED
-Dnet.bytebuddy.experimental=true
--add-opens java.base/java.lang=ALL-UNNAMED
--add-opens java.base/java.util=ALL-UNNAMED
</argLine>
</configuration>
</plugin>
</plugins>
</build>
<!-- In your pom.xml, alongside <build> and <dependencies> -->
<distributionManagement>
<repository>
<id>gitea</id>
<url>https://git.appmodel.nl/api/packages/Tour/maven</url>
</repository>
<snapshotRepository>
<id>gitea</id>
<url>https://git.appmodel.nl/api/packages/Tour/maven</url>
</snapshotRepository>
</distributionManagement>
</project>

33
scripts/BFG.ps1 Normal file
View File

@@ -0,0 +1,33 @@
# BFG.ps1 (run from C:\vibe\auctiora\scripts)
$ErrorActionPreference = "Stop"
# 1) Download BFG jar once, next to this script
$bfgJar = Join-Path $PSScriptRoot "bfg.jar"
if (-not (Test-Path $bfgJar)) {
Invoke-WebRequest `
"https://repo1.maven.org/maven2/com/madgag/bfg/1.14.0/bfg-1.14.0.jar" `
-OutFile $bfgJar
}
# 2) Clone bare mirror next to project root: C:\vibe\auctiora\auctiora.git
$rootDir = Join-Path $PSScriptRoot ".."
$mirrorPath = Join-Path $rootDir "auctiora.git"
if (Test-Path $mirrorPath) {
Remove-Item $mirrorPath -Recurse -Force
}
git clone --mirror "https://git.appmodel.nl/Tour/auctiora.git" $mirrorPath
# 3) Run BFG in mirror
Push-Location $mirrorPath
java -jar $bfgJar --strip-blobs-bigger-than 50M .
git reflog expire --expire=now --all
git gc --prune=now --aggressive
# 4) Force-push cleaned history
git push --force
Pop-Location

206
scripts/README.md Normal file
View File

@@ -0,0 +1,206 @@
# Auctiora Scripts
Utility scripts for managing the Auctiora auction monitoring system.
## 📦 Available Scripts
### 1. Production Data Sync
Sync production database and images from `athena.lan` to your local development environment.
#### Quick Start
**Linux/Mac (Bash)**:
```bash
# Make executable (first time only)
chmod +x scripts/sync-production-data.sh
# Sync database only
./scripts/sync-production-data.sh --db-only
# Sync everything
./scripts/sync-production-data.sh --all
# Sync images only
./scripts/sync-production-data.sh --images-only
```
## 🔧 Prerequisites
### Required
- **SSH Client**: OpenSSH or equivalent
- Windows: Built-in on Windows 10+, or install [Git Bash](https://git-scm.com/downloads)
- Linux/Mac: Pre-installed
- **SCP**: Secure copy (usually comes with SSH)
- **SSH Access**: SSH key configured for `tour@athena.lan`
### Optional
- **rsync**: For efficient incremental image sync
- Windows: Install via [WSL](https://docs.microsoft.com/en-us/windows/wsl/install) or [Cygwin](https://www.cygwin.com/)
- Linux/Mac: Usually pre-installed
- **sqlite3**: For showing database statistics
- Windows: Download from [sqlite.org](https://www.sqlite.org/download.html)
- Linux: `sudo apt install sqlite3`
- Mac: Pre-installed
## 📊 What Gets Synced
### Database (`cache.db`)
- **Size**: ~8.9 GB (as of Dec 2024)
- **Contains**:
- Auctions metadata
- Lots (kavels) with bid information
- Images metadata and URLs
- HTTP cache for scraper
- **Local Path**: `c:\mnt\okcomputer\cache.db`
### Images Directory
- **Size**: Varies (can be large)
- **Contains**:
- Downloaded lot images
- Organized by lot ID
- **Local Path**: `c:\mnt\okcomputer\images\`
## 🚀 Usage Examples
## 📁 File Locations
### Remote (Production)
```
athena.lan
├── Docker Volume: shared-auction-data
│ ├── /data/cache.db (SQLite database)
│ └── /data/images/ (Image files)
└── /tmp/ (Temporary staging area)
```
### Local (Development)
```
c:\mnt\okcomputer\
├── cache.db (SQLite database)
├── cache.db.backup-* (Automatic backups)
└── images\ (Image files)
```
## 🔒 Safety Features
### Automatic Backups
- Existing local database is automatically backed up before sync
- Backup format: `cache.db.backup-YYYYMMDD-HHMMSS`
- Keep recent backups manually or clean up old ones
### Confirmation Prompts
- PowerShell script prompts for confirmation (unless `-Force` is used)
- Shows configuration before executing
- Safe to cancel at any time
### Error Handling
- Validates SSH connection before starting
- Cleans up temporary files on remote server
- Reports clear error messages
## ⚡ Performance Tips
### Faster Image Sync with rsync
Install rsync for incremental image sync (only new/changed files):
**Windows (WSL)**:
```powershell
wsl --install
wsl -d Ubuntu
sudo apt install rsync
```
**Windows (Chocolatey)**:
```powershell
choco install rsync
```
**Benefit**: First sync downloads everything, subsequent syncs only transfer changed files.
Images can be synced separately when needed for image processing tests.
## 🐛 Troubleshooting
### SSH Connection Issues
```powershell
# Test SSH connection
ssh tour@athena.lan "echo 'Connection OK'"
# Check SSH key
ssh-add -l
```
### Permission Denied
```bash
# Add SSH key (Linux/Mac)
chmod 600 ~/.ssh/id_rsa
ssh-add ~/.ssh/id_rsa
# Windows: Use PuTTY or OpenSSH for Windows
```
### Database Locked Error
```powershell
# Make sure no other process is using the database
Get-Process | Where-Object {$_.Path -like "*java*"} | Stop-Process
# Or restart the monitor
```
### Slow Image Sync
- Use rsync instead of scp (see Performance Tips)
- Consider syncing only database for code development
- Images only needed for object detection testing
## 📝 Script Details
### sync-production-data.sh (Bash)
- **Platform**: Linux, Mac, Git Bash on Windows
- **Best for**: Unix-like environments
- **Features**: Color output, progress bars, statistics
## 🔄 Automation
### Linux/Mac Cron
```bash
# Edit crontab
crontab -e
# Add daily sync at 7 AM
0 7 * * * /path/to/auctiora/scripts/sync-production-data.sh --db-only
```
## 🆘 Support
### Getting Help
```bash
# Bash
./scripts/sync-production-data.sh --help
```
### Common Commands
```powershell
# Check database size
ls c:\mnt\okcomputer\cache.db -h
# View database contents
sqlite3 c:\mnt\okcomputer\cache.db
.tables
.schema lots
SELECT COUNT(*) FROM lots;
.quit
# Check image count
(Get-ChildItem c:\mnt\okcomputer\images -Recurse -File).Count
```
## 📚 Related Documentation
- [Database Architecture](../wiki/DATABASE_ARCHITECTURE.md)
- [Integration Flowchart](../docs/INTEGRATION_FLOWCHART.md)
- [Main README](../README.md)
---
**Last Updated**: December 2025
**Maintainer**: Auctiora Development Team

160
scripts/cleanup-database.sh Normal file
View File

@@ -0,0 +1,160 @@
#!/bin/bash
#
# Database Cleanup Utility
#
# Removes invalid/old data from the local database
#
# Usage:
# ./scripts/cleanup-database.sh [--dry-run]
#
# Options:
# --dry-run Show what would be deleted without actually deleting
#
set -e
# Configuration
LOCAL_DB_PATH="${1:-c:/mnt/okcomputer/cache.db}"
DRY_RUN=false
# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m'
# Parse arguments
if [ "$1" = "--dry-run" ] || [ "$2" = "--dry-run" ]; then
DRY_RUN=true
fi
if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then
grep '^#' "$0" | sed 's/^# \?//'
exit 0
fi
echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║ Database Cleanup - Auctiora Monitor ║${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
echo ""
if [ ! -f "${LOCAL_DB_PATH}" ]; then
echo -e "${RED}Error: Database not found at ${LOCAL_DB_PATH}${NC}"
exit 1
fi
# Backup database before cleanup
if [ "$DRY_RUN" = false ]; then
BACKUP_PATH="${LOCAL_DB_PATH}.backup-before-cleanup-$(date +%Y%m%d-%H%M%S)"
echo -e "${YELLOW}Creating backup: ${BACKUP_PATH}${NC}"
cp "${LOCAL_DB_PATH}" "${BACKUP_PATH}"
echo ""
fi
# Show current state
echo -e "${BLUE}Current database state:${NC}"
sqlite3 "${LOCAL_DB_PATH}" <<EOF
.mode box
SELECT
'Total lots' as metric,
COUNT(*) as count
FROM lots
UNION ALL
SELECT
'Valid lots (with auction_id)',
COUNT(*)
FROM lots
WHERE auction_id IS NOT NULL AND auction_id != ''
UNION ALL
SELECT
'Invalid lots (missing auction_id)',
COUNT(*)
FROM lots
WHERE auction_id IS NULL OR auction_id = '';
EOF
echo ""
# Count items to be deleted
echo -e "${YELLOW}Analyzing data to clean up...${NC}"
INVALID_LOTS=$(sqlite3 "${LOCAL_DB_PATH}" "SELECT COUNT(*) FROM lots WHERE auction_id IS NULL OR auction_id = '';")
ORPHANED_IMAGES=$(sqlite3 "${LOCAL_DB_PATH}" "SELECT COUNT(*) FROM images WHERE lot_id NOT IN (SELECT lot_id FROM lots);")
echo -e " ${RED}→ Invalid lots to delete: ${INVALID_LOTS}${NC}"
echo -e " ${YELLOW}→ Orphaned images to delete: ${ORPHANED_IMAGES}${NC}"
echo ""
if [ "$INVALID_LOTS" -eq 0 ] && [ "$ORPHANED_IMAGES" -eq 0 ]; then
echo -e "${GREEN}✓ Database is clean! No cleanup needed.${NC}"
exit 0
fi
if [ "$DRY_RUN" = true ]; then
echo -e "${BLUE}DRY RUN MODE - No changes will be made${NC}"
echo ""
echo "Would delete:"
echo " - $INVALID_LOTS invalid lots"
echo " - $ORPHANED_IMAGES orphaned images"
echo ""
echo "Run without --dry-run to perform cleanup"
exit 0
fi
# Confirm cleanup
echo -e "${YELLOW}This will permanently delete the above records.${NC}"
read -p "Continue? (y/N) " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
echo "Cleanup cancelled"
exit 0
fi
# Perform cleanup
echo ""
echo -e "${YELLOW}Cleaning up database...${NC}"
# Delete invalid lots
if [ "$INVALID_LOTS" -gt 0 ]; then
echo -e " ${BLUE}[1/2] Deleting invalid lots...${NC}"
sqlite3 "${LOCAL_DB_PATH}" "DELETE FROM lots WHERE auction_id IS NULL OR auction_id = '';"
echo -e " ${GREEN}✓ Deleted ${INVALID_LOTS} invalid lots${NC}"
fi
# Delete orphaned images
if [ "$ORPHANED_IMAGES" -gt 0 ]; then
echo -e " ${BLUE}[2/2] Deleting orphaned images...${NC}"
sqlite3 "${LOCAL_DB_PATH}" "DELETE FROM images WHERE lot_id NOT IN (SELECT lot_id FROM lots);"
echo -e " ${GREEN}✓ Deleted ${ORPHANED_IMAGES} orphaned images${NC}"
fi
# Vacuum database to reclaim space
echo -e " ${BLUE}[3/3] Compacting database...${NC}"
sqlite3 "${LOCAL_DB_PATH}" "VACUUM;"
echo -e " ${GREEN}✓ Database compacted${NC}"
echo ""
echo -e "${GREEN}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Cleanup completed successfully ║${NC}"
echo -e "${GREEN}╚════════════════════════════════════════════════════════╝${NC}"
echo ""
# Show final state
echo -e "${BLUE}Final database state:${NC}"
sqlite3 "${LOCAL_DB_PATH}" <<EOF
.mode box
SELECT
'Total lots' as metric,
COUNT(*) as count
FROM lots
UNION ALL
SELECT
'Total images',
COUNT(*)
FROM images;
EOF
echo ""
DB_SIZE=$(du -h "${LOCAL_DB_PATH}" | cut -f1)
echo -e "${BLUE}Database size: ${DB_SIZE}${NC}"
echo ""

15
scripts/smb.ps1 Normal file
View File

@@ -0,0 +1,15 @@
# PowerShell: map the remote share, copy the folder, then clean up
$remote = '\\192.168.1.159\shared-auction-data'
$local = 'C:\mnt\okcomputer\output\models'
# (1) create/verify the PSDrive (prompts for password if needed)
if (-not (Get-PSDrive -Name Z -ErrorAction SilentlyContinue)) {
$cred = Get-Credential -UserName 'tour' -Message 'SMB password for tour@192.168.1.159'
New-PSDrive -Name Z -PSProvider FileSystem -Root $remote -Credential $cred -Persist | Out-Null
}
# (2) copy the local folder into the share
Copy-Item -Path $local -Destination 'Z:\' -Recurse -Force
# (3) optional cleanup
Remove-PSDrive -Name Z -Force

200
scripts/sync-production-data.sh Normal file
View File

@@ -0,0 +1,200 @@
#!/bin/bash
#
# Sync Production Data to Local
#
# This script copies the production SQLite database and images from the remote
# server (athena.lan) to your local development environment.
#
# Usage:
# ./scripts/sync-production-data.sh [--db-only|--images-only|--all]
#
# Options:
# --db-only Only sync the database (default)
# --images-only Only sync the images
# --all Sync both database and images
# --help Show this help message
#
set -e # Exit on error
# Configuration
REMOTE_HOST="tour@athena.lan"
REMOTE_VOLUME="shared-auction-data"
LOCAL_DB_PATH="c:/mnt/okcomputer/output/cache.db"
LOCAL_IMAGES_PATH="c:/mnt/okcomputer/images"
REMOTE_TMP="/tmp"
# Colors for output
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color
# Parse arguments
SYNC_MODE="db" # Default: database only
case "${1:-}" in
--db-only)
SYNC_MODE="db"
;;
--images-only)
SYNC_MODE="images"
;;
--all)
SYNC_MODE="all"
;;
--help|-h)
grep '^#' "$0" | sed 's/^# \?//'
exit 0
;;
"")
SYNC_MODE="db"
;;
*)
echo -e "${RED}Error: Unknown option '$1'${NC}"
echo "Use --help for usage information"
exit 1
;;
esac
echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║ Production Data Sync - Auctiora Monitor ║${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
echo ""
# Function to sync database
sync_database() {
echo -e "${YELLOW}[1/3] Copying database from Docker volume to /tmp...${NC}"
ssh ${REMOTE_HOST} "docker run --rm -v ${REMOTE_VOLUME}:/data -v ${REMOTE_TMP}:${REMOTE_TMP} alpine cp /data/cache.db ${REMOTE_TMP}/cache.db"
echo -e "${YELLOW}[2/3] Downloading database from remote server...${NC}"
# Create backup and remove old local database
if [ -f "${LOCAL_DB_PATH}" ]; then
BACKUP_PATH="${LOCAL_DB_PATH}.backup-$(date +%Y%m%d-%H%M%S)"
echo -e "${BLUE} Backing up existing database to: ${BACKUP_PATH}${NC}"
cp "${LOCAL_DB_PATH}" "${BACKUP_PATH}"
echo -e "${BLUE} Removing old local database...${NC}"
rm -f "${LOCAL_DB_PATH}"
fi
# Download new database
scp ${REMOTE_HOST}:${REMOTE_TMP}/cache.db "${LOCAL_DB_PATH}"
echo -e "${YELLOW}[3/3] Cleaning up remote /tmp...${NC}"
ssh ${REMOTE_HOST} "rm -f ${REMOTE_TMP}/cache.db"
# Show database info
DB_SIZE=$(du -h "${LOCAL_DB_PATH}" | cut -f1)
echo -e "${GREEN}✓ Database synced successfully (${DB_SIZE})${NC}"
# Show table counts
echo -e "${BLUE} Database statistics:${NC}"
sqlite3 "${LOCAL_DB_PATH}" <<EOF
.mode box
SELECT
'auctions' as table_name, COUNT(*) as count FROM auctions
UNION ALL
SELECT 'lots', COUNT(*) FROM lots
UNION ALL
SELECT 'images', COUNT(*) FROM images
UNION ALL
SELECT 'cache', COUNT(*) FROM cache;
EOF
# Show data quality report
echo -e "${BLUE} Data quality:${NC}"
sqlite3 "${LOCAL_DB_PATH}" <<EOF
.mode box
SELECT
'Valid lots' as metric,
COUNT(*) as count,
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%' as percentage
FROM lots
WHERE auction_id IS NOT NULL AND auction_id != ''
UNION ALL
SELECT
'Invalid lots (missing auction_id)',
COUNT(*),
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
FROM lots
WHERE auction_id IS NULL OR auction_id = ''
UNION ALL
SELECT
'Lots with intelligence fields',
COUNT(*),
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
FROM lots
WHERE followers_count IS NOT NULL OR estimated_min IS NOT NULL;
EOF
}
# Function to sync images
sync_images() {
echo -e "${YELLOW}[1/4] Getting image directory structure from Docker volume...${NC}"
# Create local images directory if it doesn't exist
mkdir -p "${LOCAL_IMAGES_PATH}"
echo -e "${YELLOW}[2/4] Copying images from Docker volume to /tmp...${NC}"
# Copy entire images directory from volume to /tmp
ssh ${REMOTE_HOST} "docker run --rm -v ${REMOTE_VOLUME}:/data -v ${REMOTE_TMP}:${REMOTE_TMP} alpine sh -c 'mkdir -p ${REMOTE_TMP}/auction-images && cp -r /data/images/* ${REMOTE_TMP}/auction-images/ 2>/dev/null || true'"
echo -e "${YELLOW}[3/4] Syncing images to local directory (this may take a while)...${NC}"
# Use rsync for efficient incremental sync
if command -v rsync &> /dev/null; then
echo -e "${BLUE} Using rsync for efficient transfer...${NC}"
rsync -avz --progress ${REMOTE_HOST}:${REMOTE_TMP}/auction-images/ "${LOCAL_IMAGES_PATH}/"
else
echo -e "${BLUE} Using scp for transfer (install rsync for faster incremental sync)...${NC}"
scp -r ${REMOTE_HOST}:${REMOTE_TMP}/auction-images/* "${LOCAL_IMAGES_PATH}/"
fi
echo -e "${YELLOW}[4/4] Cleaning up remote /tmp...${NC}"
ssh ${REMOTE_HOST} "rm -rf ${REMOTE_TMP}/auction-images"
# Show image stats
IMAGE_COUNT=$(find "${LOCAL_IMAGES_PATH}" -type f 2>/dev/null | wc -l)
IMAGE_SIZE=$(du -sh "${LOCAL_IMAGES_PATH}" 2>/dev/null | cut -f1)
echo -e "${GREEN}✓ Images synced successfully${NC}"
echo -e "${BLUE} Total images: ${IMAGE_COUNT}${NC}"
echo -e "${BLUE} Total size: ${IMAGE_SIZE}${NC}"
}
# Execute sync based on mode
START_TIME=$(date +%s)
case "$SYNC_MODE" in
db)
echo -e "${BLUE}Mode: Database only${NC}"
echo ""
sync_database
;;
images)
echo -e "${BLUE}Mode: Images only${NC}"
echo ""
sync_images
;;
all)
echo -e "${BLUE}Mode: Database + Images${NC}"
echo ""
sync_database
echo ""
sync_images
;;
esac
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))
echo ""
echo -e "${GREEN}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Sync completed successfully in ${DURATION} seconds ║${NC}"
echo -e "${GREEN}╚════════════════════════════════════════════════════════╝${NC}"
echo ""
echo -e "${BLUE}Next steps:${NC}"
echo -e " 1. Verify data: sqlite3 ${LOCAL_DB_PATH} 'SELECT COUNT(*) FROM lots;'"
echo -e " 2. Start monitor: mvn quarkus:dev"
echo -e " 3. Open dashboard: http://localhost:8080"
echo ""

19
src/main/java/auctiora/AuctionInfo.java Normal file
View File

@@ -0,0 +1,19 @@
package auctiora;
import java.time.LocalDateTime;
/**
* Represents auction metadata (veiling informatie)
* Data typically populated by the external scraper process
*/
public record AuctionInfo(
long auctionId, // Unique auction ID (from URL)
String title, // Auction title
String location, // Location (e.g., "Amsterdam, NL")
String city, // City name
String country, // Country code (e.g., "NL")
String url, // Full auction URL
String typePrefix, // Auction type (A1 or A7)
int lotCount, // Number of lots/kavels
LocalDateTime firstLotClosingTime // Closing time if available
) { }

82
src/main/java/auctiora/AuctionMonitorHealthCheck.java Normal file
View File

@@ -0,0 +1,82 @@
package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.eclipse.microprofile.health.HealthCheck;
import org.eclipse.microprofile.health.HealthCheckResponse;
import org.eclipse.microprofile.health.Liveness;
import org.eclipse.microprofile.health.Readiness;
import org.eclipse.microprofile.health.Startup;
import java.nio.file.Files;
import java.nio.file.Paths;
@ApplicationScoped
public class AuctionMonitorHealthCheck {
@Liveness
public static class LivenessCheck
implements HealthCheck {
@Override public HealthCheckResponse call() {
return HealthCheckResponse.up("Auction Monitor is alive");
}
}
@Readiness
@ApplicationScoped
public static class ReadinessCheck
implements HealthCheck {
@Inject DatabaseService db;
@Override
public HealthCheckResponse call() {
try {
var auctions = db.getAllAuctions();
var dbPath = Paths.get("C:\\mnt\\okcomputer\\output\\cache.db");
if (!Files.exists(dbPath.getParent())) {
return HealthCheckResponse.down("Database directory does not exist");
}
return HealthCheckResponse.named("database")
.up()
.withData("auctions", auctions.size())
.build();
} catch (Exception e) {
return HealthCheckResponse.named("database")
.down()
.withData("error", e.getMessage())
.build();
}
}
}
@Startup
@ApplicationScoped
public static class StartupCheck
implements HealthCheck {
@Inject DatabaseService db;
@Override
public HealthCheckResponse call() {
try {
// Verify database schema
db.ensureSchema();
return HealthCheckResponse.named("startup")
.up()
.withData("message", "Database schema initialized")
.build();
} catch (Exception e) {
return HealthCheckResponse.named("startup")
.down()
.withData("error", e.getMessage())
.build();
}
}
}
}

61
src/main/java/auctiora/AuctionMonitorProducer.java Normal file
View File

@@ -0,0 +1,61 @@
package auctiora;
import io.quarkus.runtime.Startup;
import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.inject.Produces;
import jakarta.inject.Singleton;
import nu.pattern.OpenCV;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import org.opencv.core.Core;
import java.io.IOException;
import java.sql.SQLException;
/**
* CDI Producer for auction monitor services.
* Creates and configures singleton instances of core services.
*/
@Startup
@ApplicationScoped
public class AuctionMonitorProducer {
private static final Logger LOG = Logger.getLogger(AuctionMonitorProducer.class);
@PostConstruct void init() {
try {
OpenCV.loadLocally();
LOG.info("✓ OpenCV loaded successfully");
} catch (Exception e) {
LOG.warn("⚠️ OpenCV not available - image detection will be disabled: " + e.getMessage());
}
}
@Produces @Singleton public DatabaseService produceDatabaseService(
@ConfigProperty(name = "auction.database.path") String dbPath) throws SQLException {
var db = new DatabaseService(dbPath);
db.ensureSchema();
return db;
}
@Produces @Singleton public NotificationService produceNotificationService(
@ConfigProperty(name = "auction.notification.config") String config) {
return new NotificationService(config);
}
@Produces @Singleton public ObjectDetectionService produceObjectDetectionService(
@ConfigProperty(name = "auction.yolo.config") String cfgPath,
@ConfigProperty(name = "auction.yolo.weights") String weightsPath,
@ConfigProperty(name = "auction.yolo.classes") String classesPath) throws IOException {
return new ObjectDetectionService(cfgPath, weightsPath, classesPath);
}
@Produces @Singleton public ImageProcessingService produceImageProcessingService(
DatabaseService db,
ObjectDetectionService detector) {
return new ImageProcessingService(db, detector);
}
}

925
src/main/java/auctiora/AuctionMonitorResource.java Normal file
View File

@@ -0,0 +1,925 @@
package auctiora;
import jakarta.inject.Inject;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.jboss.logging.Logger;
import java.time.LocalDateTime;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.stream.Collectors;
/**
* REST API for Auction Monitor control and status.
* Provides endpoints for:
* - Status checking
* - Manual workflow triggers
* - Statistics
*/
@Path("/api/monitor")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public class AuctionMonitorResource {
private static final Logger LOG = Logger.getLogger(AuctionMonitorResource.class);
@Inject DatabaseService db;
@Inject QuarkusWorkflowScheduler scheduler;
@Inject NotificationService notifier;
@Inject RateLimitedHttpClient httpClient;
@Inject LotEnrichmentService enrichmentService;
/**
* GET /api/monitor/status
* Returns current monitoring status
*/
@GET
@Path("/status")
public Response getStatus() {
try {
Map<String, Object> status = new HashMap<>();
status.put("running", true);
status.put("auctions", db.getAllAuctions().size());
status.put("lots", db.getAllLots().size());
status.put("images", db.getImageCount());
// Count closing soon (within 30 minutes, excluding already-closed)
var closingSoon = 0;
for (var lot : db.getAllLots()) {
if (lot.closingTime() != null) {
long minutes = lot.minutesUntilClose();
if (minutes > 0 && minutes < 30) {
closingSoon++;
}
}
}
status.put("closingSoon", closingSoon);
return Response.ok(status).build();
} catch (Exception e) {
LOG.error("Failed to get status", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/statistics
* Returns detailed statistics
*/
@GET
@Path("/statistics")
public Response getStatistics() {
try {
Map<String, Object> stats = new HashMap<>();
var auctions = db.getAllAuctions();
var lots = db.getAllLots();
stats.put("totalAuctions", auctions.size());
stats.put("totalLots", lots.size());
stats.put("totalImages", db.getImageCount());
// Lot statistics
var activeLots = 0;
var lotsWithBids = 0;
double totalBids = 0;
var hotLots = 0;
var sleeperLots = 0;
var bargainLots = 0;
var lotsClosing1h = 0;
var lotsClosing6h = 0;
double totalBidVelocity = 0;
int velocityCount = 0;
for (var lot : lots) {
long minutesLeft = lot.closingTime() != null ? lot.minutesUntilClose() : Long.MAX_VALUE;
if (lot.closingTime() != null && minutesLeft > 0) {
activeLots++;
// Time-based counts
if (minutesLeft < 60) lotsClosing1h++;
if (minutesLeft < 360) lotsClosing6h++;
}
if (lot.currentBid() > 0) {
lotsWithBids++;
totalBids += lot.currentBid();
}
// Intelligence metrics (require GraphQL enrichment)
if (lot.followersCount() != null && lot.followersCount() > 20) {
hotLots++;
}
if (lot.isSleeperLot()) {
sleeperLots++;
}
if (lot.isBelowEstimate()) {
bargainLots++;
}
// Bid velocity
if (lot.bidVelocity() != null && lot.bidVelocity() > 0) {
totalBidVelocity += lot.bidVelocity();
velocityCount++;
}
}
// Calculate bids per hour (average velocity across all lots with velocity data)
double bidsPerHour = velocityCount > 0 ? totalBidVelocity / velocityCount : 0;
stats.put("activeLots", activeLots);
stats.put("lotsWithBids", lotsWithBids);
stats.put("totalBidValue", String.format("€%.2f", totalBids));
stats.put("averageBid", lotsWithBids > 0 ? String.format("€%.2f", totalBids / lotsWithBids) : "€0.00");
// Bidding intelligence
stats.put("bidsPerHour", String.format("%.1f", bidsPerHour));
stats.put("hotLots", hotLots);
stats.put("sleeperLots", sleeperLots);
stats.put("bargainLots", bargainLots);
stats.put("lotsClosing1h", lotsClosing1h);
stats.put("lotsClosing6h", lotsClosing6h);
// Conversion rate
double conversionRate = activeLots > 0 ? (lotsWithBids * 100.0 / activeLots) : 0;
stats.put("conversionRate", String.format("%.1f%%", conversionRate));
return Response.ok(stats).build();
} catch (Exception e) {
LOG.error("Failed to get statistics", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/closing-soon
* Returns lots closing within the next specified hours (default: 24 hours)
*/
@GET
@Path("/closing-soon")
public Response getClosingSoon(@QueryParam("hours") @DefaultValue("24") int hours) {
try {
var lots = db.getAllLots();
var closingSoon = lots.stream()
.filter(lot -> lot.closingTime() != null)
.filter(lot -> lot.minutesUntilClose() > 0 && lot.minutesUntilClose() <= hours * 60)
.sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose()))
.limit(100)
.toList();
return Response.ok(closingSoon).build();
} catch (Exception e) {
LOG.error("Failed to get closing soon lots", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/lots/{lotId}/bid-history
* Returns bid history for a specific lot
*/
@GET
@Path("/lots/{lotId}/bid-history")
public Response getBidHistory(@PathParam("lotId") String lotId) {
try {
var history = db.getBidHistory(lotId);
return Response.ok(history).build();
} catch (Exception e) {
LOG.error("Failed to get bid history for lot {}", lotId, e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/trigger/scraper-import
* Manually trigger scraper import workflow
*/
@POST
@Path("/trigger/scraper-import")
public Response triggerScraperImport() {
try {
scheduler.importScraperData();
return Response.ok(Map.of("message", "Scraper import triggered successfully")).build();
} catch (Exception e) {
LOG.error("Failed to trigger scraper import", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/trigger/image-processing
* Manually trigger image processing workflow
*/
@POST
@Path("/trigger/image-processing")
public Response triggerImageProcessing() {
try {
scheduler.processImages();
return Response.ok(Map.of("message", "Image processing triggered successfully")).build();
} catch (Exception e) {
LOG.error("Failed to trigger image processing", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/trigger/bid-monitoring
* Manually trigger bid monitoring workflow
*/
@POST
@Path("/trigger/bid-monitoring")
public Response triggerBidMonitoring() {
try {
scheduler.monitorBids();
return Response.ok(Map.of("message", "Bid monitoring triggered successfully")).build();
} catch (Exception e) {
LOG.error("Failed to trigger bid monitoring", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/trigger/closing-alerts
* Manually trigger closing alerts workflow
*/
@POST
@Path("/trigger/closing-alerts")
public Response triggerClosingAlerts() {
try {
scheduler.checkClosingTimes();
return Response.ok(Map.of("message", "Closing alerts triggered successfully")).build();
} catch (Exception e) {
LOG.error("Failed to trigger closing alerts", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/trigger/graphql-enrichment
* Manually trigger GraphQL enrichment for all lots or lots closing soon
*/
@POST
@Path("/trigger/graphql-enrichment")
public Response triggerGraphQLEnrichment(@QueryParam("hoursUntilClose") @DefaultValue("24") int hours) {
try {
int enriched;
if (hours > 0) {
enriched = enrichmentService.enrichClosingSoonLots(hours);
return Response.ok(Map.of(
"message", "GraphQL enrichment triggered for lots closing within " + hours + " hours",
"enrichedCount", enriched
)).build();
} else {
enriched = enrichmentService.enrichAllActiveLots();
return Response.ok(Map.of(
"message", "GraphQL enrichment triggered for all lots",
"enrichedCount", enriched
)).build();
}
} catch (Exception e) {
LOG.error("Failed to trigger GraphQL enrichment", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/auctions
* Returns list of all auctions
*/
@GET
@Path("/auctions")
public Response getAuctions(@QueryParam("country") String country) {
try {
var auctions = country != null && !country.isEmpty()
? db.getAuctionsByCountry(country)
: db.getAllAuctions();
return Response.ok(auctions).build();
} catch (Exception e) {
LOG.error("Failed to get auctions", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/lots
* Returns list of active lots
*/
@GET
@Path("/lots")
public Response getActiveLots() {
try {
var lots = db.getActiveLots();
return Response.ok(lots).build();
} catch (Exception e) {
LOG.error("Failed to get lots", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/lots/closing-soon
* Returns lots closing within specified minutes (default 30)
*/
@GET
@Path("/lots/closing-soon")
public Response getLotsClosingSoon(@QueryParam("minutes") @DefaultValue("30") int minutes) {
try {
var allLots = db.getActiveLots();
var closingSoon = allLots.stream()
.filter(lot -> lot.closingTime() != null)
.filter(lot -> {
long minutesLeft = lot.minutesUntilClose();
return minutesLeft > 0 && minutesLeft < minutes;
})
.sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose()))
.toList();
return Response.ok(closingSoon).build();
} catch (Exception e) {
LOG.error("Failed to get closing lots", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/lots/{lotId}/images
* Returns images for a specific lot
*/
@GET
@Path("/lots/{lotId}/images")
public Response getLotImages(@PathParam("lotId") int lotId) {
try {
var images = db.getImagesForLot(lotId);
return Response.ok(images).build();
} catch (Exception e) {
LOG.error("Failed to get lot images", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/test-notification
* Send a test notification
*/
@POST
@Path("/test-notification")
public Response sendTestNotification(Map<String, String> request) {
try {
var message = request.getOrDefault("message", "Test notification from Auction Monitor");
var title = request.getOrDefault("title", "Test Notification");
var priority = Integer.parseInt(request.getOrDefault("priority", "0"));
notifier.sendNotification(message, title, priority);
return Response.ok(Map.of("message", "Test notification sent successfully")).build();
} catch (Exception e) {
LOG.error("Failed to send test notification", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/rate-limit/stats
* Returns HTTP rate limiting statistics for all hosts
*/
@GET
@Path("/rate-limit/stats")
public Response getRateLimitStats() {
try {
var stats = httpClient.getAllStats();
Map<String, Object> response = new HashMap<>();
response.put("hosts", stats.size());
Map<String, Object> hostStats = new HashMap<>();
for (var entry : stats.entrySet()) {
var stat = entry.getValue();
hostStats.put(entry.getKey(), Map.of(
"totalRequests", stat.getTotalRequests(),
"successfulRequests", stat.getSuccessfulRequests(),
"failedRequests", stat.getFailedRequests(),
"rateLimitedRequests", stat.getRateLimitedRequests(),
"averageDurationMs", stat.getAverageDurationMs()
));
}
response.put("statistics", hostStats);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get rate limit stats", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/rate-limit/stats/{host}
* Returns HTTP rate limiting statistics for a specific host
*/
@GET
@Path("/rate-limit/stats/{host}")
public Response getRateLimitStatsForHost(@PathParam("host") String host) {
try {
var stat = httpClient.getStats(host);
if (stat == null) {
return Response.status(Response.Status.NOT_FOUND)
.entity(Map.of("error", "No statistics found for host: " + host))
.build();
}
Map<String, Object> response = Map.of(
"host", stat.getHost(),
"totalRequests", stat.getTotalRequests(),
"successfulRequests", stat.getSuccessfulRequests(),
"failedRequests", stat.getFailedRequests(),
"rateLimitedRequests", stat.getRateLimitedRequests(),
"averageDurationMs", stat.getAverageDurationMs()
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get rate limit stats for host", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/charts/country-distribution
* Returns dynamic country distribution for charts
*/
@GET
@Path("/charts/country-distribution")
public Response getCountryDistribution() {
try {
var auctions = db.getAllAuctions();
Map<String, Long> distribution = auctions.stream()
.filter(a -> a.country() != null && !a.country().isEmpty())
.collect(Collectors.groupingBy(
AuctionInfo::country,
Collectors.counting()
));
return Response.ok(distribution).build();
} catch (Exception e) {
LOG.error("Failed to get country distribution", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/charts/category-distribution
* Returns dynamic category distribution with intelligence for charts
*/
@GET
@Path("/charts/category-distribution")
public Response getCategoryDistribution() {
try {
var lots = db.getAllLots();
// Category distribution
Map<String, Long> distribution = lots.stream()
.filter(l -> l.category() != null && !l.category().isEmpty())
.collect(Collectors.groupingBy(
l -> l.category().length() > 20 ? l.category().substring(0, 20) + "..." : l.category(),
Collectors.counting()
));
// Find top category by count
var topCategory = distribution.entrySet().stream()
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse("N/A");
// Calculate average bids per category
Map<String, Double> avgBidsByCategory = lots.stream()
.filter(l -> l.category() != null && !l.category().isEmpty() && l.currentBid() > 0)
.collect(Collectors.groupingBy(
l -> l.category().length() > 20 ? l.category().substring(0, 20) + "..." : l.category(),
Collectors.averagingDouble(Lot::currentBid)
));
double overallAvgBid = lots.stream()
.filter(l -> l.currentBid() > 0)
.mapToDouble(Lot::currentBid)
.average()
.orElse(0.0);
Map<String, Object> response = new HashMap<>();
response.put("distribution", distribution);
response.put("topCategory", topCategory);
response.put("categoryCount", distribution.size());
response.put("averageBidOverall", String.format("€%.2f", overallAvgBid));
response.put("avgBidsByCategory", avgBidsByCategory);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get category distribution", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/charts/bidding-trend
* Returns time series data for last N hours
*/
@GET
@Path("/charts/bidding-trend")
public Response getBiddingTrend(@QueryParam("hours") @DefaultValue("24") int hours) {
try {
var lots = db.getAllLots();
Map<Integer, TrendHour> trends = new HashMap<>();
// Initialize hours
LocalDateTime now = LocalDateTime.now();
for (int i = hours - 1; i >= 0; i--) {
LocalDateTime hour = now.minusHours(i);
int hourKey = hour.getHour();
trends.put(hourKey, new TrendHour(hourKey, 0, 0));
}
// Count lots and bids per hour (mock implementation - in real app, use timestamp data)
// This is a simplified version - you'd need actual timestamps in DB
for (var lot : lots) {
if (lot.closingTime() != null) {
int hour = lot.closingTime().getHour();
TrendHour trend = trends.getOrDefault(hour, new TrendHour(hour, 0, 0));
trend.lots++;
if (lot.currentBid() > 0) trend.bids++;
}
}
return Response.ok(trends.values()).build();
} catch (Exception e) {
LOG.error("Failed to get bidding trend", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/charts/insights
* Returns intelligent insights
*/
@GET
@Path("/charts/insights")
public Response getInsights() {
try {
var lots = db.getAllLots();
var auctions = db.getAllAuctions();
List<Map<String, String>> insights = new ArrayList<>();
// Calculate insights
long criticalCount = lots.stream().filter(l -> l.minutesUntilClose() < 30).count();
if (criticalCount > 10) {
insights.add(Map.of(
"icon", "fa-exclamation-circle",
"title", criticalCount + " lots closing soon",
"description", "High urgency items require attention"
));
}
double bidRate = lots.stream().filter(l -> l.currentBid() > 0).count() * 100.0 / lots.size();
if (bidRate > 60) {
insights.add(Map.of(
"icon", "fa-chart-line",
"title", String.format("%.1f%% bid rate", bidRate),
"description", "Strong market engagement detected"
));
}
long imageCoverage = db.getImageCount() * 100 / Math.max(lots.size(), 1);
if (imageCoverage < 80) {
insights.add(Map.of(
"icon", "fa-images",
"title", imageCoverage + "% image coverage",
"description", "Consider processing more images"
));
}
// Add geographic insight (filter out null countries)
String topCountry = auctions.stream()
.filter(a -> a.country() != null)
.collect(Collectors.groupingBy(AuctionInfo::country, Collectors.counting()))
.entrySet().stream()
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse("N/A");
if (!"N/A".equals(topCountry)) {
insights.add(Map.of(
"icon", "fa-globe",
"title", topCountry + " leading",
"description", "Top performing country"
));
}
// Add sleeper lots insight
long sleeperCount = lots.stream().filter(Lot::isSleeperLot).count();
if (sleeperCount > 0) {
insights.add(Map.of(
"icon", "fa-eye",
"title", sleeperCount + " sleeper lots",
"description", "High interest, low bids - opportunity?"
));
}
// Add bargain insight
long bargainCount = lots.stream().filter(Lot::isBelowEstimate).count();
if (bargainCount > 5) {
insights.add(Map.of(
"icon", "fa-tag",
"title", bargainCount + " bargains",
"description", "Priced below auction house estimates"
));
}
// Add watch/followers insight
long highWatchCount = lots.stream()
.filter(l -> l.followersCount() != null && l.followersCount() > 20)
.count();
if (highWatchCount > 0) {
insights.add(Map.of(
"icon", "fa-fire",
"title", highWatchCount + " hot lots",
"description", "High follower count, strong competition"
));
}
return Response.ok(insights).build();
} catch (Exception e) {
LOG.error("Failed to get insights", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/intelligence/sleepers
* Returns "sleeper" lots (high watch count, low bids)
*/
@GET
@Path("/intelligence/sleepers")
public Response getSleeperLots(@QueryParam("minFollowers") @DefaultValue("10") int minFollowers) {
try {
var allLots = db.getAllLots();
var sleepers = allLots.stream()
.filter(Lot::isSleeperLot)
.toList();
Map<String, Object> response = Map.of(
"count", sleepers.size(),
"lots", sleepers
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get sleeper lots", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/intelligence/bargains
* Returns lots priced below auction house estimates
*/
@GET
@Path("/intelligence/bargains")
public Response getBargains() {
try {
var allLots = db.getAllLots();
var bargains = allLots.stream()
.filter(Lot::isBelowEstimate)
.sorted((a, b) -> {
Double ratioA = a.getPriceVsEstimateRatio();
Double ratioB = b.getPriceVsEstimateRatio();
if (ratioA == null) return 1;
if (ratioB == null) return -1;
return ratioA.compareTo(ratioB);
})
.toList();
Map<String, Object> response = Map.of(
"count", bargains.size(),
"lots", bargains
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get bargains", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/intelligence/popular
* Returns lots by popularity level
*/
@GET
@Path("/intelligence/popular")
public Response getPopularLots(@QueryParam("level") @DefaultValue("HIGH") String level) {
try {
var allLots = db.getAllLots();
var popular = allLots.stream()
.filter(lot -> level.equalsIgnoreCase(lot.getPopularityLevel()))
.sorted((a, b) -> {
Integer followersA = a.followersCount() != null ? a.followersCount() : 0;
Integer followersB = b.followersCount() != null ? b.followersCount() : 0;
return followersB.compareTo(followersA);
})
.toList();
Map<String, Object> response = Map.of(
"count", popular.size(),
"level", level,
"lots", popular
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get popular lots", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/intelligence/price-analysis
* Returns price vs estimate analysis
*/
@GET
@Path("/intelligence/price-analysis")
public Response getPriceAnalysis() {
try {
var allLots = db.getAllLots();
long belowEstimate = allLots.stream().filter(Lot::isBelowEstimate).count();
long aboveEstimate = allLots.stream().filter(Lot::isAboveEstimate).count();
long withEstimates = allLots.stream()
.filter(lot -> lot.estimatedMin() != null && lot.estimatedMax() != null)
.count();
double avgPriceVsEstimate = allLots.stream()
.map(Lot::getPriceVsEstimateRatio)
.filter(ratio -> ratio != null)
.mapToDouble(Double::doubleValue)
.average()
.orElse(0.0);
Map<String, Object> response = Map.of(
"totalLotsWithEstimates", withEstimates,
"belowEstimate", belowEstimate,
"aboveEstimate", aboveEstimate,
"averagePriceVsEstimatePercent", Math.round(avgPriceVsEstimate),
"bargainOpportunities", belowEstimate
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get price analysis", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/lots/{lotId}/intelligence
* Returns detailed intelligence for a specific lot
*/
@GET
@Path("/lots/{lotId}/intelligence")
public Response getLotIntelligence(@PathParam("lotId") long lotId) {
try {
var lot = db.getAllLots().stream()
.filter(l -> l.lotId() == lotId)
.findFirst()
.orElse(null);
if (lot == null) {
return Response.status(Response.Status.NOT_FOUND)
.entity(Map.of("error", "Lot not found"))
.build();
}
Map<String, Object> intelligence = new HashMap<>();
intelligence.put("lotId", lot.lotId());
intelligence.put("followersCount", lot.followersCount());
intelligence.put("popularityLevel", lot.getPopularityLevel());
intelligence.put("estimatedMidpoint", lot.getEstimatedMidpoint());
intelligence.put("priceVsEstimatePercent", lot.getPriceVsEstimateRatio());
intelligence.put("isBargain", lot.isBelowEstimate());
intelligence.put("isOvervalued", lot.isAboveEstimate());
intelligence.put("isSleeperLot", lot.isSleeperLot());
intelligence.put("nextBidAmount", lot.calculateNextBid());
intelligence.put("totalCostWithFees", lot.calculateTotalCost());
intelligence.put("viewCount", lot.viewCount());
intelligence.put("bidVelocity", lot.bidVelocity());
intelligence.put("condition", lot.condition());
intelligence.put("vat", lot.vat());
intelligence.put("buyerPremium", lot.buyerPremiumPercentage());
return Response.ok(intelligence).build();
} catch (Exception e) {
LOG.error("Failed to get lot intelligence", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/charts/watch-distribution
* Returns follower/watch count distribution
*/
@GET
@Path("/charts/watch-distribution")
public Response getWatchDistribution() {
try {
var lots = db.getAllLots();
Map<String, Long> distribution = new HashMap<>();
distribution.put("0 watchers", lots.stream().filter(l -> l.followersCount() == null || l.followersCount() == 0).count());
distribution.put("1-5 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 1 && l.followersCount() <= 5).count());
distribution.put("6-20 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 6 && l.followersCount() <= 20).count());
distribution.put("21-50 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 21 && l.followersCount() <= 50).count());
distribution.put("50+ watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() > 50).count());
return Response.ok(distribution).build();
} catch (Exception e) {
LOG.error("Failed to get watch distribution", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
// Helper class for trend data
public static class TrendHour {
public int hour;
public int lots;
public int bids;
public TrendHour(int hour, int lots, int bids) {
this.hour = hour;
this.lots = lots;
this.bids = bids;
}
}
}

16
src/main/java/auctiora/BidHistory.java Normal file
View File

@@ -0,0 +1,16 @@
package auctiora;
import java.time.LocalDateTime;
/**
* Represents a bid in the bid history
*/
public record BidHistory(
int id,
String lotId,
double bidAmount,
LocalDateTime bidTime,
boolean isAutobid,
String bidderId,
Integer bidderNumber
) {}

218
src/main/java/auctiora/DatabaseService.java Normal file
View File

@@ -0,0 +1,218 @@
package auctiora;
import auctiora.db.*;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Jdbi;
import java.util.List;
/**
* Refactored database service using repository pattern and JDBI3.
* Delegates operations to specialized repositories for better separation of concerns.
*
* @deprecated Legacy methods maintained for backward compatibility.
* New code should use repositories directly via dependency injection.
*/
@Slf4j
public class DatabaseService {
private final Jdbi jdbi;
private final LotRepository lotRepository;
private final AuctionRepository auctionRepository;
private final ImageRepository imageRepository;
/**
* Constructor for programmatic instantiation (tests, CLI tools).
*/
public DatabaseService(String dbPath) {
String url = "jdbc:sqlite:" + dbPath + "?journal_mode=WAL&busy_timeout=10000";
this.jdbi = Jdbi.create(url);
// Initialize schema
DatabaseSchema.ensureSchema(jdbi);
// Create repositories
this.lotRepository = new LotRepository(jdbi);
this.auctionRepository = new AuctionRepository(jdbi);
this.imageRepository = new ImageRepository(jdbi);
}
/**
* Constructor with JDBI instance (for dependency injection).
*/
public DatabaseService(Jdbi jdbi) {
this.jdbi = jdbi;
DatabaseSchema.ensureSchema(jdbi);
this.lotRepository = new LotRepository(jdbi);
this.auctionRepository = new AuctionRepository(jdbi);
this.imageRepository = new ImageRepository(jdbi);
}
// ==================== LEGACY COMPATIBILITY METHODS ====================
// These methods delegate to repositories for backward compatibility
void ensureSchema() {
DatabaseSchema.ensureSchema(jdbi);
}
synchronized void upsertAuction(AuctionInfo auction) {
auctionRepository.upsert(auction);
}
synchronized List<AuctionInfo> getAllAuctions() {
return auctionRepository.getAll();
}
synchronized List<AuctionInfo> getAuctionsByCountry(String countryCode) {
return auctionRepository.getByCountry(countryCode);
}
synchronized void upsertLot(Lot lot) {
lotRepository.upsert(lot);
}
synchronized void upsertLotWithIntelligence(Lot lot) {
lotRepository.upsertWithIntelligence(lot);
}
synchronized void updateLotCurrentBid(Lot lot) {
lotRepository.updateCurrentBid(lot);
}
synchronized void updateLotNotificationFlags(Lot lot) {
lotRepository.updateNotificationFlags(lot);
}
synchronized List<Lot> getActiveLots() {
return lotRepository.getActiveLots();
}
synchronized List<Lot> getAllLots() {
return lotRepository.getAllLots();
}
synchronized List<BidHistory> getBidHistory(String lotId) {
return lotRepository.getBidHistory(lotId);
}
synchronized void insertBidHistory(List<BidHistory> bidHistory) {
lotRepository.insertBidHistory(bidHistory);
}
synchronized void insertImage(long lotId, String url, String filePath, List<String> labels) {
imageRepository.insert(lotId, url, filePath, labels);
}
synchronized void updateImageLabels(int imageId, List<String> labels) {
imageRepository.updateLabels(imageId, labels);
}
synchronized List<String> getImageLabels(int imageId) {
return imageRepository.getLabels(imageId);
}
synchronized List<ImageRecord> getImagesForLot(long lotId) {
return imageRepository.getImagesForLot(lotId)
.stream()
.map(img -> new ImageRecord(img.id(), img.lotId(), img.url(), img.filePath(), img.labels()))
.toList();
}
synchronized List<ImageDetectionRecord> getImagesNeedingDetection() {
return imageRepository.getImagesNeedingDetection()
.stream()
.map(img -> new ImageDetectionRecord(img.id(), img.lotId(), img.filePath()))
.toList();
}
synchronized int getImageCount() {
return imageRepository.getImageCount();
}
synchronized List<AuctionInfo> importAuctionsFromScraper() {
return jdbi.withHandle(handle -> {
var sql = """
SELECT
l.auction_id,
MIN(l.title) as title,
MIN(l.location) as location,
MIN(l.url) as url,
COUNT(*) as lots_count,
MIN(l.closing_time) as first_lot_closing_time,
MIN(l.scraped_at) as scraped_at
FROM lots l
WHERE l.auction_id IS NOT NULL
GROUP BY l.auction_id
""";
return handle.createQuery(sql)
.map((rs, ctx) -> {
try {
var auction = ScraperDataAdapter.fromScraperAuction(rs);
if (auction.auctionId() != 0L) {
auctionRepository.upsert(auction);
return auction;
}
} catch (Exception e) {
log.warn("Failed to import auction: {}", e.getMessage());
}
return null;
})
.list()
.stream()
.filter(a -> a != null)
.toList();
});
}
synchronized List<Lot> importLotsFromScraper() {
return jdbi.withHandle(handle -> {
var sql = "SELECT * FROM lots";
return handle.createQuery(sql)
.map((rs, ctx) -> {
try {
var lot = ScraperDataAdapter.fromScraperLot(rs);
if (lot.lotId() != 0L && lot.saleId() != 0L) {
lotRepository.upsert(lot);
return lot;
}
} catch (Exception e) {
log.warn("Failed to import lot: {}", e.getMessage());
}
return null;
})
.list()
.stream()
.filter(l -> l != null)
.toList();
});
}
// ==================== DIRECT REPOSITORY ACCESS ====================
// Expose repositories for modern usage patterns
public LotRepository lots() {
return lotRepository;
}
public AuctionRepository auctions() {
return auctionRepository;
}
public ImageRepository images() {
return imageRepository;
}
public Jdbi getJdbi() {
return jdbi;
}
// ==================== LEGACY RECORDS ====================
// Keep records for backward compatibility with existing code
public record ImageRecord(int id, long lotId, String url, String filePath, String labels) {}
public record ImageDetectionRecord(int id, long lotId, String filePath) {}
}

55
src/main/java/auctiora/ImageProcessingService.java Normal file
View File

@@ -0,0 +1,55 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public record ImageProcessingService(DatabaseService db, ObjectDetectionService detector) {
boolean processImage(int id, String path, long lot) {
try {
path = path.replace('\\', '/');
var f = new java.io.File(path);
if (!f.exists() || !f.canRead()) {
log.warn("Image not accessible: {}", path);
return false;
}
if (f.length() > 50L * 1024 * 1024) {
log.warn("Image too large: {}", path);
return false;
}
var labels = detector.detectObjects(path);
db.updateImageLabels(id, labels);
if (!labels.isEmpty())
log.info("Lot {}: {}", lot, String.join(", ", labels));
return true;
} catch (Exception e) {
log.warn("Process fail {}: {}", id, e.getMessage());
return false;
}
}
void processPendingImages() {
try {
var images = db.getImagesNeedingDetection();
log.info("Pending {}", images.size());
int processed = 0, detected = 0;
for (var i : images) {
if (processImage(i.id(), i.filePath(), i.lotId())) {
processed++;
var lbl = db.getImageLabels(i.id());
if (lbl != null && !lbl.isEmpty()) detected++;
}
}
log.info("Processed {}, detected {}", processed, detected);
} catch (Exception e) {
log.warn("Batch fail: {}", e.getMessage());
}
}
}

153
src/main/java/auctiora/Lot.java Normal file
View File

@@ -0,0 +1,153 @@
package auctiora;
import lombok.With;
import java.time.Duration;
import java.time.LocalDateTime;
/// Represents a lot (kavel) in an auction.
/// Data typically populated by the external scraper process.
/// This project enriches the data with image analysis and monitoring.
@With
public record Lot(
long saleId,
long lotId,
String displayId, // Full lot ID string (e.g., "A1-34732-49") for GraphQL queries
String title,
String description,
String manufacturer,
String type,
int year,
String category,
double currentBid,
String currency,
String url,
LocalDateTime closingTime,
boolean closingNotified,
// HIGH PRIORITY FIELDS from GraphQL API
Integer followersCount, // Watch count - direct competition indicator
Double estimatedMin, // Auction house min estimate (cents)
Double estimatedMax, // Auction house max estimate (cents)
Long nextBidStepInCents, // Exact bid increment from API
String condition, // Direct condition field
String categoryPath, // Structured category (e.g., "Vehicles > Cars > Classic")
String cityLocation, // Structured location
String countryCode, // ISO country code
// MEDIUM PRIORITY FIELDS
String biddingStatus, // More detailed than minimumBidAmountMet
String appearance, // Visual condition notes
String packaging, // Packaging details
Long quantity, // Lot quantity (bulk lots)
Double vat, // VAT percentage
Double buyerPremiumPercentage, // Buyer premium
String remarks, // Viewing/pickup notes
// BID INTELLIGENCE FIELDS
Double startingBid, // Starting/opening bid
Double reservePrice, // Reserve price (if disclosed)
Boolean reserveMet, // Reserve met status
Double bidIncrement, // Calculated bid increment
Integer viewCount, // Number of views
LocalDateTime firstBidTime, // First bid timestamp
LocalDateTime lastBidTime, // Last bid timestamp
Double bidVelocity, // Bids per hour
Double condition_score,
//Integer manufacturing_year,
Integer provenance_docs
) {
public Integer provenanceDocs() { return provenance_docs; }
/// manufacturing_year
public Integer manufacturingYear() { return year; }
public Double conditionScore() { return condition_score; }
public long minutesUntilClose() {
if (closingTime == null) return Long.MAX_VALUE;
return Duration.between(LocalDateTime.now(), closingTime).toMinutes();
}
// Intelligence Methods
/// Calculate total cost including VAT and buyer premium
public double calculateTotalCost() {
double base = currentBid > 0 ? currentBid : 0;
if (vat != null && vat > 0) {
base += (base * vat / 100.0);
}
if (buyerPremiumPercentage != null && buyerPremiumPercentage > 0) {
base += (base * buyerPremiumPercentage / 100.0);
}
return base;
}
/// Calculate next bid amount using API-provided increment
public double calculateNextBid() {
if (nextBidStepInCents != null && nextBidStepInCents > 0) {
return currentBid + (nextBidStepInCents / 100.0);
} else if (bidIncrement != null && bidIncrement > 0) {
return currentBid + bidIncrement;
}
// Fallback: 5% increment
return currentBid * 1.05;
}
/// Check if current bid is below estimate (potential bargain)
public boolean isBelowEstimate() {
if (estimatedMin == null || estimatedMin == 0) return false;
return currentBid < (estimatedMin / 100.0);
}
/// Check if current bid exceeds estimate (overvalued)
public boolean isAboveEstimate() {
if (estimatedMax == null || estimatedMax == 0) return false;
return currentBid > (estimatedMax / 100.0);
}
/// Calculate interest-to-bid conversion rate
public double getInterestToBidRatio() {
if (followersCount == null || followersCount == 0) return 0.0;
return currentBid > 0 ? 100.0 : 0.0;
}
/// Determine lot popularity level
public String getPopularityLevel() {
if (followersCount == null) return "UNKNOWN";
if (followersCount > 50) return "HIGH";
if (followersCount > 20) return "MEDIUM";
if (followersCount > 5) return "LOW";
return "MINIMAL";
}
/// Check if lot is a "sleeper" (high interest, low bids)
public boolean isSleeperLot() {
return followersCount != null && followersCount > 10 && currentBid < 100;
}
/// Calculate estimated value range midpoint
public Double getEstimatedMidpoint() {
if (estimatedMin == null || estimatedMax == null) return null;
return (estimatedMin + estimatedMax) / 200.0; // Convert from cents
}
/// Calculate price vs estimate ratio (for analytics)
public Double getPriceVsEstimateRatio() {
Double midpoint = getEstimatedMidpoint();
if (midpoint == null || midpoint == 0 || currentBid == 0) return null;
return (currentBid / midpoint) * 100.0;
}
/// Factory method for creating a basic Lot without intelligence fields (for tests and backward compatibility)
public static Lot basic(
long saleId, long lotId, String title, String description,
String manufacturer, String type, int year, String category,
double currentBid, String currency, String url,
LocalDateTime closingTime, boolean closingNotified) {
return new Lot(
saleId, lotId, null, title, description, manufacturer, type, year, category,
currentBid, currency, url, closingTime, closingNotified,
null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null,
null, null
);
}
}

81
src/main/java/auctiora/LotEnrichmentScheduler.java Normal file
View File

@@ -0,0 +1,81 @@
package auctiora;
import io.quarkus.scheduler.Scheduled;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
/**
* Scheduled tasks for enriching lots with GraphQL intelligence data.
* Uses dynamic frequencies based on lot closing times:
* - Critical (< 1 hour): Every 5 minutes
* - Urgent (< 6 hours): Every 30 minutes
* - Normal (< 24 hours): Every 2 hours
* - All lots: Every 6 hours
*/
@Slf4j
@ApplicationScoped
public class LotEnrichmentScheduler {
@Inject LotEnrichmentService enrichmentService;
/**
* Enriches lots closing within 1 hour - HIGH PRIORITY
* Runs every 5 minutes
*/
@Scheduled(cron = "0 */5 * * * ?")
public void enrichCriticalLots() {
try {
log.debug("Enriching critical lots (closing < 1 hour)");
int enriched = enrichmentService.enrichClosingSoonLots(1);
if (enriched > 0) log.info("Enriched {} critical lots", enriched);
} catch (Exception e) {
log.error("Failed to enrich critical lots", e);
}
}
/**
* Enriches lots closing within 6 hours - MEDIUM PRIORITY
* Runs every 30 minutes
*/
@Scheduled(cron = "0 */30 * * * ?")
public void enrichUrgentLots() {
try {
log.debug("Enriching urgent lots (closing < 6 hours)");
int enriched = enrichmentService.enrichClosingSoonLots(6);
if (enriched > 0) log.info("Enriched {} urgent lots", enriched);
} catch (Exception e) {
log.error("Failed to enrich urgent lots", e);
}
}
/**
* Enriches lots closing within 24 hours - NORMAL PRIORITY
* Runs every 2 hours
*/
@Scheduled(cron = "0 0 */2 * * ?")
public void enrichDailyLots() {
try {
log.debug("Enriching daily lots (closing < 24 hours)");
int enriched = enrichmentService.enrichClosingSoonLots(24);
if (enriched > 0) log.info("Enriched {} daily lots", enriched);
} catch (Exception e) {
log.error("Failed to enrich daily lots", e);
}
}
/**
* Enriches all active lots - LOW PRIORITY
* Runs every 6 hours to keep all data fresh
*/
@Scheduled(cron = "0 0 */6 * * ?")
public void enrichAllLots() {
try {
log.info("Starting full enrichment of all lots");
int enriched = enrichmentService.enrichAllActiveLots();
log.info("Full enrichment complete: {} lots updated", enriched);
} catch (Exception e) {
log.error("Failed to enrich all lots", e);
}
}
}

201
src/main/java/auctiora/LotEnrichmentService.java Normal file
View File

@@ -0,0 +1,201 @@
package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Service for enriching lots with intelligence data from GraphQL API.
* Updates existing lot records with followers, estimates, velocity, etc.
*/
@Slf4j
@ApplicationScoped
public class LotEnrichmentService {
@Inject TroostwijkGraphQLClient graphQLClient;
@Inject DatabaseService db;
/**
* Enriches a single lot with GraphQL intelligence data
*/
public boolean enrichLot(Lot lot) {
if (lot.displayId() == null || lot.displayId().isBlank()) {
log.debug("Cannot enrich lot {} - missing displayId", lot.lotId());
return false;
}
try {
var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId());
if (intelligence == null) {
log.debug("No intelligence data for lot {}", lot.displayId());
return false;
}
// Merge intelligence with existing lot data
var enrichedLot = mergeLotWithIntelligence(lot, intelligence);
db.upsertLotWithIntelligence(enrichedLot);
log.debug("Enriched lot {} with GraphQL data", lot.lotId());
return true;
} catch (Exception e) {
log.warn("Failed to enrich lot {}: {}", lot.lotId(), e.getMessage());
return false;
}
}
/**
* Enriches multiple lots sequentially
* @param lots List of lots to enrich
* @return Number of successfully enriched lots
*/
public int enrichLotsBatch(List<Lot> lots) {
if (lots.isEmpty()) {
return 0;
}
log.info("Enriching {} lots via GraphQL", lots.size());
int enrichedCount = 0;
for (var lot : lots) {
if (lot.displayId() == null || lot.displayId().isBlank()) {
log.debug("Skipping lot {} - missing displayId", lot.lotId());
continue;
}
try {
var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId());
if (intelligence != null) {
var enrichedLot = mergeLotWithIntelligence(lot, intelligence);
db.upsertLotWithIntelligence(enrichedLot);
enrichedCount++;
} else {
log.debug("No intelligence data for lot {}", lot.displayId());
}
} catch (Exception e) {
log.warn("Failed to enrich lot {}: {}", lot.displayId(), e.getMessage());
}
// Small delay to respect rate limits (handled by RateLimitedHttpClient)
}
log.info("Successfully enriched {}/{} lots", enrichedCount, lots.size());
return enrichedCount;
}
/**
* Enriches lots closing soon (within specified hours) with higher priority
*/
public int enrichClosingSoonLots(int hoursUntilClose) {
try {
var allLots = db.getAllLots();
var closingSoon = allLots.stream()
.filter(lot -> lot.closingTime() != null)
.filter(lot -> {
long minutes = lot.minutesUntilClose();
return minutes > 0 && minutes <= hoursUntilClose * 60;
})
.toList();
if (closingSoon.isEmpty()) {
log.debug("No lots closing within {} hours", hoursUntilClose);
return 0;
}
log.info("Enriching {} lots closing within {} hours", closingSoon.size(), hoursUntilClose);
return enrichLotsBatch(closingSoon);
} catch (Exception e) {
log.error("Failed to enrich closing soon lots: {}", e.getMessage());
return 0;
}
}
/**
* Enriches all active lots (can be slow for large datasets)
*/
public int enrichAllActiveLots() {
try {
var allLots = db.getAllLots();
log.info("Enriching all {} active lots", allLots.size());
// Process in batches to avoid overwhelming the API
int batchSize = 50;
int totalEnriched = 0;
for (int i = 0; i < allLots.size(); i += batchSize) {
int end = Math.min(i + batchSize, allLots.size());
List<Lot> batch = allLots.subList(i, end);
int enriched = enrichLotsBatch(batch);
totalEnriched += enriched;
// Small delay between batches to respect rate limits
if (end < allLots.size()) {
Thread.sleep(1000);
}
}
log.info("Finished enriching all lots. Total enriched: {}/{}", totalEnriched, allLots.size());
return totalEnriched;
} catch (Exception e) {
log.error("Failed to enrich all lots: {}", e.getMessage());
return 0;
}
}
/**
* Merges existing lot data with GraphQL intelligence
*/
private Lot mergeLotWithIntelligence(Lot lot, LotIntelligence intel) {
return new Lot(
lot.saleId(),
lot.lotId(),
lot.displayId(), // Preserve displayId
lot.title(),
lot.description(),
lot.manufacturer(),
lot.type(),
lot.year(),
lot.category(),
lot.currentBid(),
lot.currency(),
lot.url(),
lot.closingTime(),
lot.closingNotified(),
// HIGH PRIORITY FIELDS from GraphQL
intel.followersCount(),
intel.estimatedMin(),
intel.estimatedMax(),
intel.nextBidStepInCents(),
intel.condition(),
intel.categoryPath(),
intel.cityLocation(),
intel.countryCode(),
// MEDIUM PRIORITY FIELDS
intel.biddingStatus(),
intel.appearance(),
intel.packaging(),
intel.quantity(),
intel.vat(),
intel.buyerPremiumPercentage(),
intel.remarks(),
// BID INTELLIGENCE FIELDS
intel.startingBid(),
intel.reservePrice(),
intel.reserveMet(),
intel.bidIncrement(),
intel.viewCount(),
intel.firstBidTime(),
intel.lastBidTime(),
intel.bidVelocity(),
null, // condition_score (computed separately)
null // provenance_docs (computed separately)
);
}
}

33
src/main/java/auctiora/LotIntelligence.java Normal file
View File

@@ -0,0 +1,33 @@
package auctiora;
import java.time.LocalDateTime;
/**
* Record holding enriched intelligence data fetched from GraphQL API
*/
public record LotIntelligence(
long lotId,
Integer followersCount,
Double estimatedMin,
Double estimatedMax,
Long nextBidStepInCents,
String condition,
String categoryPath,
String cityLocation,
String countryCode,
String biddingStatus,
String appearance,
String packaging,
Long quantity,
Double vat,
Double buyerPremiumPercentage,
String remarks,
Double startingBid,
Double reservePrice,
Boolean reserveMet,
Double bidIncrement,
Integer viewCount,
LocalDateTime firstBidTime,
LocalDateTime lastBidTime,
Double bidVelocity
) {}

170
src/main/java/auctiora/NotificationService.java Normal file
View File

@@ -0,0 +1,170 @@
package auctiora;
import javax.mail.Authenticator;
import javax.mail.Message;
import javax.mail.PasswordAuthentication;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import lombok.extern.slf4j.Slf4j;
import java.awt.SystemTray;
import java.awt.Toolkit;
import java.awt.TrayIcon;
import java.util.Date;
import java.util.Properties;
@Slf4j
public record NotificationService(Config cfg) {
// Extra convenience constructor: raw string → Config
public NotificationService(String raw) {
this(Config.parse(raw));
}
public void sendNotification(String msg, String title, int prio) {
if (cfg.useDesktop()) sendDesktop(title, msg, prio);
if (cfg.useEmail()) sendEmail(title, msg, prio);
}
private void sendDesktop(String title, String msg, int prio) {
try {
if (!SystemTray.isSupported()) {
log.info("Desktop not supported: {}", title);
return;
}
var tray = SystemTray.getSystemTray();
var icon = new TrayIcon(
Toolkit.getDefaultToolkit().createImage(new byte[0]),
"notify"
);
icon.setImageAutoSize(true);
tray.add(icon);
var type = prio > 0 ? TrayIcon.MessageType.WARNING : TrayIcon.MessageType.INFO;
icon.displayMessage(title, msg, type);
// Remove tray icon asynchronously to avoid blocking the caller
int delayMs = Integer.getInteger("auctiora.desktop.delay.ms", 0);
if (delayMs <= 0) {
var t = new Thread(() -> {
try {
Thread.sleep(50);
} catch (InterruptedException ignored) {
}
try {
tray.remove(icon);
} catch (Exception ignored) {
}
}, "tray-remove");
t.setDaemon(true);
t.start();
} else {
try {
Thread.sleep(delayMs);
} catch (InterruptedException ignored) {
} finally {
try {
tray.remove(icon);
} catch (Exception ignored) {
}
}
}
log.info("Desktop notification: {}", title);
} catch (Exception e) {
log.warn("Desktop failed: {}", e.getMessage());
}
}
private void sendEmail(String title, String msg, int prio) {
try {
var props = new Properties();
props.put("mail.smtp.auth", "true");
props.put("mail.smtp.starttls.enable", "true");
props.put("mail.smtp.starttls.required", "true");
props.put("mail.smtp.host", "smtp.gmail.com");
props.put("mail.smtp.port", "587");
props.put("mail.smtp.ssl.trust", "smtp.gmail.com");
props.put("mail.smtp.ssl.protocols", "TLSv1.2");
// Connection timeouts (configurable; short during tests, longer otherwise)
int smtpTimeoutMs = Integer.getInteger("auctiora.smtp.timeout.ms", isUnderTest() ? 200 : 10000);
String t = String.valueOf(smtpTimeoutMs);
props.put("mail.smtp.connectiontimeout", t);
props.put("mail.smtp.timeout", t);
props.put("mail.smtp.writetimeout", t);
var session = Session.getInstance(props, new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(cfg.smtpUsername(), cfg.smtpPassword());
}
});
var m = new MimeMessage(session);
m.setFrom(new InternetAddress(cfg.smtpUsername()));
m.setRecipients(Message.RecipientType.TO, InternetAddress.parse(cfg.toEmail()));
m.setSubject("[Troostwijk] " + title);
m.setText(msg);
m.setSentDate(new Date());
if (prio > 0) {
m.setHeader("X-Priority", "1");
m.setHeader("Importance", "High");
}
Transport.send(m);
log.info("Email notification sent: {}", title);
} catch (javax.mail.AuthenticationFailedException e) {
log.warn("Email authentication failed - check Gmail App Password: {}", e.getMessage());
} catch (javax.mail.MessagingException e) {
log.warn("Email connection failed (network/firewall issue?): {}", e.getMessage());
} catch (Exception e) {
log.warn("Email failed: {}", e.getMessage());
}
}
public record Config(
boolean useDesktop,
boolean useEmail,
String smtpUsername,
String smtpPassword,
String toEmail
) {
public static Config parse(String raw) {
if ("desktop".equalsIgnoreCase(raw)) {
return new Config(true, false, null, null, null);
}
if (raw != null && raw.startsWith("smtp:")) {
var p = raw.split(":", -1);
if (p.length < 4) {
throw new IllegalArgumentException("Format: smtp:username:password:toEmail");
}
return new Config(true, true, p[1], p[2], p[3]);
}
throw new IllegalArgumentException("Use 'desktop' or 'smtp:username:password:toEmail'");
}
}
private static boolean isUnderTest() {
try {
// Explicit override
if (Boolean.getBoolean("auctiora.test")) return true;
// Maven Surefire commonly sets this property
if (System.getProperty("surefire.test.class.path") != null) return true;
// Fallback: check classpath hint
String cp = System.getProperty("java.class.path", "");
return cp.contains("surefire") || cp.contains("junit");
} catch (Exception ignored) {
return false;
}
}
}

244
src/main/java/auctiora/ObjectDetectionService.java Normal file
View File

@@ -0,0 +1,244 @@
package auctiora;
import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import lombok.extern.slf4j.Slf4j;
import nu.pattern.OpenCV;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgcodecs.Imgcodecs;
import java.io.Console;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import static org.opencv.dnn.Dnn.DNN_BACKEND_OPENCV;
import static org.opencv.dnn.Dnn.DNN_TARGET_CPU;
/**
* Service for performing object detection on images using OpenCV's DNN
* module. The DNN module can load pre‑trained models from several
* frameworks (Darknet, TensorFlow, ONNX, etc.). Here
* we load a YOLO model (Darknet) by specifying the configuration and
* weights files. For each image we run a forward pass and return a
* list of detected class labels.
*
* If model files are not found, the service operates in disabled mode
* and returns empty lists.
*/
@Slf4j
public class ObjectDetectionService {
private Net net;
private List<String> classNames;
private boolean enabled;
private int warnCount = 0;
private static final int MAX_WARNINGS = 5;
private static boolean openCvLoaded = false;
private final String cfgPath;
private final String weightsPath;
private final String classNamesPath;
ObjectDetectionService(String cfgPath, String weightsPath, String classNamesPath) throws IOException {
this.cfgPath = cfgPath;
this.weightsPath = weightsPath;
this.classNamesPath = classNamesPath;
}
@PostConstruct
void init() {
// Load OpenCV native libraries first
if (!openCvLoaded) {
try {
OpenCV.loadLocally();
openCvLoaded = true;
log.info("✓ OpenCV {} loaded successfully", Core.VERSION);
} catch (Exception e) {
log.warn("⚠️ Object detection disabled: OpenCV native libraries not loaded");
enabled = false;
net = null;
classNames = new ArrayList<>();
return;
}
}
initializeModel();
}
private void initializeModel() {
// Check if model files exist
var cfgFile = Paths.get(cfgPath);
var weightsFile = Paths.get(weightsPath);
var classNamesFile = Paths.get(classNamesPath);
if (!Files.exists(cfgFile) || !Files.exists(weightsFile) || !Files.exists(classNamesFile)) {
log.info("⚠️ Object detection disabled: YOLO model files not found");
log.info(" Expected files:");
log.info(" - {}", cfgPath);
log.info(" - {}", weightsPath);
log.info(" - {}", classNamesPath);
log.info(" Scraper will continue without image analysis.");
enabled = false;
net = null;
classNames = new ArrayList<>();
return;
}
try {
// Load network
net = Dnn.readNetFromDarknet(cfgPath, weightsPath);
// Try to use GPU/CUDA if available, fallback to CPU
try {
net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA);
net.setPreferableTarget(Dnn.DNN_TARGET_CUDA);
log.info("✓ Object detection enabled with YOLO (CUDA/GPU acceleration)");
} catch (Exception e) {
// CUDA not available, try Vulkan for AMD GPUs
try {
net.setPreferableBackend(Dnn.DNN_BACKEND_VKCOM);
net.setPreferableTarget(Dnn.DNN_TARGET_VULKAN);
log.info("✓ Object detection enabled with YOLO (Vulkan/GPU acceleration)");
} catch (Exception e2) {
// GPU not available, fallback to CPU
net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(DNN_TARGET_CPU);
log.info("✓ Object detection enabled with YOLO (CPU only)");
}
}
// Load class names (one per line)
classNames = Files.readAllLines(classNamesFile);
enabled = true;
} catch (UnsatisfiedLinkError e) {
log.error("⚠️ Object detection disabled: OpenCV native libraries not loaded", e);
enabled = false;
net = null;
classNames = new ArrayList<>();
} catch (Exception e) {
log.error("⚠️ Object detection disabled: " + e.getMessage(), e);
enabled = false;
net = null;
classNames = new ArrayList<>();
}
}
/**
* Detects objects in the given image file and returns a list of
* human‑readable labels. Only detections above a confidence
* threshold are returned. For brevity this method omits drawing
* bounding boxes. See the OpenCV DNN documentation for details on
* post‑processing【784097309529506†L324-L344】.
*
* @param imagePath absolute path to the image
* @return list of detected class names (empty if detection disabled)
*/
List<String> detectObjects(String imagePath) {
if (!enabled) {
return new ArrayList<>();
}
List<String> labels = new ArrayList<>();
var image = Imgcodecs.imread(imagePath);
if (image.empty()) return labels;
// Create a 4D blob from the image
var blob = Dnn.blobFromImage(image, 1.0 / 255.0, new Size(416, 416), new Scalar(0, 0, 0), true, false);
net.setInput(blob);
List<Mat> outs = new ArrayList<>();
var outNames = getOutputLayerNames(net);
net.forward(outs, outNames);
// Post‑process: for each detection compute score and choose class
var confThreshold = 0.5f;
for (var out : outs) {
// YOLO output shape: [num_detections, 85] where 85 = 4 (bbox) + 1 (objectness) + 80 (classes)
int numDetections = out.rows();
int numElements = out.cols();
int expectedLength = 5 + classNames.size();
if (numElements < expectedLength) {
// Rate-limit warnings to prevent thread blocking from excessive logging
if (warnCount < MAX_WARNINGS) {
log.warn("Output matrix has wrong dimensions: expected {} columns, got {}. Output shape: [{}, {}]",
expectedLength, numElements, numDetections, numElements);
warnCount++;
if (warnCount == MAX_WARNINGS) {
log.warn("Suppressing further dimension warnings (reached {} warnings)", MAX_WARNINGS);
}
}
continue;
}
for (var i = 0; i < numDetections; i++) {
// Get entire row (all 85 elements)
var data = new double[numElements];
for (int j = 0; j < numElements; j++) {
data[j] = out.get(i, j)[0];
}
// Extract objectness score (index 4) and class scores (index 5+)
double objectness = data[4];
if (objectness < confThreshold) {
continue; // Skip low-confidence detections
}
// Extract class scores
var scores = new double[classNames.size()];
System.arraycopy(data, 5, scores, 0, Math.min(scores.length, data.length - 5));
var classId = argMax(scores);
var confidence = scores[classId] * objectness; // Combine objectness with class confidence
if (confidence > confThreshold) {
var label = classNames.get(classId);
if (!labels.contains(label)) {
labels.add(label);
}
}
}
}
// Release resources
image.release();
blob.release();
for (var out : outs) {
out.release();
}
return labels;
}
/**
* Returns the indexes of the output layers in the network. YOLO
* automatically discovers its output layers; other models may require
* manually specifying them【784097309529506†L356-L365】.
*/
private List<String> getOutputLayerNames(Net net) {
List<String> names = new ArrayList<>();
var outLayers = net.getUnconnectedOutLayers().toList();
var layersNames = net.getLayerNames();
for (var i : outLayers) {
names.add(layersNames.get(i - 1));
}
return names;
}
/**
* Returns the index of the maximum value in the array.
*/
private int argMax(double[] array) {
var best = 0;
var max = array[0];
for (var i = 1; i < array.length; i++) {
if (array[i] > max) {
max = array[i];
best = i;
}
}
return best;
}
}

309
src/main/java/auctiora/QuarkusWorkflowScheduler.java Normal file
View File

@@ -0,0 +1,309 @@
package auctiora;
import io.quarkus.runtime.StartupEvent;
import io.quarkus.scheduler.Scheduled;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import java.util.List;
/**
* Quarkus-based Workflow Scheduler using @Scheduled annotations.
* Replaces the manual ScheduledExecutorService with Quarkus Scheduler.
*
* This class coordinates all scheduled workflows using Quarkus's built-in
* scheduling capabilities with cron expressions.
*/
@ApplicationScoped
public class QuarkusWorkflowScheduler {
private static final Logger LOG = Logger.getLogger(QuarkusWorkflowScheduler.class);
@Inject DatabaseService db;
@Inject NotificationService notifier;
@Inject ObjectDetectionService detector;
@Inject ImageProcessingService imageProcessor;
@Inject LotEnrichmentService enrichmentService;
@ConfigProperty(name = "auction.database.path") String databasePath;
/**
* Triggered on application startup to enrich existing lots with bid intelligence
*/
void onStart(@Observes StartupEvent ev) {
LOG.info("🚀 Application started - triggering initial lot enrichment...");
// Run enrichment in background thread to not block startup
new Thread(() -> {
try {
Thread.sleep(5000); // Wait 5 seconds for application to fully start
LOG.info("Starting full lot enrichment in background...");
int enriched = enrichmentService.enrichAllActiveLots();
LOG.infof("✓ Startup enrichment complete: %d lots enriched", enriched);
} catch (Exception e) {
LOG.errorf(e, "❌ Startup enrichment failed: %s", e.getMessage());
}
}).start();
}
/**
* Workflow 1: Import Scraper Data
* Cron: Every 30 minutes (0 -/30 - - - ?)
* Purpose: Import new auctions and lots from external scraper
*/
@Scheduled(cron = "{auction.workflow.scraper-import.cron}", identity = "scraper-import")
void importScraperData() {
try {
LOG.info("📥 [WORKFLOW 1] Importing scraper data...");
var start = System.currentTimeMillis();
// Import auctions
var auctions = db.importAuctionsFromScraper();
LOG.infof(" → Imported %d auctions", auctions.size());
// Import lots
var lots = db.importLotsFromScraper();
LOG.infof(" → Imported %d lots", lots.size());
// Check for images needing detection
var images = db.getImagesNeedingDetection();
LOG.infof(" → Found %d images needing detection", images.size());
var duration = System.currentTimeMillis() - start;
LOG.infof(" ✓ Scraper import completed in %dms", duration);
// Trigger notification if significant data imported
if (auctions.size() > 0 || lots.size() > 10) {
notifier.sendNotification(
String.format("Imported %d auctions, %d lots", auctions.size(), lots.size()),
"Data Import Complete",
0
);
}
} catch (Exception e) {
LOG.errorf(e, " ❌ Scraper import failed: %s", e.getMessage());
}
}
/**
* Workflow 2: Process Pending Images
* Cron: Every 1 hour (0 0 * * * ?)
* Purpose: Run object detection on images already downloaded by scraper
*/
@Scheduled(cron = "{auction.workflow.image-processing.cron}", identity = "image-processing")
void processImages() {
try {
LOG.info("🖼️ [WORKFLOW 2] Processing pending images...");
var start = System.currentTimeMillis();
// Get images that have been downloaded but need object detection
var pendingImages = db.getImagesNeedingDetection();
if (pendingImages.isEmpty()) {
LOG.info(" → No pending images to process");
return;
}
// Limit batch size to prevent thread blocking (max 100 images per run)
final int MAX_BATCH_SIZE = 100;
int totalPending = pendingImages.size();
if (totalPending > MAX_BATCH_SIZE) {
LOG.infof(" → Found %d pending images, processing first %d (batch limit)",
totalPending, MAX_BATCH_SIZE);
pendingImages = pendingImages.subList(0, MAX_BATCH_SIZE);
} else {
LOG.infof(" → Processing %d images", totalPending);
}
var processed = 0;
var detected = 0;
var failed = 0;
for (var image : pendingImages) {
try {
// Run object detection on already-downloaded image
if (imageProcessor.processImage(image.id(), image.filePath(), image.lotId())) {
processed++;
// Check if objects were detected
var labels = db.getImageLabels(image.id());
if (labels != null && !labels.isEmpty()) {
detected++;
// Send notification for interesting detections
if (labels.size() >= 3) {
notifier.sendNotification(
String.format("Lot %d: Detected %s",
image.lotId(),
String.join(", ", labels)),
"Objects Detected",
0
);
}
}
} else {
failed++;
}
// Rate limiting (lighter since no network I/O)
Thread.sleep(100);
} catch (Exception e) {
failed++;
LOG.warnf(" ⚠️ Failed to process image: %s", e.getMessage());
}
}
var duration = System.currentTimeMillis() - start;
LOG.infof(" ✓ Processed %d/%d images, detected objects in %d, failed %d (%.1fs)",
processed, totalPending, detected, failed, duration / 1000.0);
if (totalPending > MAX_BATCH_SIZE) {
LOG.infof(" → %d images remaining for next run", totalPending - MAX_BATCH_SIZE);
}
} catch (Exception e) {
LOG.errorf(e, " ❌ Image processing failed: %s", e.getMessage());
}
}
/**
* Workflow 3: Monitor Bids
* Cron: Every 15 minutes (0 -/15 * * * ?)
* Purpose: Check for bid changes and send notifications
*/
@Scheduled(cron = "{auction.workflow.bid-monitoring.cron}", identity = "bid-monitoring")
void monitorBids() {
try {
LOG.info("💰 [WORKFLOW 3] Monitoring bids...");
var start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
LOG.infof(" → Checking %d active lots", activeLots.size());
// Note: In production, this would call Troostwijk API
// For now, we just track what's in the database
// The external scraper updates bids, we just notify
var duration = System.currentTimeMillis() - start;
LOG.infof(" ✓ Bid monitoring completed in %dms", duration);
} catch (Exception e) {
LOG.errorf(e, " ❌ Bid monitoring failed: %s", e.getMessage());
}
}
/**
* Workflow 4: Check Closing Times
* Cron: Every 5 minutes (0 -/5 * * * ?)
* Purpose: Send alerts for lots closing soon
*/
@Scheduled(cron = "{auction.workflow.closing-alerts.cron}", identity = "closing-alerts")
void checkClosingTimes() {
try {
LOG.info("⏰ [WORKFLOW 4] Checking closing times...");
var start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
var alertsSent = 0;
for (var lot : activeLots) {
if (lot.closingTime() == null) continue;
var minutesLeft = lot.minutesUntilClose();
// Alert for lots closing in 5 minutes
if (minutesLeft <= 5 && minutesLeft > 0 && !lot.closingNotified()) {
var message = String.format("Kavel %d sluit binnen %d min.",
lot.lotId(), minutesLeft);
notifier.sendNotification(message, "Lot Closing Soon", 1);
// Mark as notified
var updated = Lot.basic(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
lot.currentBid(), lot.currency(), lot.url(),
lot.closingTime(), true
);
db.updateLotNotificationFlags(updated);
alertsSent++;
}
}
var duration = System.currentTimeMillis() - start;
LOG.infof(" → Sent %d closing alerts in %dms", alertsSent, duration);
} catch (Exception e) {
LOG.errorf(e, " ❌ Closing alerts failed: %s", e.getMessage());
}
}
/**
* Event-driven trigger: New auction discovered
*/
public void onNewAuctionDiscovered(AuctionInfo auction) {
LOG.infof("📣 EVENT: New auction discovered - %s", auction.title());
try {
db.upsertAuction(auction);
notifier.sendNotification(
String.format("New auction: %s\nLocation: %s\nLots: %d",
auction.title(), auction.location(), auction.lotCount()),
"New Auction Discovered",
0
);
} catch (Exception e) {
LOG.errorf(e, " ❌ Failed to handle new auction: %s", e.getMessage());
}
}
/**
* Event-driven trigger: Bid change detected
*/
public void onBidChange(Lot lot, double previousBid, double newBid) {
LOG.infof("📣 EVENT: Bid change on lot %d (€%.2f → €%.2f)",
lot.lotId(), previousBid, newBid);
try {
db.updateLotCurrentBid(lot);
notifier.sendNotification(
String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previousBid),
"Kavel Bieding Update",
0
);
} catch (Exception e) {
LOG.errorf(e, " ❌ Failed to handle bid change: %s", e.getMessage());
}
}
/**
* Event-driven trigger: Objects detected in image
*/
public void onObjectsDetected(int lotId, List<String> labels) {
LOG.infof("📣 EVENT: Objects detected in lot %d - %s",
lotId, String.join(", ", labels));
try {
if (labels.size() >= 2) {
notifier.sendNotification(
String.format("Lot %d contains: %s", lotId, String.join(", ", labels)),
"Objects Detected",
0
);
}
} catch (Exception e) {
LOG.errorf(e, " ❌ Failed to send detection notification: %s", e.getMessage());
}
}
}

246
src/main/java/auctiora/RateLimitedHttpClient.java Normal file
View File

@@ -0,0 +1,246 @@
package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/**
* Rate-limited HTTP client that enforces per-host request limits.
*
* Features:
* - Per-host rate limiting (configurable max requests per second)
* - Request counting and monitoring
* - Thread-safe using semaphores
* - Automatic host extraction from URLs
*
* This prevents overloading external services like Troostwijk and getting blocked.
*/
@ApplicationScoped
public class RateLimitedHttpClient {
private static final Logger LOG = Logger.getLogger(RateLimitedHttpClient.class);
private final HttpClient httpClient;
private final Map<String, RateLimiter> rateLimiters;
private final Map<String, RequestStats> requestStats;
@ConfigProperty(name = "auction.http.rate-limit.default-max-rps", defaultValue = "2")
int defaultMaxRequestsPerSecond;
@ConfigProperty(name = "auction.http.rate-limit.troostwijk-max-rps", defaultValue = "1")
int troostwijkMaxRequestsPerSecond;
@ConfigProperty(name = "auction.http.timeout-seconds", defaultValue = "30")
int timeoutSeconds;
public RateLimitedHttpClient() {
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(30))
.build();
this.rateLimiters = new ConcurrentHashMap<>();
this.requestStats = new ConcurrentHashMap<>();
}
/**
* Sends a GET request with automatic rate limiting based on host.
*/
public HttpResponse<String> sendGet(String url) throws IOException, InterruptedException {
var request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofString());
}
/**
* Sends a request for binary data (like images) with rate limiting.
*/
public HttpResponse<byte[]> sendGetBytes(String url) throws IOException, InterruptedException {
var request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofByteArray());
}
/**
* Sends any HTTP request with automatic rate limiting.
*/
public <T> HttpResponse<T> send(HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler)
throws IOException, InterruptedException {
var host = extractHost(request.uri());
var limiter = getRateLimiter(host);
var stats = getRequestStats(host);
// Enforce rate limit (blocks if necessary)
limiter.acquire();
// Track request
stats.incrementTotal();
var startTime = System.currentTimeMillis();
try {
var response = httpClient.send(request, bodyHandler);
var duration = System.currentTimeMillis() - startTime;
stats.recordSuccess(duration);
LOG.debugf("HTTP %d %s %s (%dms)",
response.statusCode(), request.method(), host, duration);
// Track rate limit violations (429 = Too Many Requests)
if (response.statusCode() == 429) {
stats.incrementRateLimited();
LOG.warnf("⚠️ Rate limited by %s (HTTP 429)", host);
}
return response;
} catch (IOException | InterruptedException e) {
stats.incrementFailed();
LOG.warnf("❌ HTTP request failed for %s: %s", host, e.getMessage());
throw e;
}
}
/**
* Gets or creates a rate limiter for a specific host.
*/
private RateLimiter getRateLimiter(String host) {
return rateLimiters.computeIfAbsent(host, h -> {
var maxRps = getMaxRequestsPerSecond(h);
LOG.infof("Initializing rate limiter for %s: %d req/s", h, maxRps);
return new RateLimiter(maxRps);
});
}
/**
* Gets or creates request stats for a specific host.
*/
private RequestStats getRequestStats(String host) {
return requestStats.computeIfAbsent(host, h -> new RequestStats(h));
}
/**
* Determines max requests per second for a given host.
*/
private int getMaxRequestsPerSecond(String host) {
return host.contains("troostwijk") ? troostwijkMaxRequestsPerSecond : defaultMaxRequestsPerSecond;
}
private String extractHost(URI uri) {
return uri.getHost() != null ? uri.getHost() : uri.toString();
}
public Map<String, RequestStats> getAllStats() {
return Map.copyOf(requestStats);
}
public RequestStats getStats(String host) {
return requestStats.get(host);
}
/**
* Rate limiter implementation using token bucket algorithm.
* Allows burst traffic up to maxRequestsPerSecond, then enforces steady rate.
*/
private static class RateLimiter {
private final Semaphore semaphore;
private final int maxRequestsPerSecond;
private final long intervalNanos;
RateLimiter(int maxRequestsPerSecond) {
this.maxRequestsPerSecond = maxRequestsPerSecond;
this.intervalNanos = TimeUnit.SECONDS.toNanos(1) / maxRequestsPerSecond;
this.semaphore = new Semaphore(maxRequestsPerSecond);
// Refill tokens periodically
startRefillThread();
}
void acquire() throws InterruptedException {
semaphore.acquire();
// Enforce minimum delay between requests
var delayMillis = intervalNanos / 1_000_000;
if (delayMillis > 0) {
Thread.sleep(delayMillis);
}
}
private void startRefillThread() {
var refillThread = new Thread(() -> {
while (!Thread.currentThread().isInterrupted()) {
try {
Thread.sleep(1000); // Refill every second
var toRelease = maxRequestsPerSecond - semaphore.availablePermits();
if (toRelease > 0) {
semaphore.release(toRelease);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}, "RateLimiter-Refill");
refillThread.setDaemon(true);
refillThread.start();
}
}
public static final class RequestStats {
private final String host;
private final AtomicLong totalRequests = new AtomicLong(0);
private final AtomicLong successfulRequests = new AtomicLong(0);
private final AtomicLong failedRequests = new AtomicLong(0);
private final AtomicLong rateLimitedRequests = new AtomicLong(0);
private final AtomicLong totalDurationMs = new AtomicLong(0);
RequestStats(String host) {
this.host = host;
}
void incrementTotal() { totalRequests.incrementAndGet(); }
void recordSuccess(long durationMs) {
successfulRequests.incrementAndGet();
totalDurationMs.addAndGet(durationMs);
}
void incrementFailed() { failedRequests.incrementAndGet(); }
void incrementRateLimited() { rateLimitedRequests.incrementAndGet(); }
public String getHost() { return host; }
public long getTotalRequests() { return totalRequests.get(); }
public long getSuccessfulRequests() { return successfulRequests.get(); }
public long getFailedRequests() { return failedRequests.get(); }
public long getRateLimitedRequests() { return rateLimitedRequests.get(); }
public long getAverageDurationMs() {
var successful = successfulRequests.get();
return successful > 0 ? totalDurationMs.get() / successful : 0;
}
@Override
public String toString() {
return String.format("%s: %d total, %d success, %d failed, %d rate-limited, avg %dms",
host, getTotalRequests(), getSuccessfulRequests(),
getFailedRequests(), getRateLimitedRequests(), getAverageDurationMs());
}
}
}

196
src/main/java/auctiora/ScraperDataAdapter.java Normal file
View File

@@ -0,0 +1,196 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
@Slf4j
public class ScraperDataAdapter {
private static final DateTimeFormatter[] TIMESTAMP_FORMATS = {
DateTimeFormatter.ISO_LOCAL_DATE_TIME,
DateTimeFormatter.ISO_DATE_TIME,
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
};
static AuctionInfo fromScraperAuction(ResultSet rs) throws SQLException {
// Parse "A7-39813" → auctionId=39813, type="A7"
var auctionIdStr = rs.getString("auction_id");
var auctionId = extractNumericId(auctionIdStr);
var type = extractTypePrefix(auctionIdStr);
// Split "Cluj-Napoca, RO" → city="Cluj-Napoca", country="RO"
var location = rs.getString("location");
var locationParts = parseLocation(location);
var city = locationParts[0];
var country = locationParts[1];
// Map field names
var lotCount = getIntOrDefault(rs, "lots_count", 0);
var closingTime = parseTimestamp(getStringOrNull(rs, "first_lot_closing_time"));
return new AuctionInfo(
auctionId,
rs.getString("title"),
location,
city,
country,
rs.getString("url"),
type,
lotCount,
closingTime
);
}
public static Lot fromScraperLot(ResultSet rs) throws SQLException {
var lotIdStr = rs.getString("lot_id"); // Full display ID (e.g., "A1-34732-49")
var lotId = extractNumericId(lotIdStr);
var saleId = extractNumericId(rs.getString("auction_id"));
var bidStr = getStringOrNull(rs, "current_bid");
var bid = parseBidAmount(bidStr);
var currency = parseBidCurrency(bidStr);
var closing = parseTimestamp(getStringOrNull(rs, "closing_time"));
return new Lot(
saleId,
lotId,
lotIdStr, // Store full displayId for GraphQL queries
rs.getString("title"),
getStringOrDefault(rs, "description", ""),
getStringOrDefault(rs, "manufacturer", ""),
getStringOrDefault(rs, "type", ""),
getIntOrDefault(rs, "year", 0),
getStringOrDefault(rs, "category", ""),
bid,
currency,
rs.getString("url"),
closing,
getBooleanOrDefault(rs, "closing_notified", false),
// New intelligence fields - set to null for now
null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null,
null, null
);
}
public static long extractNumericId(String id) {
if (id == null || id.isBlank()) return 0L;
// Remove the type prefix (e.g., "A7-") first, then extract all digits
// "A7-39813" → "39813" → 39813
// "A1-28505-5" → "28505-5" → "285055"
var afterPrefix = id.indexOf('-') >= 0 ? id.substring(id.indexOf('-') + 1) : id;
var digits = afterPrefix.replaceAll("\\D+", "");
if (digits.isEmpty()) return 0L;
// Check if number is too large for long (> 19 digits or value > Long.MAX_VALUE)
if (digits.length() > 19) {
log.debug("ID too large for long, skipping: {}", id);
return 0L;
}
try {
return Long.parseLong(digits);
} catch (NumberFormatException e) {
log.debug("Invalid numeric ID, skipping: {}", id);
return 0L;
}
}
private static String extractTypePrefix(String id) {
if (id == null) return "";
var idx = id.indexOf('-');
return idx > 0 ? id.substring(0, idx) : "";
}
private static String[] parseLocation(String location) {
if (location == null || location.isBlank()) return new String[]{ "", "" };
var parts = location.split(",\\s*");
var city = parts[0].trim();
var country = parts.length > 1 ? parts[parts.length - 1].trim() : "";
return new String[]{ city, country };
}
private static double parseBidAmount(String bid) {
if (bid == null || bid.isBlank() || bid.toLowerCase().contains("no")) return 0.0;
var cleaned = bid.replaceAll("[^0-9.]", "");
try {
return cleaned.isEmpty() ? 0.0 : Double.parseDouble(cleaned);
} catch (NumberFormatException e) {
return 0.0;
}
}
private static String parseBidCurrency(String bid) {
if (bid == null) return "EUR";
return bid.contains("€") ? "EUR"
: bid.contains("$") ? "USD"
: bid.contains("£") ? "GBP"
: "EUR";
}
public static LocalDateTime parseTimestamp(String ts) {
if (ts == null || ts.isBlank()) return null;
String trimmed = ts.trim();
String tsLower = trimmed.toLowerCase();
// Filter out known invalid values
if (tsLower.equals("gap") || tsLower.equals("null") || tsLower.equals("n/a") ||
tsLower.equals("unknown") || tsLower.equals("tbd")) {
log.debug("Skipping invalid timestamp value: {}", ts);
return null;
}
// Recognize numeric epoch values (seconds or milliseconds)
if (trimmed.matches("^[0-9]{10,16}$")) {
try {
long epoch = Long.parseLong(trimmed);
java.time.Instant instant;
// Heuristic: 13 digits (>= 10^12) => milliseconds, else seconds
if (trimmed.length() >= 13) {
instant = java.time.Instant.ofEpochMilli(epoch);
} else {
instant = java.time.Instant.ofEpochSecond(epoch);
}
return java.time.LocalDateTime.ofInstant(instant, java.time.ZoneId.systemDefault());
} catch (NumberFormatException e) {
// fall through to formatter parsing
}
}
// Try known text formats (ISO first)
for (var fmt : TIMESTAMP_FORMATS) {
try {
return LocalDateTime.parse(trimmed, fmt);
} catch (DateTimeParseException ignored) { }
}
log.debug("Unable to parse timestamp: {}", ts);
return null;
}
private static String getStringOrNull(ResultSet rs, String col) throws SQLException {
return rs.getString(col);
}
private static String getStringOrDefault(ResultSet rs, String col, String def) throws SQLException {
var v = rs.getString(col);
return v != null ? v : def;
}
private static int getIntOrDefault(ResultSet rs, String col, int def) throws SQLException {
var v = rs.getInt(col);
return rs.wasNull() ? def : v;
}
private static boolean getBooleanOrDefault(ResultSet rs, String col, boolean def) throws SQLException {
var v = rs.getInt(col);
return rs.wasNull() ? def : v != 0;
}
}

74
src/main/java/auctiora/StatusResource.java Normal file
View File

@@ -0,0 +1,74 @@
package auctiora;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import lombok.extern.slf4j.Slf4j;
import nu.pattern.OpenCV;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.opencv.core.Core;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Map;
@Slf4j
@Path("/api")
public class StatusResource {
private static final DateTimeFormatter FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z")
.withZone(ZoneId.systemDefault());
@ConfigProperty(name = "application.version", defaultValue = "1.0-SNAPSHOT") String appVersion;
@ConfigProperty(name = "application.groupId") String groupId;
@ConfigProperty(name = "application.artifactId") String artifactId;
@ConfigProperty(name = "application.version") String version;
public record StatusResponse(
String groupId,
String artifactId,
String version,
String status,
String timestamp,
String mvnVersion,
String javaVersion,
String os,
String openCvVersion
) { }
@GET
@Path("/status")
@Produces(MediaType.APPLICATION_JSON)
public StatusResponse getStatus() {
return new StatusResponse(groupId, artifactId, version,
"running",
FORMATTER.format(Instant.now()),
appVersion,
System.getProperty("java.version"),
System.getProperty("os.name") + " " + System.getProperty("os.arch"),
getOpenCvVersion()
);
}
@GET
@Path("/hello")
@Produces(MediaType.APPLICATION_JSON)
public Map<String, String> sayHello() {
return Map.of(
"message", "Hello from Scrape-UI!",
"timestamp", FORMATTER.format(Instant.now()),
"openCvVersion", getOpenCvVersion()
);
}
private String getOpenCvVersion() {
try {
// OpenCV is already loaded by AuctionMonitorProducer
return Core.VERSION;
} catch (Exception e) {
return "Not loaded";
}
}
}

378
src/main/java/auctiora/TroostwijkGraphQLClient.java Normal file
View File

@@ -0,0 +1,378 @@
package auctiora;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
/**
* GraphQL client for fetching enriched lot data from Troostwijk API.
* Fetches intelligence fields: followers, estimates, bid velocity, condition, etc.
*/
@Slf4j
@ApplicationScoped
public class TroostwijkGraphQLClient {
private static final String GRAPHQL_ENDPOINT = "https://storefront.tbauctions.com/storefront/graphql";
private static final String LOCALE = "nl";
private static final String PLATFORM = "TWK";
private static final ObjectMapper objectMapper = new ObjectMapper();
@Inject
RateLimitedHttpClient rateLimitedClient;
/**
* Fetches enriched lot data from GraphQL API
* @param displayId The lot display ID (e.g., "A1-34732-49")
* @param lotId The numeric lot ID for mapping back to database
* @return LotIntelligence with enriched fields, or null if failed
*/
public LotIntelligence fetchLotIntelligence(String displayId, long lotId) {
if (displayId == null || displayId.isBlank()) {
log.debug("Cannot fetch intelligence for null/blank displayId");
return null;
}
try {
var query = buildLotQuery();
var variables = buildVariables(displayId);
// Proper GraphQL request format with query and variables
var requestBody = String.format(
"{\"query\":\"%s\",\"variables\":%s}",
escapeJson(query),
variables
);
var request = java.net.http.HttpRequest.newBuilder()
.uri(java.net.URI.create(GRAPHQL_ENDPOINT))
.header("Content-Type", "application/json")
.header("Accept", "application/json")
.POST(java.net.http.HttpRequest.BodyPublishers.ofString(requestBody))
.build();
var response = rateLimitedClient.send(
request,
java.net.http.HttpResponse.BodyHandlers.ofString()
);
if (response == null || response.body() == null) {
log.debug("No response from GraphQL for lot {}", displayId);
return null;
}
log.debug("GraphQL response for lot {}: {}", displayId, response.body().substring(0, Math.min(200, response.body().length())));
return parseLotIntelligence(response.body(), lotId);
} catch (Exception e) {
log.warn("Failed to fetch lot intelligence for {}: {}", lotId, e.getMessage());
return null;
}
}
/**
* Batch fetch multiple lots in a single query (more efficient)
*/
public List<LotIntelligence> fetchBatchLotIntelligence(List<Long> lotIds) {
List<LotIntelligence> results = new ArrayList<>();
// Split into batches of 50 to avoid query size limits
var batchSize = 50;
for (var i = 0; i < lotIds.size(); i += batchSize) {
var end = Math.min(i + batchSize, lotIds.size());
var batch = lotIds.subList(i, end);
try {
var query = buildBatchLotQuery(batch);
var requestBody = String.format("{\"query\":\"%s\"}",
escapeJson(query));
var request = java.net.http.HttpRequest.newBuilder()
.uri(java.net.URI.create(GRAPHQL_ENDPOINT))
.header("Content-Type", "application/json")
.POST(java.net.http.HttpRequest.BodyPublishers.ofString(requestBody))
.build();
var response = rateLimitedClient.send(
request,
java.net.http.HttpResponse.BodyHandlers.ofString()
);
if (response != null && response.body() != null) {
results.addAll(parseBatchLotIntelligence(response.body(), batch));
}
} catch (Exception e) {
log.warn("Failed to fetch batch lot intelligence: {}", e.getMessage());
}
}
return results;
}
private String buildLotQuery() {
// Match Python scraper's LOT_BIDDING_QUERY structure
// Uses lotDetails(displayId:...) instead of lot(id:...)
return """
query LotBiddingData($lotDisplayId: String!, $locale: String!, $platform: Platform!) {
lotDetails(displayId: $lotDisplayId, locale: $locale, platform: $platform) {
id
displayId
followersCount
currentBidInCents
nextBidStepInCents
condition
description
biddingStatus
buyersPremium
viewCount
estimatedValueInCentsMin
estimatedValueInCentsMax
categoryPath
location {
city
country
}
biddingStatistics {
numberOfBids
}
}
}
""".replaceAll("\\s+", " ");
}
private String buildVariables(String displayId) {
return String.format("""
{
"lotDisplayId": "%s",
"locale": "%s",
"platform": "%s"
}
""", displayId, LOCALE, PLATFORM).replaceAll("\\s+", " ");
}
private String buildBatchLotQuery(List<Long> lotIds) {
var query = new StringBuilder("query {");
for (var i = 0; i < lotIds.size(); i++) {
query.append(String.format("""
lot%d: lot(id: %d) {
id
followersCount
estimatedMin
estimatedMax
nextBidStepInCents
condition
categoryPath
city
countryCode
biddingStatus
vat
buyerPremiumPercentage
viewCount
bidsCount
}
""", i, lotIds.get(i)));
}
query.append("}");
return query.toString().replaceAll("\\s+", " ");
}
private LotIntelligence parseLotIntelligence(String json, long lotId) {
try {
// Check if response is HTML (error page) instead of JSON
if (json != null && json.trim().startsWith("<")) {
log.debug("GraphQL API returned HTML instead of JSON - likely auth required or wrong endpoint");
return null;
}
var root = objectMapper.readTree(json);
var lotNode = root.path("data").path("lotDetails");
if (lotNode.isMissingNode()) {
log.debug("No lotDetails in GraphQL response");
return null;
}
// Extract location from nested object
var locationNode = lotNode.path("location");
var city = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "city");
var countryCode = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "country");
// Extract bids count from nested biddingStatistics
var statsNode = lotNode.path("biddingStatistics");
var bidsCount = statsNode.isMissingNode() ? null : getIntOrNull(statsNode, "numberOfBids");
// Convert cents to euros for estimates
var estimatedMinCents = getLongOrNull(lotNode, "estimatedValueInCentsMin");
var estimatedMaxCents = getLongOrNull(lotNode, "estimatedValueInCentsMax");
var estimatedMin = estimatedMinCents != null ? estimatedMinCents.doubleValue() : null;
var estimatedMax = estimatedMaxCents != null ? estimatedMaxCents.doubleValue() : null;
return new LotIntelligence(
lotId,
getIntOrNull(lotNode, "followersCount"),
estimatedMin,
estimatedMax,
getLongOrNull(lotNode, "nextBidStepInCents"),
getStringOrNull(lotNode, "condition"),
getStringOrNull(lotNode, "categoryPath"),
city,
countryCode,
getStringOrNull(lotNode, "biddingStatus"),
null, // appearance - not in API response
null, // packaging - not in API response
null, // quantity - not in API response
null, // vat - not in API response
null, // buyerPremiumPercentage - could extract from buyersPremium
null, // remarks - not in API response
null, // startingBid - not in API response
null, // reservePrice - not in API response
null, // reserveMet - not in API response
null, // bidIncrement - not in API response
getIntOrNull(lotNode, "viewCount"),
null, // firstBidTime - not in API response
null, // lastBidTime - not in API response
null // bidVelocity - could calculate from bidsCount if we had timing data
);
} catch (Exception e) {
log.warn("Failed to parse lot intelligence: {}", e.getMessage());
return null;
}
}
private List<LotIntelligence> parseBatchLotIntelligence(String json, List<Long> lotIds) {
List<LotIntelligence> results = new ArrayList<>();
try {
var root = objectMapper.readTree(json);
var data = root.path("data");
for (var i = 0; i < lotIds.size(); i++) {
var lotNode = data.path("lot" + i);
if (!lotNode.isMissingNode()) {
var intelligence = parseLotIntelligenceFromNode(lotNode, lotIds.get(i));
if (intelligence != null) {
results.add(intelligence);
}
}
}
} catch (Exception e) {
log.warn("Failed to parse batch lot intelligence: {}", e.getMessage());
}
return results;
}
private LotIntelligence parseLotIntelligenceFromNode(JsonNode lotNode, long lotId) {
try {
return new LotIntelligence(
lotId,
getIntOrNull(lotNode, "followersCount"),
getDoubleOrNull(lotNode, "estimatedMin"),
getDoubleOrNull(lotNode, "estimatedMax"),
getLongOrNull(lotNode, "nextBidStepInCents"),
getStringOrNull(lotNode, "condition"),
getStringOrNull(lotNode, "categoryPath"),
getStringOrNull(lotNode, "city"),
getStringOrNull(lotNode, "countryCode"),
getStringOrNull(lotNode, "biddingStatus"),
null, // appearance not in batch query
null, // packaging not in batch query
null, // quantity not in batch query
getDoubleOrNull(lotNode, "vat"),
getDoubleOrNull(lotNode, "buyerPremiumPercentage"),
null, // remarks not in batch query
null, // startingBid not in batch query
null, // reservePrice not in batch query
null, // reserveMet not in batch query
null, // bidIncrement not in batch query
getIntOrNull(lotNode, "viewCount"),
null, // firstBidTime not in batch query
null, // lastBidTime not in batch query
calculateBidVelocity(lotNode)
);
} catch (Exception e) {
log.warn("Failed to parse lot node: {}", e.getMessage());
return null;
}
}
private Double calculateBidVelocity(JsonNode lotNode) {
try {
var bidsCount = getIntOrNull(lotNode, "bidsCount");
var firstBidStr = getStringOrNull(lotNode, "firstBidTime");
if (bidsCount == null || firstBidStr == null || bidsCount == 0) {
return null;
}
var firstBid = parseDateTime(firstBidStr);
if (firstBid == null) return null;
var hoursElapsed = java.time.Duration.between(firstBid, LocalDateTime.now()).toHours();
if (hoursElapsed == 0) return (double) bidsCount;
return (double) bidsCount / hoursElapsed;
} catch (Exception e) {
return null;
}
}
private LocalDateTime parseDateTime(String dateStr) {
if (dateStr == null || dateStr.isBlank()) return null;
try {
return LocalDateTime.parse(dateStr, DateTimeFormatter.ISO_DATE_TIME);
} catch (Exception e) {
return null;
}
}
private String escapeJson(String str) {
return str.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r")
.replace("\t", "\\t");
}
private Integer getIntOrNull(JsonNode node, String field) {
var fieldNode = node.path(field);
return fieldNode.isNumber() ? fieldNode.asInt() : null;
}
private Long getLongOrNull(JsonNode node, String field) {
var fieldNode = node.path(field);
return fieldNode.isNumber() ? fieldNode.asLong() : null;
}
private Double getDoubleOrNull(JsonNode node, String field) {
var fieldNode = node.path(field);
return fieldNode.isNumber() ? fieldNode.asDouble() : null;
}
private String getStringOrNull(JsonNode node, String field) {
var fieldNode = node.path(field);
return fieldNode.isTextual() ? fieldNode.asText() : null;
}
private Boolean getBooleanOrNull(JsonNode node, String field) {
var fieldNode = node.path(field);
return fieldNode.isBoolean() ? fieldNode.asBoolean() : null;
}
}

132
src/main/java/auctiora/TroostwijkMonitor.java Normal file
View File

@@ -0,0 +1,132 @@
package auctiora;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.sql.SQLException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Slf4j
public class TroostwijkMonitor {
private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list";
RateLimitedHttpClient httpClient;
ObjectMapper objectMapper;
@Getter DatabaseService db;
NotificationService notifier;
ObjectDetectionService detector;
ImageProcessingService imageProcessor;
ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
var t = new Thread(r, "troostwijk-monitor-thread");
t.setDaemon(true);
return t;
});
public TroostwijkMonitor(String databasePath,
String notificationConfig,
String yoloCfgPath,
String yoloWeightsPath,
String classNamesPath)
throws SQLException, IOException {
httpClient = new RateLimitedHttpClient();
objectMapper = new ObjectMapper();
db = new DatabaseService(databasePath);
notifier = new NotificationService(notificationConfig);
detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
imageProcessor = new ImageProcessingService(db, detector);
db.ensureSchema();
}
public void scheduleMonitoring() {
scheduler.scheduleAtFixedRate(this::monitorAllLots, 0, 1, TimeUnit.HOURS);
log.info("✓ Monitoring service started");
}
private void monitorAllLots() {
try {
var activeLots = db.getActiveLots();
log.info("Monitoring {} active lots …", activeLots.size());
for (var lot : activeLots) {
checkAndUpdateLot(lot);
}
} catch (Exception e) {
log.error("Error during scheduled monitoring", e);
}
}
private void checkAndUpdateLot(Lot lot) {
refreshLotBid(lot);
var minutesLeft = lot.minutesUntilClose();
if (minutesLeft < 30) {
if (minutesLeft <= 5 && !lot.closingNotified()) {
notifier.sendNotification(
"Kavel " + lot.lotId() + " sluit binnen " + minutesLeft + " min.",
"Lot nearing closure", 1);
db.updateLotNotificationFlags(lot.withClosingNotified(true));
}
scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
}
}
private void refreshLotBid(Lot lot) {
try {
var url = LOT_API +
"?batchSize=1&listType=7&offset=0&sortOption=0" +
"&saleID=" + lot.saleId() +
"&parentID=0&relationID=0&buildversion=201807311" +
"&lotID=" + lot.lotId();
var resp = httpClient.sendGet(url);
if (resp.statusCode() != 200) return;
var root = objectMapper.readTree(resp.body());
var results = root.path("results");
if (results.isArray() && results.size() > 0) {
var newBid = results.get(0).path("cb").asDouble();
if (Double.compare(newBid, lot.currentBid()) > 0) {
var previous = lot.currentBid();
var updatedLot = lot.withCurrentBid(newBid);
db.updateLotCurrentBid(updatedLot);
var msg = String.format(
"Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previous);
notifier.sendNotification(msg, "Kavel bieding update", 0);
}
}
} catch (IOException | InterruptedException e) {
log.warn("Failed to refresh bid for lot {}", lot.lotId(), e);
if (e instanceof InterruptedException) Thread.currentThread().interrupt();
}
}
public void printDatabaseStats() {
try {
var allLots = db.getAllLots();
var imageCount = db.getImageCount();
log.info("📊 Database Summary: total lots = {}, total images = {}",
allLots.size(), imageCount);
if (!allLots.isEmpty()) {
var sum = allLots.stream().mapToDouble(Lot::currentBid).sum();
log.info("Total current bids: €{}", String.format("%.2f", sum));
}
} catch (Exception e) {
log.warn("Could not retrieve database stats", e);
}
}
public void processPendingImages() {
imageProcessor.processPendingImages();
}
}

378
src/main/java/auctiora/ValuationAnalyticsResource.java Normal file
View File

@@ -0,0 +1,378 @@
package auctiora;
import jakarta.inject.Inject;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.jboss.logging.Logger;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.stream.Collectors;
/**
* REST API for Auction Valuation Analytics
* Implements the mathematical framework for fair market value calculation,
* undervaluation detection, and bidding strategy recommendations.
*/
@Path("/api/analytics")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public class ValuationAnalyticsResource {
private static final Logger LOG = Logger.getLogger(ValuationAnalyticsResource.class);
@Inject
DatabaseService db;
/**
* GET /api/analytics/valuation/health
* Health check endpoint to verify API availability
*/
@GET
@Path("/valuation/health")
public Response healthCheck() {
return Response.ok(Map.of(
"status", "healthy",
"service", "valuation-analytics",
"timestamp", java.time.LocalDateTime.now().toString()
)).build();
}
/**
* POST /api/analytics/valuation
* Main valuation endpoint that calculates FMV, undervaluation score,
* predicted final price, and bidding strategy
*/
@POST
@Path("/valuation")
public Response calculateValuation(ValuationRequest request) {
try {
LOG.infof("Valuation request for lot: %s", request.lotId);
var startTime = System.currentTimeMillis();
// Step 1: Fetch comparable sales from database
var comparables = fetchComparables(request);
// Step 2: Calculate Fair Market Value (FMV)
var fmv = calculateFairMarketValue(request, comparables);
// Step 3: Calculate undervaluation score
var undervaluationScore = calculateUndervaluationScore(request, fmv.value);
// Step 4: Predict final price
var prediction = calculateFinalPrice(request, fmv.value);
// Step 5: Generate bidding strategy
var strategy = generateBiddingStrategy(request, fmv, prediction);
// Step 6: Compile response
var response = new ValuationResponse();
response.lotId = request.lotId;
response.timestamp = LocalDateTime.now().toString();
response.fairMarketValue = fmv;
response.undervaluationScore = undervaluationScore;
response.pricePrediction = prediction;
response.biddingStrategy = strategy;
response.parameters = request;
var duration = System.currentTimeMillis() - startTime;
LOG.infof("Valuation completed in %d ms", duration);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Valuation calculation failed", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* Fetches comparable lots from database based on category, manufacturer,
* year, and condition similarity
*/
private List<ComparableLot> fetchComparables(ValuationRequest req) {
// TODO: Replace with actual database query
// For now, return mock data simulating real comparables
return List.of(
new ComparableLot("NL-2023-4451", 8200.0, 8.0, 2016, 1, 30),
new ComparableLot("BE-2023-9823", 7800.0, 7.0, 2014, 0, 45),
new ComparableLot("DE-2024-1234", 8500.0, 9.0, 2017, 1, 60),
new ComparableLot("NL-2023-5678", 7500.0, 6.0, 2013, 0, 25),
new ComparableLot("BE-2024-7890", 7900.0, 7.5, 2015, 1, 15),
new ComparableLot("NL-2023-2345", 8100.0, 8.5, 2016, 0, 40),
new ComparableLot("DE-2024-4567", 8300.0, 7.0, 2015, 1, 55),
new ComparableLot("BE-2023-3456", 7700.0, 6.5, 2014, 0, 35)
);
}
/**
* Formula: FMV = Σ(P_i · ω_c · ω_t · ω_p · ω_h) / Σ(ω_c · ω_t · ω_p · ω_h)
* Where weights are exponential/logistic functions of similarity
*/
private FairMarketValue calculateFairMarketValue(ValuationRequest req, List<ComparableLot> comparables) {
var weightedSum = 0.0;
var weightSum = 0.0;
List<WeightedComparable> weightedComps = new ArrayList<>();
for (var comp : comparables) {
// Condition weight: ω_c = exp(-λ_c · |C_target - C_i|)
var omegaC = Math.exp(-0.693 * Math.abs(req.conditionScore - comp.conditionScore));
// Time weight: ω_t = exp(-λ_t · |T_target - T_i|)
var omegaT = Math.exp(-0.048 * Math.abs(req.manufacturingYear - comp.manufacturingYear));
// Provenance weight: ω_p = 1 + δ_p · (P_target - P_i)
var omegaP = 1 + 0.15 * ((req.provenanceDocs > 0 ? 1 : 0) - comp.hasProvenance);
// Historical weight: ω_h = 1 / (1 + e^(-kh · (D_i - D_median)))
var omegaH = 1.0 / (1 + Math.exp(-0.01 * (comp.daysAgo - 40)));
var totalWeight = omegaC * omegaT * omegaP * omegaH;
weightedSum += comp.finalPrice * totalWeight;
weightSum += totalWeight;
// Store for transparency
weightedComps.add(new WeightedComparable(comp, totalWeight, omegaC, omegaT, omegaP, omegaH));
}
var baseFMV = weightSum > 0 ? weightedSum / weightSum : (req.estimatedMin + req.estimatedMax) / 2;
// Apply condition multiplier: M_cond = exp(α_c · √C_target - β_c)
var conditionMultiplier = Math.exp(0.15 * Math.sqrt(req.conditionScore) - 0.40);
baseFMV *= conditionMultiplier;
// Apply provenance premium: Δ_prov = V_base · (η_0 + η_1 · ln(1 + N_docs))
if (req.provenanceDocs > 0) {
var provenancePremium = 0.08 + 0.035 * Math.log(1 + req.provenanceDocs);
baseFMV *= (1 + provenancePremium);
}
var fmv = new FairMarketValue();
fmv.value = Math.round(baseFMV * 100.0) / 100.0;
fmv.conditionMultiplier = Math.round(conditionMultiplier * 1000.0) / 1000.0;
fmv.provenancePremium = req.provenanceDocs > 0 ? 0.08 + 0.035 * Math.log(1 + req.provenanceDocs) : 0.0;
fmv.comparablesUsed = comparables.size();
fmv.confidence = calculateFMVConfidence(comparables.size(), weightSum);
fmv.weightedComparables = weightedComps;
return fmv;
}
/**
* Calculates undervaluation score:
* U_score = (FMV - P_current)/FMV · σ_market · (1 + B_velocity/10) · ln(1 + W_watch/W_bid)
*/
private double calculateUndervaluationScore(ValuationRequest req, double fmv) {
if (fmv <= 0) return 0.0;
var priceGap = (fmv - req.currentBid) / fmv;
var velocityFactor = 1 + req.bidVelocity / 10.0;
var watchRatio = Math.log(1 + req.watchCount / Math.max(req.bidCount, 1));
var uScore = priceGap * req.marketVolatility * velocityFactor * watchRatio;
return Math.max(0.0, Math.round(uScore * 1000.0) / 1000.0);
}
/**
* Predicts final price: P̂_final = FMV · (1 + ε_bid + ε_time + ε_comp)
* Where each epsilon represents auction dynamics
*/
private PricePrediction calculateFinalPrice(ValuationRequest req, double fmv) {
// Bid momentum error: ε_bid = tanh(φ_1 · Λ_b - φ_2 · P_current/FMV)
var epsilonBid = Math.tanh(0.15 * req.bidVelocity - 0.10 * (req.currentBid / fmv));
// Time pressure error: ε_time = ψ · exp(-t_close/30)
var epsilonTime = 0.20 * Math.exp(-req.minutesUntilClose / 30.0);
// Competition error: ε_comp = ρ · ln(1 + W_watch/50)
var epsilonComp = 0.08 * Math.log(1 + req.watchCount / 50.0);
var predictedPrice = fmv * (1 + epsilonBid + epsilonTime + epsilonComp);
// 95% confidence interval: ± 1.96 · σ_residual
var residualStdDev = fmv * 0.08; // Mock residual standard deviation
var ciLower = predictedPrice - 1.96 * residualStdDev;
var ciUpper = predictedPrice + 1.96 * residualStdDev;
var pred = new PricePrediction();
pred.predictedPrice = Math.round(predictedPrice * 100.0) / 100.0;
pred.confidenceIntervalLower = Math.round(ciLower * 100.0) / 100.0;
pred.confidenceIntervalUpper = Math.round(ciUpper * 100.0) / 100.0;
pred.components = Map.of(
"bidMomentum", Math.round(epsilonBid * 1000.0) / 1000.0,
"timePressure", Math.round(epsilonTime * 1000.0) / 1000.0,
"competition", Math.round(epsilonComp * 1000.0) / 1000.0
);
return pred;
}
/**
* Generates optimal bidding strategy based on market conditions
*/
private BiddingStrategy generateBiddingStrategy(ValuationRequest req, FairMarketValue fmv, PricePrediction pred) {
var strategy = new BiddingStrategy();
// Determine competition level
if (req.bidVelocity > 5.0) {
strategy.competitionLevel = "HIGH";
strategy.recommendedTiming = "FINAL_30_SECONDS";
strategy.maxBid = pred.predictedPrice + 50; // Slight overbid for hot lots
strategy.riskFactors = List.of("Bidding war likely", "Sniping detected");
} else if (req.minutesUntilClose < 10) {
strategy.competitionLevel = "EXTREME";
strategy.recommendedTiming = "FINAL_10_SECONDS";
strategy.maxBid = pred.predictedPrice * 1.02;
strategy.riskFactors = List.of("Last-minute sniping", "Price volatility");
} else {
strategy.competitionLevel = "MEDIUM";
strategy.recommendedTiming = "FINAL_10_MINUTES";
// Adjust max bid based on undervaluation
var undervaluationScore = calculateUndervaluationScore(req, fmv.value);
if (undervaluationScore > 0.25) {
// Aggressive strategy for undervalued lots
strategy.maxBid = fmv.value * (1 + 0.05); // Conservative overbid
strategy.analysis = "Significant undervaluation detected. Consider aggressive bidding.";
} else {
// Standard strategy
strategy.maxBid = fmv.value * (1 + 0.03);
}
strategy.riskFactors = List.of("Standard competition level");
}
// Generate detailed analysis
strategy.analysis = String.format(
"Bid velocity is %.1f bids/min with %d watchers. %s competition detected. " +
"Predicted final: €%.2f (%.0f%% confidence).",
req.bidVelocity,
req.watchCount,
strategy.competitionLevel,
pred.predictedPrice,
fmv.confidence * 100
);
// Round the max bid
strategy.maxBid = Math.round(strategy.maxBid * 100.0) / 100.0;
strategy.recommendedTimingText = strategy.recommendedTiming.replace("_", " ");
return strategy;
}
/**
* Calculates confidence score based on number and quality of comparables
*/
private double calculateFMVConfidence(int comparableCount, double totalWeight) {
var confidence = 0.5; // Base confidence
// Boost for more comparables
confidence += Math.min(comparableCount * 0.05, 0.3);
// Boost for high total weight (good matches)
confidence += Math.min(totalWeight / comparableCount * 0.1, 0.2);
// Cap at 0.95
return Math.min(confidence, 0.95);
}
// ================== DTO Classes ==================
public static class ValuationRequest {
public String lotId;
public double currentBid;
public double conditionScore; // C_target ∈ [0,10]
public int manufacturingYear; // T_target
public int watchCount; // W_watch
public int bidCount = 1; // W_bid (default 1 to avoid division by zero)
public double marketVolatility = 0.15; // σ_market ∈ [0,1]
public double bidVelocity; // Λ_b (bids/min)
public int minutesUntilClose; // t_close
public int provenanceDocs = 0; // N_docs
public double estimatedMin;
public double estimatedMax;
// Optional: override parameters for sensitivity analysis
public Map<String, Double> sensitivityParams;
}
public static class ValuationResponse {
public String lotId;
public String timestamp;
public FairMarketValue fairMarketValue;
public double undervaluationScore;
public PricePrediction pricePrediction;
public BiddingStrategy biddingStrategy;
public ValuationRequest parameters;
public long calculationTimeMs;
}
public static class FairMarketValue {
public double value;
public double conditionMultiplier;
public double provenancePremium;
public int comparablesUsed;
public double confidence; // [0,1]
public List<WeightedComparable> weightedComparables;
}
public static class WeightedComparable {
public String comparableLotId;
public double finalPrice;
public double totalWeight;
public Map<String, Double> components;
public WeightedComparable(ComparableLot comp, double totalWeight, double omegaC, double omegaT, double omegaP, double omegaH) {
this.comparableLotId = comp.lotId;
this.finalPrice = comp.finalPrice;
this.totalWeight = Math.round(totalWeight * 1000.0) / 1000.0;
this.components = Map.of(
"conditionWeight", Math.round(omegaC * 1000.0) / 1000.0,
"timeWeight", Math.round(omegaT * 1000.0) / 1000.0,
"provenanceWeight", Math.round(omegaP * 1000.0) / 1000.0,
"historicalWeight", Math.round(omegaH * 1000.0) / 1000.0
);
}
}
public static class PricePrediction {
public double predictedPrice;
public double confidenceIntervalLower;
public double confidenceIntervalUpper;
public Map<String, Double> components; // ε_bid, ε_time, ε_comp
}
public static class BiddingStrategy {
public String competitionLevel; // LOW, MEDIUM, HIGH, EXTREME
public double maxBid;
public String recommendedTiming; // FINAL_10_MINUTES, FINAL_30_SECONDS, etc.
public String recommendedTimingText;
public String analysis;
public List<String> riskFactors;
}
// Helper class for internal comparable representation
private static class ComparableLot {
String lotId;
double finalPrice;
double conditionScore;
int manufacturingYear;
int hasProvenance;
int daysAgo;
public ComparableLot(String lotId, double finalPrice, double conditionScore, int manufacturingYear, int hasProvenance, int daysAgo) {
this.lotId = lotId;
this.finalPrice = finalPrice;
this.conditionScore = conditionScore;
this.manufacturingYear = manufacturingYear;
this.hasProvenance = hasProvenance;
this.daysAgo = daysAgo;
}
}
}

433
src/main/java/auctiora/WorkflowOrchestrator.java Normal file
View File

@@ -0,0 +1,433 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* Orchestrates the complete workflow of auction monitoring, image processing,
* object detection, and notifications.
*
* This class coordinates all services and provides scheduled execution,
* event-driven triggers, and manual workflow execution.
*/
@Slf4j
public class WorkflowOrchestrator {
private final TroostwijkMonitor monitor;
private final DatabaseService db;
private final ImageProcessingService imageProcessor;
private final NotificationService notifier;
private final ObjectDetectionService detector;
private final ScheduledExecutorService scheduler;
private boolean isRunning = false;
/**
* Creates a workflow orchestrator with all necessary services.
*/
public WorkflowOrchestrator(String databasePath, String notificationConfig,
String yoloCfg, String yoloWeights, String yoloClasses)
throws SQLException, IOException {
log.info("🔧 Initializing Workflow Orchestrator...");
// Initialize core services
this.db = new DatabaseService(databasePath);
this.db.ensureSchema();
this.notifier = new NotificationService(notificationConfig);
this.detector = new ObjectDetectionService(yoloCfg, yoloWeights, yoloClasses);
this.imageProcessor = new ImageProcessingService(db, detector);
this.monitor = new TroostwijkMonitor(databasePath, notificationConfig,
yoloCfg, yoloWeights, yoloClasses);
this.scheduler = Executors.newScheduledThreadPool(3);
log.info("✓ Workflow Orchestrator initialized");
}
/**
* Starts all scheduled workflows.
* This is the main entry point for automated operation.
*/
public void startScheduledWorkflows() {
if (isRunning) {
log.info("⚠️ Workflows already running");
return;
}
log.info("\n🚀 Starting Scheduled Workflows...\n");
// Workflow 1: Import scraper data (every 30 minutes)
scheduleScraperDataImport();
// Workflow 2: Process pending images (every 1 hour)
scheduleImageProcessing();
// Workflow 3: Monitor bids (every 15 minutes)
scheduleBidMonitoring();
// Workflow 4: Check closing times (every 5 minutes)
scheduleClosingAlerts();
isRunning = true;
log.info("✓ All scheduled workflows started\n");
}
/**
* Workflow 1: Import Scraper Data
* Frequency: Every 30 minutes
* Purpose: Import new auctions and lots from external scraper
*/
private void scheduleScraperDataImport() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("📥 [WORKFLOW 1] Importing scraper data...");
var start = System.currentTimeMillis();
// Import auctions
var auctions = db.importAuctionsFromScraper();
log.info(" → Imported {} auctions", auctions.size());
// Import lots
var lots = db.importLotsFromScraper();
log.info(" → Imported {} lots", lots.size());
// Check for images needing detection
var images = db.getImagesNeedingDetection();
log.info(" → Found {} images needing detection", images.size());
var duration = System.currentTimeMillis() - start;
log.info(" ✓ Scraper import completed in {}ms\n", duration);
// Trigger notification if significant data imported
if (auctions.size() > 0 || lots.size() > 10) {
notifier.sendNotification(
String.format("Imported %d auctions, %d lots", auctions.size(), lots.size()),
"Data Import Complete",
0
);
}
} catch (Exception e) {
log.info(" ❌ Scraper import failed: {}", e.getMessage());
}
}, 0, 30, TimeUnit.MINUTES);
log.info(" ✓ Scheduled: Scraper Data Import (every 30 min)");
}
/**
* Workflow 2: Process Pending Images
* Frequency: Every 1 hour
* Purpose: Run object detection on images already downloaded by scraper
*/
private void scheduleImageProcessing() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("🖼️ [WORKFLOW 2] Processing pending images...");
var start = System.currentTimeMillis();
// Get images that have been downloaded but need object detection
var pendingImages = db.getImagesNeedingDetection();
if (pendingImages.isEmpty()) {
log.info(" → No pending images to process\n");
return;
}
log.info(" → Processing {} images", pendingImages.size());
var processed = 0;
var detected = 0;
for (var image : pendingImages) {
try {
// Run object detection on already-downloaded image
if (imageProcessor.processImage(image.id(), image.filePath(), image.lotId())) {
processed++;
// Check if objects were detected
var labels = db.getImageLabels(image.id());
if (labels != null && !labels.isEmpty()) {
detected++;
// Send notification for interesting detections
if (labels.size() >= 3) {
notifier.sendNotification(
String.format("Lot %d: Detected %s",
image.lotId(),
String.join(", ", labels)),
"Objects Detected",
0
);
}
}
}
// Rate limiting (lighter since no network I/O)
Thread.sleep(100);
} catch (Exception e) {
log.info(" ⚠\uFE0F Failed to process image: {}", e.getMessage());
}
}
var duration = System.currentTimeMillis() - start;
log.info(String.format(" ✓ Processed %d images, detected objects in %d (%.1fs)\n",
processed, detected, duration / 1000.0));
} catch (Exception e) {
log.info(" ❌ Image processing failed: {}", e.getMessage());
}
}, 5, 60, TimeUnit.MINUTES);
log.info(" ✓ Scheduled: Image Processing (every 1 hour)");
}
/**
* Workflow 3: Monitor Bids
* Frequency: Every 15 minutes
* Purpose: Check for bid changes and send notifications
*/
private void scheduleBidMonitoring() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("💰 [WORKFLOW 3] Monitoring bids...");
var start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
log.info(" → Checking {} active lots", activeLots.size());
var bidChanges = 0;
for (var lot : activeLots) {
// Note: In production, this would call Troostwijk API
// For now, we just track what's in the database
// The external scraper updates bids, we just notify
}
var duration = System.currentTimeMillis() - start;
log.info(String.format(" ✓ Bid monitoring completed in %dms\n", duration));
} catch (Exception e) {
log.info(" ❌ Bid monitoring failed: {}", e.getMessage());
}
}, 2, 15, TimeUnit.MINUTES);
log.info(" ✓ Scheduled: Bid Monitoring (every 15 min)");
}
/**
* Workflow 4: Check Closing Times
* Frequency: Every 5 minutes
* Purpose: Send alerts for lots closing soon
*/
private void scheduleClosingAlerts() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("⏰ [WORKFLOW 4] Checking closing times...");
var start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
var alertsSent = 0;
for (var lot : activeLots) {
if (lot.closingTime() == null) continue;
var minutesLeft = lot.minutesUntilClose();
// Alert for lots closing in 5 minutes
if (minutesLeft <= 5 && minutesLeft > 0 && !lot.closingNotified()) {
var message = String.format("Kavel %d sluit binnen %d min.",
lot.lotId(), minutesLeft);
notifier.sendNotification(message, "Lot Closing Soon", 1);
// Mark as notified
var updated = Lot.basic(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
lot.currentBid(), lot.currency(), lot.url(),
lot.closingTime(), true
);
db.updateLotNotificationFlags(updated);
alertsSent++;
}
}
var duration = System.currentTimeMillis() - start;
log.info(String.format(" → Sent %d closing alerts in %dms\n",
alertsSent, duration));
} catch (Exception e) {
log.info(" ❌ Closing alerts failed: {}", e.getMessage());
}
}, 1, 5, TimeUnit.MINUTES);
log.info(" ✓ Scheduled: Closing Alerts (every 5 min)");
}
/**
* Manual trigger: Run complete workflow once
* Useful for testing or on-demand execution
*/
public void runCompleteWorkflowOnce() {
log.info("\n🔄 Running Complete Workflow (Manual Trigger)...\n");
try {
// Step 1: Import data
log.info("[1/4] Importing scraper data...");
var auctions = db.importAuctionsFromScraper();
var lots = db.importLotsFromScraper();
log.info(" ✓ Imported {} auctions, {} lots", auctions.size(), lots.size());
// Step 2: Process images
log.info("[2/4] Processing pending images...");
monitor.processPendingImages();
log.info(" ✓ Image processing completed");
// Step 3: Check bids
log.info("[3/4] Monitoring bids...");
var activeLots = db.getActiveLots();
log.info(" ✓ Monitored {} lots", activeLots.size());
// Step 4: Check closing times
log.info("[4/4] Checking closing times...");
var closingSoon = 0;
for (var lot : activeLots) {
if (lot.closingTime() != null && lot.minutesUntilClose() < 30) {
closingSoon++;
}
}
log.info(" ✓ Found {} lots closing soon", closingSoon);
log.info("\n✓ Complete workflow finished successfully\n");
} catch (Exception e) {
log.info("\n❌ Workflow failed: {}\n", e.getMessage());
}
}
/**
* Event-driven trigger: New auction discovered
*/
public void onNewAuctionDiscovered(AuctionInfo auction) {
log.info("\uD83D\uDCE3 EVENT: New auction discovered - {}", auction.title());
try {
db.upsertAuction(auction);
notifier.sendNotification(
String.format("New auction: %s\nLocation: %s\nLots: %d",
auction.title(), auction.location(), auction.lotCount()),
"New Auction Discovered",
0
);
} catch (Exception e) {
log.info(" ❌ Failed to handle new auction: {}", e.getMessage());
}
}
/**
* Event-driven trigger: Bid change detected
*/
public void onBidChange(Lot lot, double previousBid, double newBid) {
log.info(String.format("📣 EVENT: Bid change on lot %d (€%.2f → €%.2f)",
lot.lotId(), previousBid, newBid));
try {
db.updateLotCurrentBid(lot);
notifier.sendNotification(
String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previousBid),
"Kavel Bieding Update",
0
);
} catch (Exception e) {
log.info(" ❌ Failed to handle bid change: {}", e.getMessage());
}
}
/**
* Event-driven trigger: Objects detected in image
*/
public void onObjectsDetected(int lotId, List<String> labels) {
log.info(String.format("📣 EVENT: Objects detected in lot %d - %s",
lotId, String.join(", ", labels)));
try {
if (labels.size() >= 2) {
notifier.sendNotification(
String.format("Lot %d contains: %s", lotId, String.join(", ", labels)),
"Objects Detected",
0
);
}
} catch (Exception e) {
log.info(" ❌ Failed to send detection notification: {}", e.getMessage());
}
}
/**
* Prints current workflow status
*/
public void printStatus() {
log.info("\n📊 Workflow Status:");
log.info(" Running: {}", isRunning ? "Yes" : "No");
try {
var auctions = db.getAllAuctions();
var lots = db.getAllLots();
var images = db.getImageCount();
log.info(" Auctions: {}", auctions.size());
log.info(" Lots: {}", lots.size());
log.info(" Images: {}", images);
// Count closing soon
var closingSoon = 0;
for (var lot : lots) {
if (lot.closingTime() != null && lot.minutesUntilClose() < 30) {
closingSoon++;
}
}
log.info(" Closing soon (< 30 min): {}", closingSoon);
} catch (Exception e) {
log.info(" ⚠\uFE0F Could not retrieve status: {}", e.getMessage());
}
IO.println();
}
/**
* Gracefully shuts down all workflows
*/
public void shutdown() {
log.info("\n🛑 Shutting down workflows...");
isRunning = false;
scheduler.shutdown();
try {
if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
scheduler.shutdownNow();
}
log.info("✓ Workflows shut down successfully\n");
} catch (InterruptedException e) {
scheduler.shutdownNow();
Thread.currentThread().interrupt();
}
}
}

159
src/main/java/auctiora/db/AuctionRepository.java Normal file
View File

@@ -0,0 +1,159 @@
package auctiora.db;
import auctiora.AuctionInfo;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Jdbi;
import java.time.LocalDateTime;
import java.util.List;
import auctiora.ScraperDataAdapter;
/**
* Repository for auction-related database operations using JDBI3.
* Handles CRUD operations and queries for auctions.
*/
@Slf4j
@RequiredArgsConstructor
public class AuctionRepository {
private final Jdbi jdbi;
/**
* Inserts or updates an auction record.
* Handles both auction_id conflicts and url uniqueness constraints.
*/
public void upsert(AuctionInfo auction) {
jdbi.useTransaction(handle -> {
try {
// Try INSERT with ON CONFLICT on auction_id
handle.createUpdate("""
INSERT INTO auctions (
auction_id, title, location, city, country, url, type, lot_count, closing_time, discovered_at
) VALUES (
:auctionId, :title, :location, :city, :country, :url, :type, :lotCount, :closingTime, :discoveredAt
)
ON CONFLICT(auction_id) DO UPDATE SET
title = excluded.title,
location = excluded.location,
city = excluded.city,
country = excluded.country,
url = excluded.url,
type = excluded.type,
lot_count = excluded.lot_count,
closing_time = excluded.closing_time
""")
.bind("auctionId", auction.auctionId())
.bind("title", auction.title())
.bind("location", auction.location())
.bind("city", auction.city())
.bind("country", auction.country())
.bind("url", auction.url())
.bind("type", auction.typePrefix())
.bind("lotCount", auction.lotCount())
.bind("closingTime", auction.firstLotClosingTime() != null ? auction.firstLotClosingTime().toString() : null)
.bind("discoveredAt", java.time.Instant.now().getEpochSecond())
.execute();
} catch (Exception e) {
// If UNIQUE constraint on url fails, try updating by url
var errMsg = e.getMessage();
if (errMsg != null && (errMsg.contains("UNIQUE constraint failed") ||
errMsg.contains("PRIMARY KEY constraint failed"))) {
log.debug("Auction conflict detected, attempting update by URL: {}", auction.url());
var updated = handle.createUpdate("""
UPDATE auctions SET
auction_id = :auctionId,
title = :title,
location = :location,
city = :city,
country = :country,
type = :type,
lot_count = :lotCount,
closing_time = :closingTime
WHERE url = :url
""")
.bind("auctionId", auction.auctionId())
.bind("title", auction.title())
.bind("location", auction.location())
.bind("city", auction.city())
.bind("country", auction.country())
.bind("type", auction.typePrefix())
.bind("lotCount", auction.lotCount())
.bind("closingTime", auction.firstLotClosingTime() != null ? auction.firstLotClosingTime().toString() : null)
.bind("url", auction.url())
.execute();
if (updated == 0) {
log.warn("Failed to update auction by URL: {}", auction.url());
}
} else {
log.error("Unexpected error upserting auction: {}", e.getMessage(), e);
throw e;
}
}
});
}
/**
* Retrieves all auctions from the database.
*/
public List<AuctionInfo> getAll() {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT * FROM auctions")
.map((rs, ctx) -> {
var closingStr = rs.getString("closing_time");
LocalDateTime closingTime = auctiora.ScraperDataAdapter.parseTimestamp(closingStr);
return new AuctionInfo(
rs.getLong("auction_id"),
rs.getString("title"),
rs.getString("location"),
rs.getString("city"),
rs.getString("country"),
rs.getString("url"),
rs.getString("type"),
rs.getInt("lot_count"),
closingTime
);
})
.list()
);
}
/**
* Retrieves auctions filtered by country code.
*/
public List<AuctionInfo> getByCountry(String countryCode) {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT * FROM auctions WHERE country = :country")
.bind("country", countryCode)
.map((rs, ctx) -> {
var closingStr = rs.getString("closing_time");
LocalDateTime closingTime = null;
if (closingStr != null && !closingStr.isBlank()) {
try {
closingTime = LocalDateTime.parse(closingStr);
} catch (Exception e) {
log.warn("Invalid closing_time format: {}", closingStr);
}
}
return new AuctionInfo(
rs.getLong("auction_id"),
rs.getString("title"),
rs.getString("location"),
rs.getString("city"),
rs.getString("country"),
rs.getString("url"),
rs.getString("type"),
rs.getInt("lot_count"),
closingTime
);
})
.list()
);
}
}

154
src/main/java/auctiora/db/DatabaseSchema.java Normal file
View File

@@ -0,0 +1,154 @@
package auctiora.db;
import lombok.experimental.UtilityClass;
import org.jdbi.v3.core.Jdbi;
/**
* Database schema DDL definitions for all tables and indexes.
* Uses text blocks (Java 15+) for clean SQL formatting.
*/
@UtilityClass
public class DatabaseSchema {
/**
* Initializes all database tables and indexes if they don't exist.
*/
public void ensureSchema(Jdbi jdbi) {
jdbi.useHandle(handle -> {
// Enable WAL mode for better concurrent access
handle.execute("PRAGMA journal_mode=WAL");
handle.execute("PRAGMA busy_timeout=10000");
handle.execute("PRAGMA synchronous=NORMAL");
createTables(handle);
createIndexes(handle);
});
}
private void createTables(org.jdbi.v3.core.Handle handle) {
// Cache table (for HTTP caching)
handle.execute("""
CREATE TABLE IF NOT EXISTS cache (
url TEXT PRIMARY KEY,
content BLOB,
timestamp REAL,
status_code INTEGER
)""");
// Auctions table (populated by external scraper)
handle.execute("""
CREATE TABLE IF NOT EXISTS auctions (
auction_id TEXT PRIMARY KEY,
url TEXT UNIQUE,
title TEXT,
location TEXT,
lots_count INTEGER,
first_lot_closing_time TEXT,
scraped_at TEXT,
city TEXT,
country TEXT,
type TEXT,
lot_count INTEGER DEFAULT 0,
closing_time TEXT,
discovered_at INTEGER
)""");
// Lots table (populated by external scraper)
handle.execute("""
CREATE TABLE IF NOT EXISTS lots (
lot_id TEXT PRIMARY KEY,
auction_id TEXT,
url TEXT UNIQUE,
title TEXT,
current_bid TEXT,
bid_count INTEGER,
closing_time TEXT,
viewing_time TEXT,
pickup_date TEXT,
location TEXT,
description TEXT,
category TEXT,
scraped_at TEXT,
sale_id INTEGER,
manufacturer TEXT,
type TEXT,
year INTEGER,
currency TEXT DEFAULT 'EUR',
closing_notified INTEGER DEFAULT 0,
starting_bid TEXT,
minimum_bid TEXT,
status TEXT,
brand TEXT,
model TEXT,
attributes_json TEXT,
first_bid_time TEXT,
last_bid_time TEXT,
bid_velocity REAL,
bid_increment REAL,
year_manufactured INTEGER,
condition_score REAL,
condition_description TEXT,
serial_number TEXT,
damage_description TEXT,
followers_count INTEGER DEFAULT 0,
estimated_min_price REAL,
estimated_max_price REAL,
lot_condition TEXT,
appearance TEXT,
estimated_min REAL,
estimated_max REAL,
next_bid_step_cents INTEGER,
condition TEXT,
category_path TEXT,
city_location TEXT,
country_code TEXT,
bidding_status TEXT,
packaging TEXT,
quantity INTEGER,
vat REAL,
buyer_premium_percentage REAL,
remarks TEXT,
reserve_price REAL,
reserve_met INTEGER,
view_count INTEGER,
FOREIGN KEY (auction_id) REFERENCES auctions(auction_id)
)""");
// Images table (populated by external scraper with URLs and local_path)
handle.execute("""
CREATE TABLE IF NOT EXISTS images (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT,
url TEXT,
local_path TEXT,
downloaded INTEGER DEFAULT 0,
labels TEXT,
processed_at INTEGER,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)""");
// Bid history table
handle.execute("""
CREATE TABLE IF NOT EXISTS bid_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT NOT NULL,
bid_amount REAL NOT NULL,
bid_time TEXT NOT NULL,
is_autobid INTEGER DEFAULT 0,
bidder_id TEXT,
bidder_number INTEGER,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)""");
}
private void createIndexes(org.jdbi.v3.core.Handle handle) {
handle.execute("CREATE INDEX IF NOT EXISTS idx_timestamp ON cache(timestamp)");
handle.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)");
handle.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)");
handle.execute("CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id)");
handle.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_lot_url ON images(lot_id, url)");
handle.execute("CREATE INDEX IF NOT EXISTS idx_bid_history_lot_time ON bid_history(lot_id, bid_time)");
handle.execute("CREATE INDEX IF NOT EXISTS idx_bid_history_bidder ON bid_history(bidder_id)");
}
}

137
src/main/java/auctiora/db/ImageRepository.java Normal file
View File

@@ -0,0 +1,137 @@
package auctiora.db;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Jdbi;
import java.time.Instant;
import java.util.List;
/**
* Repository for image-related database operations using JDBI3.
* Handles image storage, object detection labels, and processing status.
*/
@Slf4j
@RequiredArgsConstructor
public class ImageRepository {
private final Jdbi jdbi;
/**
* Image record containing all image metadata.
*/
public record ImageRecord(int id, long lotId, String url, String filePath, String labels) {}
/**
* Minimal record for images needing object detection processing.
*/
public record ImageDetectionRecord(int id, long lotId, String filePath) {}
/**
* Inserts a complete image record (for testing/legacy compatibility).
* In production, scraper inserts with local_path, monitor updates labels via updateLabels.
*/
public void insert(long lotId, String url, String filePath, List<String> labels) {
jdbi.useHandle(handle ->
handle.createUpdate("""
INSERT INTO images (lot_id, url, local_path, labels, processed_at, downloaded)
VALUES (:lotId, :url, :localPath, :labels, :processedAt, 1)
""")
.bind("lotId", lotId)
.bind("url", url)
.bind("localPath", filePath)
.bind("labels", String.join(",", labels))
.bind("processedAt", Instant.now().getEpochSecond())
.execute()
);
}
/**
* Updates the labels field for an image after object detection.
*/
public void updateLabels(int imageId, List<String> labels) {
jdbi.useHandle(handle ->
handle.createUpdate("UPDATE images SET labels = :labels, processed_at = :processedAt WHERE id = :id")
.bind("labels", String.join(",", labels))
.bind("processedAt", Instant.now().getEpochSecond())
.bind("id", imageId)
.execute()
);
}
/**
* Gets the labels for a specific image.
*/
public List<String> getLabels(int imageId) {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT labels FROM images WHERE id = :id")
.bind("id", imageId)
.mapTo(String.class)
.findOne()
.map(labelsStr -> {
if (labelsStr != null && !labelsStr.isEmpty()) {
return List.of(labelsStr.split(","));
}
return List.<String>of();
})
.orElse(List.of())
);
}
/**
* Retrieves images for a specific lot.
*/
public List<ImageRecord> getImagesForLot(long lotId) {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT id, lot_id, url, local_path, labels FROM images WHERE lot_id = :lotId")
.bind("lotId", lotId)
.map((rs, ctx) -> new ImageRecord(
rs.getInt("id"),
rs.getLong("lot_id"),
rs.getString("url"),
rs.getString("local_path"),
rs.getString("labels")
))
.list()
);
}
/**
* Gets images that have been downloaded by the scraper but need object detection.
* Only returns images that have local_path set but no labels yet.
*/
public List<ImageDetectionRecord> getImagesNeedingDetection() {
return jdbi.withHandle(handle ->
handle.createQuery("""
SELECT i.id, i.lot_id, i.local_path
FROM images i
WHERE i.local_path IS NOT NULL
AND i.local_path != ''
AND (i.labels IS NULL OR i.labels = '')
""")
.map((rs, ctx) -> {
// Extract numeric lot ID from TEXT field (e.g., "A1-34732-49" -> 3473249)
String lotIdStr = rs.getString("lot_id");
long lotId = auctiora.ScraperDataAdapter.extractNumericId(lotIdStr);
return new ImageDetectionRecord(
rs.getInt("id"),
lotId,
rs.getString("local_path")
);
})
.list()
);
}
/**
* Gets the total number of images in the database.
*/
public int getImageCount() {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT COUNT(*) FROM images")
.mapTo(Integer.class)
.one()
);
}
}

275
src/main/java/auctiora/db/LotRepository.java Normal file
View File

@@ -0,0 +1,275 @@
package auctiora.db;
import auctiora.Lot;
import auctiora.BidHistory;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Jdbi;
import java.time.LocalDateTime;
import java.util.List;
import static java.sql.Types.*;
/**
* Repository for lot-related database operations using JDBI3.
* Handles CRUD operations and queries for auction lots.
*/
@Slf4j
@RequiredArgsConstructor
public class LotRepository {
private final Jdbi jdbi;
/**
* Inserts or updates a lot (upsert operation).
* First tries UPDATE, then falls back to INSERT if lot doesn't exist.
*/
public void upsert(Lot lot) {
jdbi.useTransaction(handle -> {
// Try UPDATE first
int updated = handle.createUpdate("""
UPDATE lots SET
sale_id = :saleId,
auction_id = :auctionId,
title = :title,
description = :description,
manufacturer = :manufacturer,
type = :type,
year = :year,
category = :category,
current_bid = :currentBid,
currency = :currency,
url = :url,
closing_time = :closingTime
WHERE lot_id = :lotId
""")
.bind("saleId", String.valueOf(lot.saleId()))
.bind("auctionId", String.valueOf(lot.saleId())) // auction_id = sale_id
.bind("title", lot.title())
.bind("description", lot.description())
.bind("manufacturer", lot.manufacturer())
.bind("type", lot.type())
.bind("year", lot.year())
.bind("category", lot.category())
.bind("currentBid", lot.currentBid())
.bind("currency", lot.currency())
.bind("url", lot.url())
.bind("closingTime", lot.closingTime() != null ? lot.closingTime().toString() : null)
.bind("lotId", String.valueOf(lot.lotId()))
.execute();
if (updated == 0) {
// No rows updated, perform INSERT
handle.createUpdate("""
INSERT OR IGNORE INTO lots (
lot_id, sale_id, auction_id, title, description, manufacturer, type, year,
category, current_bid, currency, url, closing_time, closing_notified
) VALUES (
:lotId, :saleId, :auctionId, :title, :description, :manufacturer, :type, :year,
:category, :currentBid, :currency, :url, :closingTime, :closingNotified
)
""")
.bind("lotId", String.valueOf(lot.lotId()))
.bind("saleId", String.valueOf(lot.saleId()))
.bind("auctionId", String.valueOf(lot.saleId())) // auction_id = sale_id
.bind("title", lot.title())
.bind("description", lot.description())
.bind("manufacturer", lot.manufacturer())
.bind("type", lot.type())
.bind("year", lot.year())
.bind("category", lot.category())
.bind("currentBid", lot.currentBid())
.bind("currency", lot.currency())
.bind("url", lot.url())
.bind("closingTime", lot.closingTime() != null ? lot.closingTime().toString() : null)
.bind("closingNotified", lot.closingNotified() ? 1 : 0)
.execute();
}
});
}
/**
* Updates a lot with full intelligence data from GraphQL enrichment.
* Includes all 24+ intelligence fields from bidding platform.
*/
public void upsertWithIntelligence(Lot lot) {
jdbi.useHandle(handle -> {
var update = handle.createUpdate("""
UPDATE lots SET
sale_id = :saleId,
title = :title,
description = :description,
manufacturer = :manufacturer,
type = :type,
year = :year,
category = :category,
current_bid = :currentBid,
currency = :currency,
url = :url,
closing_time = :closingTime,
followers_count = :followersCount,
estimated_min = :estimatedMin,
estimated_max = :estimatedMax,
next_bid_step_cents = :nextBidStepInCents,
condition = :condition,
category_path = :categoryPath,
city_location = :cityLocation,
country_code = :countryCode,
bidding_status = :biddingStatus,
appearance = :appearance,
packaging = :packaging,
quantity = :quantity,
vat = :vat,
buyer_premium_percentage = :buyerPremiumPercentage,
remarks = :remarks,
starting_bid = :startingBid,
reserve_price = :reservePrice,
reserve_met = :reserveMet,
bid_increment = :bidIncrement,
view_count = :viewCount,
first_bid_time = :firstBidTime,
last_bid_time = :lastBidTime,
bid_velocity = :bidVelocity
WHERE lot_id = :lotId
""")
.bind("saleId", lot.saleId())
.bind("title", lot.title())
.bind("description", lot.description())
.bind("manufacturer", lot.manufacturer())
.bind("type", lot.type())
.bind("year", lot.year())
.bind("category", lot.category())
.bind("currentBid", lot.currentBid())
.bind("currency", lot.currency())
.bind("url", lot.url())
.bind("closingTime", lot.closingTime() != null ? lot.closingTime().toString() : null)
.bind("followersCount", lot.followersCount())
.bind("estimatedMin", lot.estimatedMin())
.bind("estimatedMax", lot.estimatedMax())
.bind("nextBidStepInCents", lot.nextBidStepInCents())
.bind("condition", lot.condition())
.bind("categoryPath", lot.categoryPath())
.bind("cityLocation", lot.cityLocation())
.bind("countryCode", lot.countryCode())
.bind("biddingStatus", lot.biddingStatus())
.bind("appearance", lot.appearance())
.bind("packaging", lot.packaging())
.bind("quantity", lot.quantity())
.bind("vat", lot.vat())
.bind("buyerPremiumPercentage", lot.buyerPremiumPercentage())
.bind("remarks", lot.remarks())
.bind("startingBid", lot.startingBid())
.bind("reservePrice", lot.reservePrice())
.bind("reserveMet", lot.reserveMet() != null && lot.reserveMet() ? 1 : null)
.bind("bidIncrement", lot.bidIncrement())
.bind("viewCount", lot.viewCount())
.bind("firstBidTime", lot.firstBidTime() != null ? lot.firstBidTime().toString() : null)
.bind("lastBidTime", lot.lastBidTime() != null ? lot.lastBidTime().toString() : null)
.bind("bidVelocity", lot.bidVelocity())
.bind("lotId", lot.lotId());
int updated = update.execute();
if (updated == 0) {
log.warn("Failed to update lot {} - lot not found in database", lot.lotId());
}
});
}
/**
* Updates only the current bid for a lot (lightweight update).
*/
public void updateCurrentBid(Lot lot) {
jdbi.useHandle(handle ->
handle.createUpdate("UPDATE lots SET current_bid = :bid WHERE lot_id = :lotId")
.bind("bid", lot.currentBid())
.bind("lotId", String.valueOf(lot.lotId()))
.execute()
);
}
/**
* Updates notification flags for a lot.
*/
public void updateNotificationFlags(Lot lot) {
jdbi.useHandle(handle ->
handle.createUpdate("UPDATE lots SET closing_notified = :notified WHERE lot_id = :lotId")
.bind("notified", lot.closingNotified() ? 1 : 0)
.bind("lotId", String.valueOf(lot.lotId()))
.execute()
);
}
/**
* Retrieves all active lots.
* Note: Despite the name, this returns ALL lots (legacy behavior for backward compatibility).
*/
public List<Lot> getActiveLots() {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT * FROM lots")
.map((rs, ctx) -> auctiora.ScraperDataAdapter.fromScraperLot(rs))
.list()
);
}
/**
* Retrieves all lots from the database.
*/
public List<Lot> getAllLots() {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT * FROM lots")
.map((rs, ctx) -> auctiora.ScraperDataAdapter.fromScraperLot(rs))
.list()
);
}
/**
* Retrieves bid history for a specific lot.
*/
public List<BidHistory> getBidHistory(String lotId) {
return jdbi.withHandle(handle ->
handle.createQuery("""
SELECT id, lot_id, bid_amount, bid_time, is_autobid, bidder_id, bidder_number
FROM bid_history
WHERE lot_id = :lotId
ORDER BY bid_time DESC
""")
.bind("lotId", lotId)
.map((rs, ctx) -> new BidHistory(
rs.getInt("id"),
rs.getString("lot_id"),
rs.getDouble("bid_amount"),
LocalDateTime.parse(rs.getString("bid_time")),
rs.getInt("is_autobid") != 0,
rs.getString("bidder_id"),
(Integer) rs.getObject("bidder_number")
))
.list()
);
}
/**
* Inserts bid history records in batch.
*/
public void insertBidHistory(List<BidHistory> bidHistory) {
jdbi.useHandle(handle -> {
var batch = handle.prepareBatch("""
INSERT OR IGNORE INTO bid_history (
lot_id, bid_amount, bid_time, is_autobid, bidder_id, bidder_number
) VALUES (:lotId, :bidAmount, :bidTime, :isAutobid, :bidderId, :bidderNumber)
""");
bidHistory.forEach(bid ->
batch.bind("lotId", bid.lotId())
.bind("bidAmount", bid.bidAmount())
.bind("bidTime", bid.bidTime().toString())
.bind("isAutobid", bid.isAutobid() ? 1 : 0)
.bind("bidderId", bid.bidderId())
.bind("bidderNumber", bid.bidderNumber())
.add()
);
batch.execute();
});
}
}

BIN
src/main/resources/META-INF/resources/favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.2 KiB

7
src/main/resources/META-INF/resources/favicon.svg Normal file
View File

@@ -0,0 +1,7 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
<rect width="100" height="100" fill="#2563eb" rx="15"/>
<path d="M25 40 L50 20 L75 40 L75 70 L25 70 Z" fill="#ffffff" stroke="#ffffff" stroke-width="2"/>
<circle cx="50" cy="45" r="8" fill="#2563eb"/>
<rect x="40" y="55" width="20" height="3" fill="#2563eb"/>
<text x="50" y="90" font-family="Arial" font-size="12" fill="#ffffff" text-anchor="middle" font-weight="bold">AUCTION</text>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 465 B

1611
src/main/resources/META-INF/resources/index.html Normal file
View File

File diff suppressed because it is too large Load Diff

0
src/main/resources/META-INF/resources/script.js Normal file
View File

224
src/main/resources/META-INF/resources/status.html Normal file
View File

@@ -0,0 +1,224 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Scrape-UI 1 - Enterprise</title>
<script src="https://cdn.tailwindcss.com"></script>
<style>
.gradient-bg {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
}
.card-hover {
transition: all 0.3s ease;
}
.card-hover:hover {
transform: translateY(-5px);
box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04);
}
</style>
</head>
<body class="bg-gray-50 min-h-screen">
<!-- Header -->
<header class="gradient-bg text-white py-8">
<div class="container mx-auto px-4">
<h1 class="text-4xl font-bold mb-2">Scrape-UI Enterprise</h1>
<p class="text-xl opacity-90">Powered by Quarkus + Modern Frontend</p>
</div>
</header>
<!-- Main Content -->
<main class="container mx-auto px-4 py-8">
<!-- API Status Card -->
<!-- API & Build Status Card -->
<div class="bg-white rounded-lg shadow-md p-6 mb-8 card-hover">
<h2 class="text-2xl font-bold mb-4 text-gray-800">Build & Runtime Status</h2>
<div id="api-status" class="space-y-4">
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
<!-- Build Information -->
<div class="bg-blue-50 p-4 rounded-lg">
<h3 class="font-semibold text-blue-800 mb-2">📦 Maven Build</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
<span class="text-gray-600">Group:</span>
<span class="font-mono font-medium" id="build-group">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Artifact:</span>
<span class="font-mono font-medium" id="build-artifact">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Version:</span>
<span class="font-mono font-medium px-2 py-1 bg-blue-100 rounded" id="build-version">-</span>
</div>
</div>
</div>
<!-- Runtime Information -->
<div class="bg-green-50 p-4 rounded-lg">
<h3 class="font-semibold text-green-800 mb-2">🚀 Runtime</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
<span class="text-gray-600">Status:</span>
<span class="px-2 py-1 rounded-full text-xs font-semibold bg-green-100 text-green-800" id="runtime-status">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Java:</span>
<span class="font-mono" id="java-version">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Platform:</span>
<span class="font-mono" id="runtime-os">-</span>
</div>
</div>
</div>
</div>
<!-- Timestamp & Additional Info -->
<div class="pt-4 border-t">
<div class="flex justify-between items-center">
<div>
<p class="text-sm text-gray-500">Last Updated</p>
<p class="font-medium" id="last-updated">-</p>
</div>
<button onclick="fetchStatus()" class="bg-gray-100 hover:bg-gray-200 text-gray-700 px-4 py-2 rounded-lg text-sm transition-colors">
🔄 Refresh
</button>
</div>
</div>
</div>
</div>
<!-- API Response Card -->
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h2 class="text-2xl font-bold mb-4 text-gray-800">API Test</h2>
<button id="test-api" class="bg-blue-600 text-white px-6 py-2 rounded-lg hover:bg-blue-700 transition-colors mb-4">
Test Greeting API
</button>
<div id="api-response" class="bg-gray-100 p-4 rounded-lg">
<pre class="text-sm text-gray-700">Click the button to test the API</pre>
</div>
</div>
<!-- Features Grid -->
<div class="grid grid-cols-1 md:grid-cols-3 gap-6 mt-8">
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">⚡ Quarkus Backend</h3>
<p class="text-gray-600">Fast startup, low memory footprint, optimized for containers</p>
</div>
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">🚀 REST API</h3>
<p class="text-gray-600">RESTful endpoints with JSON responses</p>
</div>
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">🎨 Modern UI</h3>
<p class="text-gray-600">Responsive design with Tailwind CSS</p>
</div>
</div>
</main>
<script>
// Fetch API status on load
async function fetchStatus() {
try {
const response = await fetch('/api/status')
if (!response.ok) {
throw new Error(`HTTP ${ response.status }: ${ response.statusText }`)
}
const data = await response.json()
// Update Build Information
document.getElementById('build-group').textContent = data.groupId || 'N/A'
document.getElementById('build-artifact').textContent = data.artifactId || data.name || 'N/A'
document.getElementById('build-version').textContent = data.version || 'N/A'
// Update Runtime Information
document.getElementById('runtime-status').textContent = data.status || 'unknown'
document.getElementById('java-version').textContent = data.javaVersion || System.getProperty?.('java.version') || 'N/A'
document.getElementById('runtime-os').textContent = data.os || 'N/A'
// Update Timestamp
const timestamp = data.timestamp ? new Date(data.timestamp).toLocaleString() : 'N/A'
document.getElementById('last-updated').textContent = timestamp
// Update status badge color based on status
const statusBadge = document.getElementById('runtime-status')
if (data.status?.toLowerCase() === 'running') {
statusBadge.className = 'px-2 py-1 rounded-full text-xs font-semibold bg-green-100 text-green-800'
} else {
statusBadge.className = 'px-2 py-1 rounded-full text-xs font-semibold bg-yellow-100 text-yellow-800'
}
} catch (error) {
console.error('Error fetching status:', error)
document.getElementById('api-status').innerHTML = `
<div class="bg-red-50 border-l-4 border-red-500 p-4">
<div class="flex">
<div class="flex-shrink-0">
<svg class="h-5 w-5 text-red-400" viewBox="0 0 20 20" fill="currentColor">
<path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clip-rule="evenodd" />
</svg>
</div>
<div class="ml-3">
<p class="text-sm text-red-700">Failed to load status: ${ error.message }</p>
<button onclick="fetchStatus()" class="mt-2 text-sm text-red-700 hover:text-red-600 font-medium">
Retry →
</button>
</div>
</div>
</div>
`
}
}
// Fetch API status on load
async function fetchStatus3() {
try {
const response = await fetch('/api/status')
const data = await response.json()
document.getElementById('api-status').innerHTML = `
<p><strong>Application:</strong> ${ data.application }</p>
<p><strong>Status:</strong> <span class="text-green-600 font-semibold">${ data.status }</span></p>
<p><strong>Version:</strong> ${ data.version }</p>
<p><strong>Timestamp:</strong> ${ data.timestamp }</p>
`
} catch (error) {
document.getElementById('api-status').innerHTML = `
<p class="text-red-600">Error loading status: ${ error.message }</p>
`
}
}
// Test greeting API
document.getElementById('test-api').addEventListener('click', async () => {
try {
const response = await fetch('/api/hello')
const data = await response.json()
document.getElementById('api-response').innerHTML = `
<pre class="text-sm text-gray-700">${ JSON.stringify(data, null, 2) }</pre>
`
} catch (error) {
document.getElementById('api-response').innerHTML = `
<pre class="text-sm text-red-600">Error: ${ error.message }</pre>
`
}
})
// Auto-refresh every 30 seconds
let refreshInterval = setInterval(fetchStatus, 30000);
// Stop auto-refresh when page loses focus (optional)
document.addEventListener('visibilitychange', function() {
if (document.hidden) {
clearInterval(refreshInterval);
} else {
refreshInterval = setInterval(fetchStatus, 30000);
fetchStatus(); // Refresh immediately when returning to tab
}
});
// Load status on page load
fetchStatus()
</script>
</body>
</html>

0
src/main/resources/META-INF/resources/style.css Normal file
View File

1086
src/main/resources/META-INF/resources/valuation-analytics.html Normal file
View File

File diff suppressed because it is too large Load Diff

75
src/main/resources/application.properties Normal file
View File

@@ -0,0 +1,75 @@
# Application Configuration
# Values will be injected from pom.xml during build
quarkus.application.name=${project.artifactId}
quarkus.application.version=${project.version}
# Custom properties for groupId if needed
application.groupId=${project.groupId}
application.artifactId=${project.artifactId}
application.version=${project.version}
# HTTP Configuration
quarkus.http.port=8081
# ========== DEVELOPMENT (quarkus:dev) ==========
%dev.quarkus.http.host=127.0.0.1
# ========== PRODUCTION (Docker/JAR) ==========
%prod.quarkus.http.host=0.0.0.0
# ========== TEST PROFILE ==========
%test.quarkus.http.host=localhost
# Enable CORS for frontend development
quarkus.http.cors=true
quarkus.http.cors.origins=*
quarkus.http.cors.methods=GET,POST,PUT,DELETE,OPTIONS
quarkus.http.cors.headers=accept,authorization,content-type,x-requested-with
# Logging Configuration
quarkus.log.console.format=%d{HH:mm:ss} %-5p [%c{2.}] (%t) %s%e%n
quarkus.log.console.level=INFO
# Development mode settings
%dev.quarkus.log.console.level=DEBUG
%dev.quarkus.live-reload.instrumentation=true
# JVM Arguments for native access (Jansi, OpenCV, etc.)
quarkus.native.additional-build-args=--enable-native-access=ALL-UNNAMED
# Production optimizations
%prod.quarkus.package.type=fast-jar
%prod.quarkus.http.enable-compression=true
# Static resources
quarkus.http.enable-compression=true
quarkus.rest.path=/
quarkus.http.root-path=/
# Auction Monitor Configuration
auction.database.path=/mnt/okcomputer/output/cache.db
auction.images.path=/mnt/okcomputer/output/images
# auction.notification.config=desktop
# Format: smtp:username:password:recipient_email
auction.notification.config=smtp:michael.bakker1986@gmail.com:agrepolhlnvhipkv:michael.bakker1986@gmail.com
auction.yolo.config=/mnt/okcomputer/output/models/yolov4.cfg
auction.yolo.weights=/mnt/okcomputer/output/models/yolov4.weights
auction.yolo.classes=/mnt/okcomputer/output/models/coco.names
# Scheduler Configuration
quarkus.scheduler.enabled=true
quarkus.scheduler.start-halted=false
# Workflow Schedules
auction.workflow.scraper-import.cron=0 */30 * * * ?
auction.workflow.image-processing.cron=0 0 * * * ?
auction.workflow.bid-monitoring.cron=0 */15 * * * ?
auction.workflow.closing-alerts.cron=0 */5 * * * ?
# HTTP Rate Limiting Configuration
# Prevents overloading external services and getting blocked
auction.http.rate-limit.default-max-rps=2
auction.http.rate-limit.troostwijk-max-rps=1
auction.http.timeout-seconds=30
# Health Check Configuration
quarkus.smallrye-health.root-path=/health

20
src/main/resources/simplelogger.properties Normal file
View File

@@ -0,0 +1,20 @@
# SLF4J Simple Logger Configuration
# Set default log level (trace, debug, info, warn, error, off)
org.slf4j.simpleLogger.defaultLogLevel=warn
# Show date/time in logs
org.slf4j.simpleLogger.showDateTime=true
org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss
# Show thread name
org.slf4j.simpleLogger.showThreadName=false
# Show log name (logger name)
org.slf4j.simpleLogger.showLogName=false
# Show short log name
org.slf4j.simpleLogger.showShortLogName=true
# Set specific logger levels
org.slf4j.simpleLogger.log.com.microsoft.playwright=warn
org.slf4j.simpleLogger.log.org.sqlite=warn

138
src/test/java/auctiora/ClosingTimeCalculationTest.java Normal file
View File

@@ -0,0 +1,138 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.time.LocalDateTime;
import static org.junit.jupiter.api.Assertions.*;
/**
* Tests for closing time calculations that power the UI
* Tests the minutesUntilClose() logic used in dashboard and alerts
*/
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@DisplayName("Closing Time Calculation Tests")
class ClosingTimeCalculationTest {
@Test
@Order(1)
@DisplayName("Should calculate minutes until close for lot closing in 15 minutes")
void testMinutesUntilClose15Minutes() {
var lot = createLot(LocalDateTime.now().plusMinutes(15));
long minutes = lot.minutesUntilClose();
assertTrue(minutes >= 14 && minutes <= 16,
"Should be approximately 15 minutes, was: " + minutes);
}
@Test
@Order(2)
@DisplayName("Should calculate minutes until close for lot closing in 2 hours")
void testMinutesUntilClose2Hours() {
var lot = createLot(LocalDateTime.now().plusHours(2));
long minutes = lot.minutesUntilClose();
assertTrue(minutes >= 119 && minutes <= 121,
"Should be approximately 120 minutes, was: " + minutes);
}
@Test
@Order(3)
@DisplayName("Should return negative value for already closed lot")
void testMinutesUntilCloseNegative() {
var lot = createLot(LocalDateTime.now().minusHours(1));
long minutes = lot.minutesUntilClose();
assertTrue(minutes < 0,
"Should be negative for closed lots, was: " + minutes);
}
@Test
@Order(4)
@DisplayName("Should return MAX_VALUE when lot has no closing time")
void testMinutesUntilCloseNoTime() {
var lot = Lot.basic(100, 1001, "No closing time", "", "", "", 0, "General",
100.0, "EUR", "http://test.com/1001", null, false);
long minutes = lot.minutesUntilClose();
assertEquals(Long.MAX_VALUE, minutes,
"Should return MAX_VALUE when no closing time set");
}
@Test
@Order(5)
@DisplayName("Should identify lots closing within 5 minutes (critical threshold)")
void testCriticalClosingThreshold() {
var closing4Min = createLot(LocalDateTime.now().plusMinutes(4));
var closing5Min = createLot(LocalDateTime.now().plusMinutes(5));
var closing6Min = createLot(LocalDateTime.now().plusMinutes(6));
assertTrue(closing4Min.minutesUntilClose() < 5,
"Lot closing in 4 min should be < 5 minutes");
assertTrue(closing5Min.minutesUntilClose() >= 5,
"Lot closing in 5 min should be >= 5 minutes");
assertTrue(closing6Min.minutesUntilClose() > 5,
"Lot closing in 6 min should be > 5 minutes");
}
@Test
@Order(6)
@DisplayName("Should identify lots closing within 30 minutes (dashboard threshold)")
void testDashboardClosingThreshold() {
var closing20Min = createLot(LocalDateTime.now().plusMinutes(20));
var closing31Min = createLot(LocalDateTime.now().plusMinutes(31)); // Use 31 to avoid boundary timing issue
var closing40Min = createLot(LocalDateTime.now().plusMinutes(40));
assertTrue(closing20Min.minutesUntilClose() < 30,
"Lot closing in 20 min should be < 30 minutes");
assertTrue(closing31Min.minutesUntilClose() >= 30,
"Lot closing in 31 min should be >= 30 minutes");
assertTrue(closing40Min.minutesUntilClose() > 30,
"Lot closing in 40 min should be > 30 minutes");
}
@Test
@Order(7)
@DisplayName("Should calculate correctly for lots closing soon (boundary cases)")
void testBoundaryCases() {
// Just closed (< 1 minute ago)
var justClosed = createLot(LocalDateTime.now().minusSeconds(30));
assertTrue(justClosed.minutesUntilClose() <= 0, "Just closed should be <= 0");
// Closing very soon (< 1 minute)
var closingVerySoon = createLot(LocalDateTime.now().plusSeconds(30));
assertTrue(closingVerySoon.minutesUntilClose() < 1, "Closing in 30 sec should be < 1 minute");
// Closing in exactly 1 hour
var closing1Hour = createLot(LocalDateTime.now().plusHours(1));
long minutes1Hour = closing1Hour.minutesUntilClose();
assertTrue(minutes1Hour >= 59 && minutes1Hour <= 61,
"Closing in 1 hour should be ~60 minutes, was: " + minutes1Hour);
}
@Test
@Order(8)
@DisplayName("Multiple lots should sort correctly by urgency")
void testSortingByUrgency() {
var lot5Min = createLot(LocalDateTime.now().plusMinutes(5));
var lot30Min = createLot(LocalDateTime.now().plusMinutes(30));
var lot1Hour = createLot(LocalDateTime.now().plusHours(1));
var lot3Hours = createLot(LocalDateTime.now().plusHours(3));
var lots = java.util.List.of(lot3Hours, lot30Min, lot5Min, lot1Hour);
var sorted = lots.stream()
.sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose()))
.toList();
assertEquals(lot5Min, sorted.get(0), "Most urgent should be first");
assertEquals(lot30Min, sorted.get(1), "Second most urgent");
assertEquals(lot1Hour, sorted.get(2), "Third most urgent");
assertEquals(lot3Hours, sorted.get(3), "Least urgent should be last");
}
// Helper method
private Lot createLot(LocalDateTime closingTime) {
return Lot.basic(100, 1001, "Test Item", "", "", "", 0, "General",
100.0, "EUR", "http://test.com/1001", closingTime, false);
}
}

390
src/test/java/auctiora/DatabaseServiceTest.java Normal file
View File

@@ -0,0 +1,390 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* Test cases for DatabaseService.
* Tests database operations including schema creation, CRUD operations, and data retrieval.
*/
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class DatabaseServiceTest {
private DatabaseService db;
private String testDbPath;
@BeforeAll
void setUp() throws SQLException {
// Load SQLite JDBC driver
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
throw new SQLException("SQLite JDBC driver not found", e);
}
testDbPath = "test_database_" + System.currentTimeMillis() + ".db";
db = new DatabaseService(testDbPath);
db.ensureSchema();
}
@AfterAll
void tearDown() throws Exception {
// Clean up test database
Files.deleteIfExists(Paths.get(testDbPath));
}
@Test
@DisplayName("Should create database schema successfully")
void testEnsureSchema() {
assertDoesNotThrow(() -> db.ensureSchema());
}
@Test
@DisplayName("Should insert and retrieve auction")
void testUpsertAndGetAuction() throws SQLException {
var auction = new AuctionInfo(
12345,
"Test Auction",
"Amsterdam, NL",
"Amsterdam",
"NL",
"https://example.com/auction/12345",
"A7",
50,
LocalDateTime.of(2025, 12, 15, 14, 30)
);
db.upsertAuction(auction);
var auctions = db.getAllAuctions();
assertFalse(auctions.isEmpty());
var retrieved = auctions.stream()
.filter(a -> a.auctionId() == 12345)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertEquals("Test Auction", retrieved.title());
assertEquals("Amsterdam", retrieved.city());
assertEquals("NL", retrieved.country());
assertEquals(50, retrieved.lotCount());
}
@Test
@DisplayName("Should update existing auction on conflict")
void testUpsertAuctionUpdate() throws SQLException {
var auction1 = new AuctionInfo(
99999,
"Original Title",
"Rotterdam, NL",
"Rotterdam",
"NL",
"https://example.com/auction/99999",
"A1",
10,
null
);
db.upsertAuction(auction1);
// Update with same ID
var auction2 = new AuctionInfo(
99999,
"Updated Title",
"Rotterdam, NL",
"Rotterdam",
"NL",
"https://example.com/auction/99999",
"A1",
20,
null
);
db.upsertAuction(auction2);
var auctions = db.getAllAuctions();
var retrieved = auctions.stream()
.filter(a -> a.auctionId() == 99999)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertEquals("Updated Title", retrieved.title());
assertEquals(20, retrieved.lotCount());
}
@Test
@DisplayName("Should retrieve auctions by country code")
void testGetAuctionsByCountry() throws SQLException {
// Insert auctions from different countries
db.upsertAuction(new AuctionInfo(
10001, "Dutch Auction", "Amsterdam, NL", "Amsterdam", "NL",
"https://example.com/10001", "A1", 10, null
));
db.upsertAuction(new AuctionInfo(
10002, "Romanian Auction", "Cluj, RO", "Cluj", "RO",
"https://example.com/10002", "A2", 15, null
));
db.upsertAuction(new AuctionInfo(
10003, "Another Dutch", "Utrecht, NL", "Utrecht", "NL",
"https://example.com/10003", "A3", 20, null
));
var nlAuctions = db.getAuctionsByCountry("NL");
assertEquals(2, nlAuctions.stream().filter(a -> a.auctionId() >= 10001 && a.auctionId() <= 10003).count());
var roAuctions = db.getAuctionsByCountry("RO");
assertEquals(1, roAuctions.stream().filter(a -> a.auctionId() == 10002).count());
}
@Test
@DisplayName("Should insert and retrieve lot")
void testUpsertAndGetLot() throws SQLException {
var lot = Lot.basic(
12345, // saleId
67890, // lotId
"Forklift",
"Electric forklift in good condition",
"Toyota",
"Electric",
2018,
"Machinery",
1500.00,
"EUR",
"https://example.com/lot/67890",
LocalDateTime.of(2025, 12, 20, 16, 0),
false
);
db.upsertLot(lot);
var lots = db.getAllLots();
assertFalse(lots.isEmpty());
var retrieved = lots.stream()
.filter(l -> l.lotId() == 67890)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertEquals("Forklift", retrieved.title());
assertEquals("Toyota", retrieved.manufacturer());
assertEquals(2018, retrieved.year());
assertEquals(1500.00, retrieved.currentBid(), 0.01);
assertFalse(retrieved.closingNotified());
}
@Test
@DisplayName("Should update lot current bid")
void testUpdateLotCurrentBid() throws SQLException {
var lot = Lot.basic(
11111, 22222, "Test Item", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/22222", null, false
);
db.upsertLot(lot);
// Update bid
var updatedLot = Lot.basic(
11111, 22222, "Test Item", "Description", "", "", 0, "Category",
250.00, "EUR", "https://example.com/lot/22222", null, false
);
db.updateLotCurrentBid(updatedLot);
var lots = db.getAllLots();
var retrieved = lots.stream()
.filter(l -> l.lotId() == 22222)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertEquals(250.00, retrieved.currentBid(), 0.01);
}
@Test
@DisplayName("Should update lot notification flags")
void testUpdateLotNotificationFlags() throws SQLException {
var lot = Lot.basic(
33333, 44444, "Test Item", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/44444", null, false
);
db.upsertLot(lot);
// Update notification flag
var updatedLot = Lot.basic(
33333, 44444, "Test Item", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/44444", null, true
);
db.updateLotNotificationFlags(updatedLot);
var lots = db.getAllLots();
var retrieved = lots.stream()
.filter(l -> l.lotId() == 44444)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertTrue(retrieved.closingNotified());
}
@Test
@DisplayName("Should insert and retrieve image records")
void testInsertAndGetImages() throws SQLException {
// First create a lot
var lot = Lot.basic(
55555, 66666, "Test Lot", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/66666", null, false
);
db.upsertLot(lot);
// Insert images
db.insertImage(66666, "https://example.com/img1.jpg",
"C:/images/66666/img1.jpg", List.of("car", "vehicle"));
db.insertImage(66666, "https://example.com/img2.jpg",
"C:/images/66666/img2.jpg", List.of("truck"));
var images = db.getImagesForLot(66666);
assertEquals(2, images.size());
var img1 = images.stream()
.filter(i -> i.url().contains("img1.jpg"))
.findFirst()
.orElse(null);
assertNotNull(img1);
assertEquals("car,vehicle", img1.labels());
}
@Test
@DisplayName("Should count total images")
void testGetImageCount() throws SQLException {
int initialCount = db.getImageCount();
// Add a lot and image
var lot = Lot.basic(
77777, 88888, "Test Lot", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/88888", null, false
);
db.upsertLot(lot);
db.insertImage(88888, "https://example.com/test.jpg",
"C:/images/88888/test.jpg", List.of("object"));
int newCount = db.getImageCount();
assertTrue(newCount > initialCount);
}
@Test
@DisplayName("Should handle empty database gracefully")
void testEmptyDatabase() throws SQLException, IOException {
DatabaseService emptyDb = new DatabaseService("empty_test_" + System.currentTimeMillis() + ".db");
emptyDb.ensureSchema();
var auctions = emptyDb.getAllAuctions();
var lots = emptyDb.getAllLots();
int imageCount = emptyDb.getImageCount();
assertNotNull(auctions);
assertNotNull(lots);
assertTrue(auctions.isEmpty());
assertTrue(lots.isEmpty());
assertEquals(0, imageCount);
// Clean up
Files.deleteIfExists(Paths.get("empty_test_" + System.currentTimeMillis() + ".db"));
}
@Test
@DisplayName("Should handle lots with null closing time")
void testLotWithNullClosingTime() throws SQLException {
var lot = Lot.basic(
98765, 12340, "Test Item", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/12340", null, false
);
assertDoesNotThrow(() -> db.upsertLot(lot));
var retrieved = db.getAllLots().stream()
.filter(l -> l.lotId() == 12340)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertNull(retrieved.closingTime());
}
@Test
@DisplayName("Should retrieve active lots only")
void testGetActiveLots() throws SQLException {
var activeLot = Lot.basic(
11111, 55551, "Active Lot", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/55551",
LocalDateTime.now().plusDays(1), false
);
db.upsertLot(activeLot);
var activeLots = db.getActiveLots();
assertFalse(activeLots.isEmpty());
var found = activeLots.stream()
.anyMatch(l -> l.lotId() == 55551);
assertTrue(found);
}
@Test
@DisplayName("Should handle concurrent upserts")
void testConcurrentUpserts() throws InterruptedException, SQLException {
Thread t1 = new Thread(() -> {
try {
for (int i = 0; i < 10; i++) {
db.upsertLot(Lot.basic(
99000 + i, 99100 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
100.0, "EUR", "https://example.com/" + i, null, false
));
}
} catch (Exception e) {
fail("Thread 1 failed: " + e.getMessage());
}
});
Thread t2 = new Thread(() -> {
try {
for (int i = 10; i < 20; i++) {
db.upsertLot(Lot.basic(
99000 + i, 99100 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
200.0, "EUR", "https://example.com/" + i, null, false
));
}
} catch (Exception e) {
fail("Thread 2 failed: " + e.getMessage());
}
});
t1.start();
t2.start();
t1.join();
t2.join();
var lots = db.getAllLots();
long concurrentLots = lots.stream()
.filter(l -> l.lotId() >= 99100 && l.lotId() < 99120)
.count();
assertTrue(concurrentLots >= 20);
}
}

186
src/test/java/auctiora/ImageProcessingServiceTest.java Normal file
View File

@@ -0,0 +1,186 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.sql.SQLException;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
/**
* Test cases for ImageProcessingService.
* Tests object detection integration and database label updates.
*
* NOTE: Image downloading is now handled by the scraper, so these tests
* focus only on object detection and label storage.
*/
class ImageProcessingServiceTest {
private DatabaseService mockDb;
private ObjectDetectionService mockDetector;
private ImageProcessingService service;
private java.io.File testImage;
@BeforeEach
void setUp() throws Exception {
mockDb = mock(DatabaseService.class);
mockDetector = mock(ObjectDetectionService.class);
service = new ImageProcessingService(mockDb, mockDetector);
// Create a temporary test image file
testImage = java.io.File.createTempFile("test_image_", ".jpg");
testImage.deleteOnExit();
// Write minimal JPEG header to make it a valid file
try (var out = new java.io.FileOutputStream(testImage)) {
out.write(new byte[]{(byte)0xFF, (byte)0xD8, (byte)0xFF, (byte)0xE0});
}
}
@Test
@DisplayName("Should process single image and update labels")
void testProcessImage() throws SQLException {
// Normalize path (convert backslashes to forward slashes)
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
// Mock object detection with normalized path
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("car", "vehicle"));
// Process image
boolean result = service.processImage(1, testImage.getAbsolutePath(), 12345);
// Verify success
assertTrue(result);
verify(mockDetector).detectObjects(normalizedPath);
verify(mockDb).updateImageLabels(1, List.of("car", "vehicle"));
}
@Test
@DisplayName("Should handle empty detection results")
void testProcessImageWithNoDetections() throws SQLException {
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of());
boolean result = service.processImage(2, testImage.getAbsolutePath(), 12346);
assertTrue(result);
verify(mockDb).updateImageLabels(2, List.of());
}
@Test
@DisplayName("Should handle database error gracefully")
void testProcessImageDatabaseError() {
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("object"));
doThrow(new RuntimeException("Database error"))
.when(mockDb).updateImageLabels(anyInt(), anyList());
// Should return false on error
boolean result = service.processImage(3, testImage.getAbsolutePath(), 12347);
assertFalse(result);
}
@Test
@DisplayName("Should handle object detection error gracefully")
void testProcessImageDetectionError() {
when(mockDetector.detectObjects(anyString()))
.thenThrow(new RuntimeException("Detection failed"));
// Should return false on error
boolean result = service.processImage(4, "/path/to/image4.jpg", 12348);
assertFalse(result);
}
@Test
@DisplayName("Should process pending images batch")
void testProcessPendingImages() throws SQLException {
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
// Mock pending images from database - use real test image path
when(mockDb.getImagesNeedingDetection()).thenReturn(List.of(
new DatabaseService.ImageDetectionRecord(1, 100L, testImage.getAbsolutePath()),
new DatabaseService.ImageDetectionRecord(2, 101L, testImage.getAbsolutePath())
));
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("item1"))
.thenReturn(List.of("item2"));
when(mockDb.getImageLabels(anyInt()))
.thenReturn(List.of("item1"))
.thenReturn(List.of("item2"));
// Process batch
service.processPendingImages();
// Verify all images were processed
verify(mockDb).getImagesNeedingDetection();
verify(mockDetector, times(2)).detectObjects(normalizedPath);
verify(mockDb, times(2)).updateImageLabels(anyInt(), anyList());
}
@Test
@DisplayName("Should handle empty pending images list")
void testProcessPendingImagesEmpty() throws SQLException {
when(mockDb.getImagesNeedingDetection()).thenReturn(List.of());
service.processPendingImages();
verify(mockDb).getImagesNeedingDetection();
verify(mockDetector, never()).detectObjects(anyString());
}
@Test
@DisplayName("Should continue processing after single image failure")
void testProcessPendingImagesWithFailure() throws SQLException {
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDb.getImagesNeedingDetection()).thenReturn(List.of(
new DatabaseService.ImageDetectionRecord(1, 100L, testImage.getAbsolutePath()),
new DatabaseService.ImageDetectionRecord(2, 101L, testImage.getAbsolutePath())
));
// First image fails, second succeeds
when(mockDetector.detectObjects(normalizedPath))
.thenThrow(new RuntimeException("Detection error"))
.thenReturn(List.of("item"));
when(mockDb.getImageLabels(2))
.thenReturn(List.of("item"));
service.processPendingImages();
// Verify second image was still processed
verify(mockDetector, times(2)).detectObjects(normalizedPath);
}
@Test
@DisplayName("Should handle database query error in batch processing")
void testProcessPendingImagesDatabaseError() {
when(mockDb.getImagesNeedingDetection())
.thenThrow(new RuntimeException("Database connection failed"));
// Should not throw exception
assertDoesNotThrow(() -> service.processPendingImages());
}
@Test
@DisplayName("Should process images with multiple detected objects")
void testProcessImageMultipleDetections() throws SQLException {
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("car", "truck", "vehicle", "road"));
boolean result = service.processImage(5, testImage.getAbsolutePath(), 12349);
assertTrue(result);
verify(mockDb).updateImageLabels(5, List.of("car", "truck", "vehicle", "road"));
}
}

461
src/test/java/auctiora/IntegrationTest.java Normal file
View File

@@ -0,0 +1,461 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* Integration test for complete workflow.
* Tests end-to-end scenarios including:
* 1. Scraper data import
* 2. Data transformation
* 3. Image processing
* 4. Object detection
* 5. Bid monitoring
* 6. Notifications
*/
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
class IntegrationTest {
private String testDbPath;
private DatabaseService db;
private NotificationService notifier;
private ObjectDetectionService detector;
private ImageProcessingService imageProcessor;
private TroostwijkMonitor monitor;
@BeforeAll
void setUp() throws SQLException, IOException {
testDbPath = "test_integration_" + System.currentTimeMillis() + ".db";
// Initialize all services
db = new DatabaseService(testDbPath);
db.ensureSchema();
notifier = new NotificationService("desktop");
detector = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
imageProcessor = new ImageProcessingService(db, detector);
monitor = new TroostwijkMonitor(
testDbPath,
"desktop",
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
}
@AfterAll
void tearDown() throws Exception {
Files.deleteIfExists(Paths.get(testDbPath));
}
@Test
@Order(1)
@DisplayName("Integration: Complete scraper data import workflow")
void testCompleteScraperImportWorkflow() throws SQLException {
// Step 1: Import auction from scraper format
var auction = new AuctionInfo(
12345,
"Industrial Equipment Auction",
"Rotterdam, NL",
"Rotterdam",
"NL",
"https://example.com/auction/12345",
"A7",
25,
LocalDateTime.now().plusDays(3)
);
db.upsertAuction(auction);
// Step 2: Import lots for this auction
var lot1 = Lot.basic(
12345, 10001,
"Toyota Forklift 2.5T",
"Electric forklift in excellent condition",
"Toyota",
"Electric",
2018,
"Machinery",
1500.00,
"EUR",
"https://example.com/lot/10001",
LocalDateTime.now().plusDays(3),
false
);
var lot2 = Lot.basic(
12345, 10002,
"Office Furniture Set",
"Desks, chairs, and cabinets",
"",
"",
0,
"Furniture",
500.00,
"EUR",
"https://example.com/lot/10002",
LocalDateTime.now().plusDays(3),
false
);
db.upsertLot(lot1);
db.upsertLot(lot2);
// Verify import
var auctions = db.getAllAuctions();
var lots = db.getAllLots();
assertTrue(auctions.stream().anyMatch(a -> a.auctionId() == 12345));
assertEquals(2, lots.stream().filter(l -> l.saleId() == 12345).count());
}
@Test
@Order(2)
@DisplayName("Integration: Image processing and detection workflow")
void testImageProcessingWorkflow() throws SQLException {
// Add images for a lot
db.insertImage(10001, "https://example.com/img1.jpg",
"C:/images/10001/img1.jpg", List.of("truck", "vehicle"));
db.insertImage(10001, "https://example.com/img2.jpg",
"C:/images/10001/img2.jpg", List.of("forklift", "machinery"));
// Verify images were saved
var images = db.getImagesForLot(10001);
assertEquals(2, images.size());
var labels = images.stream()
.flatMap(img -> List.of(img.labels().split(",")).stream())
.distinct()
.toList();
assertTrue(labels.contains("truck") || labels.contains("forklift"));
}
@Test
@Order(3)
@DisplayName("Integration: Bid monitoring and notification workflow")
void testBidMonitoringWorkflow() throws SQLException {
// Simulate bid change
var lot = db.getAllLots().stream()
.filter(l -> l.lotId() == 10001)
.findFirst()
.orElseThrow();
// Update bid
var updatedLot = Lot.basic(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
2000.00, // Increased from 1500.00
lot.currency(), lot.url(), lot.closingTime(), lot.closingNotified()
);
db.updateLotCurrentBid(updatedLot);
// Send notification
var message = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), 2000.00, 1500.00);
assertDoesNotThrow(() ->
notifier.sendNotification(message, "Kavel bieding update", 0)
);
// Verify bid was updated
var refreshed = db.getAllLots().stream()
.filter(l -> l.lotId() == 10001)
.findFirst()
.orElseThrow();
assertEquals(2000.00, refreshed.currentBid(), 0.01);
}
@Test
@Order(4)
@DisplayName("Integration: Closing alert workflow")
void testClosingAlertWorkflow() throws SQLException {
// Create lot closing soon
var closingSoon = Lot.basic(
12345, 20001,
"Closing Soon Item",
"Description",
"",
"",
0,
"Category",
750.00,
"EUR",
"https://example.com/lot/20001",
LocalDateTime.now().plusMinutes(4),
false
);
db.upsertLot(closingSoon);
// Check if lot is closing soon
assertTrue(closingSoon.minutesUntilClose() < 5);
// Send high-priority notification
var message = "Kavel " + closingSoon.lotId() + " sluit binnen 5 min.";
assertDoesNotThrow(() ->
notifier.sendNotification(message, "Lot nearing closure", 1)
);
// Mark as notified
var notified = Lot.basic(
closingSoon.saleId(), closingSoon.lotId(), closingSoon.title(),
closingSoon.description(), closingSoon.manufacturer(), closingSoon.type(),
closingSoon.year(), closingSoon.category(), closingSoon.currentBid(),
closingSoon.currency(), closingSoon.url(), closingSoon.closingTime(),
true
);
db.updateLotNotificationFlags(notified);
// Verify notification flag
var updated = db.getAllLots().stream()
.filter(l -> l.lotId() == 20001)
.findFirst()
.orElseThrow();
assertTrue(updated.closingNotified());
}
@Test
@Order(5)
@DisplayName("Integration: Multi-country auction filtering")
void testMultiCountryFiltering() throws SQLException {
// Add auctions from different countries
db.upsertAuction(new AuctionInfo(
30001, "Dutch Auction", "Amsterdam, NL", "Amsterdam", "NL",
"https://example.com/30001", "A1", 10, null
));
db.upsertAuction(new AuctionInfo(
30002, "Romanian Auction", "Cluj, RO", "Cluj", "RO",
"https://example.com/30002", "A2", 15, null
));
db.upsertAuction(new AuctionInfo(
30003, "Belgian Auction", "Brussels, BE", "Brussels", "BE",
"https://example.com/30003", "A3", 20, null
));
// Filter by country
var nlAuctions = db.getAuctionsByCountry("NL");
var roAuctions = db.getAuctionsByCountry("RO");
var beAuctions = db.getAuctionsByCountry("BE");
assertTrue(nlAuctions.stream().anyMatch(a -> a.auctionId() == 30001));
assertTrue(roAuctions.stream().anyMatch(a -> a.auctionId() == 30002));
assertTrue(beAuctions.stream().anyMatch(a -> a.auctionId() == 30003));
}
@Test
@Order(6)
@DisplayName("Integration: Complete monitoring cycle")
void testCompleteMonitoringCycle() throws SQLException {
// Monitor should handle all lots
monitor.printDatabaseStats();
var activeLots = db.getActiveLots();
assertFalse(activeLots.isEmpty());
// Process pending images
assertDoesNotThrow(() -> monitor.processPendingImages());
// Verify database integrity
var imageCount = db.getImageCount();
assertTrue(imageCount >= 0);
}
@Test
@Order(7)
@DisplayName("Integration: Data consistency across services")
void testDataConsistency() throws SQLException {
// Verify all auctions have valid data
var auctions = db.getAllAuctions();
for (var auction : auctions) {
assertNotNull(auction.auctionId());
assertNotNull(auction.title());
assertNotNull(auction.url());
}
// Verify all lots have valid data
var lots = db.getAllLots();
for (var lot : lots) {
assertNotNull(lot.lotId());
assertNotNull(lot.title());
assertTrue(lot.currentBid() >= 0);
}
}
@Test
@Order(8)
@DisplayName("Integration: Object detection value estimation workflow")
void testValueEstimationWorkflow() throws SQLException {
// Create lot with detected objects
var lot = Lot.basic(
40000, 50000,
"Construction Equipment",
"Heavy machinery for construction",
"Caterpillar",
"Excavator",
2015,
"Machinery",
25000.00,
"EUR",
"https://example.com/lot/50000",
LocalDateTime.now().plusDays(5),
false
);
db.upsertLot(lot);
// Add images with detected objects
db.insertImage(50000, "https://example.com/excavator1.jpg",
"C:/images/50000/1.jpg", List.of("truck", "excavator", "machinery"));
db.insertImage(50000, "https://example.com/excavator2.jpg",
"C:/images/50000/2.jpg", List.of("excavator", "construction"));
// Retrieve and analyze
var images = db.getImagesForLot(50000);
assertFalse(images.isEmpty());
// Count unique objects
var allLabels = images.stream()
.flatMap(img -> List.of(img.labels().split(",")).stream())
.distinct()
.toList();
assertTrue(allLabels.contains("excavator") || allLabels.contains("machinery"));
// Simulate value estimation notification
var message = String.format(
"Lot contains: %s\nEstimated value: €%,.2f",
String.join(", ", allLabels),
lot.currentBid()
);
assertDoesNotThrow(() ->
notifier.sendNotification(message, "Object Detected", 0)
);
}
@Test
@Order(9)
@DisplayName("Integration: Handle rapid concurrent updates")
void testConcurrentOperations() throws InterruptedException, SQLException {
var auctionThread = new Thread(() -> {
try {
for (var i = 0; i < 10; i++) {
db.upsertAuction(new AuctionInfo(
60000 + i, "Concurrent Auction " + i, "Test, NL", "Test", "NL",
"https://example.com/60" + i, "A1", 5, null
));
}
} catch (Exception e) {
fail("Auction thread failed: " + e.getMessage());
}
});
var lotThread = new Thread(() -> {
try {
for (var i = 0; i < 10; i++) {
db.upsertLot(Lot.basic(
60000 + i, 70000 + i, "Concurrent Lot " + i, "Desc", "", "", 0, "Cat",
100.0 * i, "EUR", "https://example.com/70" + i, null, false
));
}
} catch (Exception e) {
fail("Lot thread failed: " + e.getMessage());
}
});
auctionThread.start();
lotThread.start();
auctionThread.join();
lotThread.join();
// Verify all were inserted
var auctions = db.getAllAuctions();
var lots = db.getAllLots();
var auctionCount = auctions.stream()
.filter(a -> a.auctionId() >= 60000 && a.auctionId() < 60010)
.count();
var lotCount = lots.stream()
.filter(l -> l.lotId() >= 70000 && l.lotId() < 70010)
.count();
assertEquals(10, auctionCount);
assertEquals(10, lotCount);
}
@Test
@Order(10)
@DisplayName("Integration: End-to-end notification scenarios")
void testAllNotificationScenarios() {
// 1. Bid change notification
assertDoesNotThrow(() ->
notifier.sendNotification(
"Nieuw bod op kavel 12345: €150.00 (was €125.00)",
"Kavel bieding update",
0
)
);
// 2. Closing alert
assertDoesNotThrow(() ->
notifier.sendNotification(
"Kavel 67890 sluit binnen 5 min.",
"Lot nearing closure",
1
)
);
// 3. Object detection
assertDoesNotThrow(() ->
notifier.sendNotification(
"Detected: car, truck, machinery",
"Object Detected",
0
)
);
// 4. Value estimate
assertDoesNotThrow(() ->
notifier.sendNotification(
"Geschatte waarde: €5,000 - €7,500",
"Value Estimate",
0
)
);
// 5. Viewing day reminder
assertDoesNotThrow(() ->
notifier.sendNotification(
"Bezichtiging op 15-12-2025 om 14:00",
"Viewing Day Reminder",
0
)
);
}
}

243
src/test/java/auctiora/NotificationServiceTest.java Normal file
View File

@@ -0,0 +1,243 @@
package auctiora;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
/**
* Test cases for NotificationService.
* Tests desktop and email notification configuration and delivery.
*/
class NotificationServiceTest {
@Test
@DisplayName("Should initialize with desktop-only configuration")
void testDesktopOnlyConfiguration() {
var service = new NotificationService("desktop");
assertNotNull(service);
}
@Test
@DisplayName("Should initialize with SMTP configuration")
void testSMTPConfiguration() {
var service = new NotificationService(
"smtp:test@gmail.com:app_password:recipient@example.com"
);
assertNotNull(service);
}
@Test
@DisplayName("Should reject invalid SMTP configuration format")
void testInvalidSMTPConfiguration() {
// Missing parts (only 2 parts total)
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:incomplete")
);
// Wrong format (only 3 parts total, needs 4)
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:only:two")
);
}
@Test
@DisplayName("Should reject unknown configuration type")
void testUnknownConfiguration() {
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("unknown_type")
);
}
@Test
@DisplayName("Should send desktop notification without error")
void testDesktopNotification() {
var service = new NotificationService("desktop");
// Should not throw exception even if system tray not available
assertDoesNotThrow(() ->
service.sendNotification("Test message", "Test title", 0)
);
}
@Test
@DisplayName("Should send high priority notification")
void testHighPriorityNotification() {
var service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("Urgent message", "High Priority", 1)
);
}
@Test
@DisplayName("Should send normal priority notification")
void testNormalPriorityNotification() {
var service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("Regular message", "Normal Priority", 0)
);
}
@Test
@DisplayName("Should handle notification when system tray not supported")
void testNoSystemTraySupport() {
var service = new NotificationService("desktop");
// Should gracefully handle missing system tray
assertDoesNotThrow(() ->
service.sendNotification("Test", "Test", 0)
);
}
@Test
@DisplayName("Should send email notification with valid SMTP config")
void testEmailNotificationWithValidConfig() {
// Note: This won't actually send email without valid credentials
// But it should initialize properly
var service = new NotificationService(
"smtp:test@gmail.com:fake_password:test@example.com"
);
// Should not throw during initialization
assertNotNull(service);
// Sending will fail with fake credentials, but shouldn't crash
assertDoesNotThrow(() ->
service.sendNotification("Test email", "Email Test", 0)
);
}
@Test
@DisplayName("Should include both desktop and email when SMTP configured")
void testBothNotificationChannels() {
var service = new NotificationService(
"smtp:user@gmail.com:password:recipient@example.com"
);
// Both desktop and email should be attempted
assertDoesNotThrow(() ->
service.sendNotification("Dual channel test", "Test", 0)
);
}
@Test
@DisplayName("Should handle empty message gracefully")
void testEmptyMessage() {
var service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("", "", 0)
);
}
@Test
@DisplayName("Should handle very long message")
void testLongMessage() {
var service = new NotificationService("desktop");
var longMessage = "A".repeat(1000);
assertDoesNotThrow(() ->
service.sendNotification(longMessage, "Long Message Test", 0)
);
}
@Test
@DisplayName("Should handle special characters in message")
void testSpecialCharactersInMessage() {
var service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification(
"€123.45 - Kavel sluit binnen 5 min! ⚠️",
"Special Chars Test",
1
)
);
}
@Test
@DisplayName("Should accept case-insensitive desktop config")
void testCaseInsensitiveDesktopConfig() {
assertDoesNotThrow(() -> {
new NotificationService("DESKTOP");
new NotificationService("Desktop");
new NotificationService("desktop");
});
}
@Test
@DisplayName("Should validate SMTP config parts count")
void testSMTPConfigPartsValidation() {
// Too few parts
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:user:pass")
);
// Too many parts should work (extras ignored in split)
assertDoesNotThrow(() ->
new NotificationService("smtp:user:pass:email:extra")
);
}
@Test
@DisplayName("Should handle multiple rapid notifications")
void testRapidNotifications() {
var service = new NotificationService("desktop");
assertDoesNotThrow(() -> {
for (var i = 0; i < 5; i++) {
service.sendNotification("Notification " + i, "Rapid Test", 0);
}
});
}
@Test
@DisplayName("Should handle notification with null config parameter")
void testNullConfigParameter() {
// Second parameter can be empty string (kept for compatibility)
assertDoesNotThrow(() ->
new NotificationService("desktop")
);
}
@Test
@DisplayName("Should send bid change notification format")
void testBidChangeNotificationFormat() {
var service = new NotificationService("desktop");
var message = "Nieuw bod op kavel 12345: €150.00 (was €125.00)";
var title = "Kavel bieding update";
assertDoesNotThrow(() ->
service.sendNotification(message, title, 0)
);
}
@Test
@DisplayName("Should send closing alert notification format")
void testClosingAlertNotificationFormat() {
var service = new NotificationService("desktop");
var message = "Kavel 12345 sluit binnen 5 min.";
var title = "Lot nearing closure";
assertDoesNotThrow(() ->
service.sendNotification(message, title, 1)
);
}
@Test
@DisplayName("Should send object detection notification format")
void testObjectDetectionNotificationFormat() {
var service = new NotificationService("desktop");
var message = "Lot contains: car, truck, machinery\nEstimated value: €5000";
var title = "Object Detected";
assertDoesNotThrow(() ->
service.sendNotification(message, title, 0)
);
}
}

185
src/test/java/auctiora/ObjectDetectionServiceTest.java Normal file
View File

@@ -0,0 +1,185 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import static org.junit.jupiter.api.Assertions.*;
/**
* Test cases for ObjectDetectionService.
* Tests YOLO model loading and object detection functionality.
*/
class ObjectDetectionServiceTest {
private static final String TEST_CFG = "test_yolo.cfg";
private static final String TEST_WEIGHTS = "test_yolo.weights";
private static final String TEST_CLASSES = "test_classes.txt";
@Test
@DisplayName("Should initialize with missing YOLO models (disabled mode)")
void testInitializeWithoutModels() throws IOException {
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
assertNotNull(service);
}
@Test
@DisplayName("Should return empty list when detection is disabled")
void testDetectObjectsWhenDisabled() throws IOException {
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
var result = service.detectObjects("any_image.jpg");
assertNotNull(result);
assertTrue(result.isEmpty());
}
@Test
@DisplayName("Should handle invalid image path gracefully")
void testInvalidImagePath() throws IOException {
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
var result = service.detectObjects("completely_invalid_path.jpg");
assertNotNull(result);
assertTrue(result.isEmpty());
}
@Test
@DisplayName("Should handle empty image file")
void testEmptyImageFile() throws IOException {
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
// Create empty test file
var tempFile = Files.createTempFile("test_image", ".jpg");
try {
var result = service.detectObjects(tempFile.toString());
assertNotNull(result);
assertTrue(result.isEmpty());
} finally {
Files.deleteIfExists(tempFile);
}
}
@Test
@DisplayName("Should gracefully handle when model files exist but OpenCV fails to load")
void testInitializeWithValidModels() throws IOException {
var cfgPath = Paths.get(TEST_CFG);
var weightsPath = Paths.get(TEST_WEIGHTS);
var classesPath = Paths.get(TEST_CLASSES);
try {
Files.writeString(cfgPath, "[net]\nwidth=416\nheight=416\n");
Files.write(weightsPath, new byte[]{0, 1, 2, 3});
Files.writeString(classesPath, "person\ncar\ntruck\n");
// When files exist but OpenCV native library isn't loaded,
// service should construct successfully but be disabled (handled in @PostConstruct)
var service = new ObjectDetectionService(TEST_CFG, TEST_WEIGHTS, TEST_CLASSES);
// Service is created, but init() handles failures gracefully
// detectObjects should return empty list when disabled
assertNotNull(service);
} finally {
Files.deleteIfExists(cfgPath);
Files.deleteIfExists(weightsPath);
Files.deleteIfExists(classesPath);
}
}
@Test
@DisplayName("Should handle missing class names file")
void testMissingClassNamesFile() throws IOException {
// When model files don't exist, service initializes in disabled mode (no exception)
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
assertNotNull(service);
// Verify it returns empty results when disabled
assertTrue(service.detectObjects("test.jpg").isEmpty());
}
@Test
@DisplayName("Should detect when model files are missing")
void testDetectMissingModelFiles() throws IOException {
// Should initialize in disabled mode
ObjectDetectionService service = new ObjectDetectionService(
"missing.cfg",
"missing.weights",
"missing.names"
);
// Should return empty results when disabled
var results = service.detectObjects("test.jpg");
assertTrue(results.isEmpty());
}
@Test
@DisplayName("Should return unique labels only")
void testUniqueLabels() throws IOException {
// When disabled, returns empty list (unique by default)
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
var result = service.detectObjects("test.jpg");
assertNotNull(result);
assertEquals(0, result.size());
}
@Test
@DisplayName("Should handle multiple detections in same image")
void testMultipleDetections() throws IOException {
// Test structure for when detection works
// With actual YOLO models, this would return multiple objects
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
var result = service.detectObjects("test_image.jpg");
assertNotNull(result);
// When disabled, returns empty list
assertTrue(result.isEmpty());
}
@Test
@DisplayName("Should respect confidence threshold")
void testConfidenceThreshold() throws IOException {
// The service uses 0.5 confidence threshold
// This test documents that behavior
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
// Low confidence detections should be filtered out
// (when detection is working)
var result = service.detectObjects("test.jpg");
assertNotNull(result);
}
}

62
src/test/java/auctiora/ParserTest.java Normal file
View File

File diff suppressed because one or more lines are too long

255
src/test/java/auctiora/ScraperDataAdapterTest.java Normal file
View File

@@ -0,0 +1,255 @@
package auctiora;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.DisplayName;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
/**
* Test cases for ScraperDataAdapter.
* Tests conversion from external scraper schema to monitor schema.
*/
class ScraperDataAdapterTest {
@Test
@DisplayName("Should extract numeric ID from text format auction ID")
void testExtractNumericIdFromAuctionId() {
assertEquals(39813, ScraperDataAdapter.extractNumericId("A7-39813"));
assertEquals(12345, ScraperDataAdapter.extractNumericId("A1-12345"));
assertEquals(0, ScraperDataAdapter.extractNumericId(null));
assertEquals(0, ScraperDataAdapter.extractNumericId(""));
assertEquals(0, ScraperDataAdapter.extractNumericId("ABC"));
}
@Test
@DisplayName("Should extract numeric ID from text format lot ID")
void testExtractNumericIdFromLotId() {
// "A1-28505-5" → 285055 (concatenates all digits)
assertEquals(285055, ScraperDataAdapter.extractNumericId("A1-28505-5"));
assertEquals(123456, ScraperDataAdapter.extractNumericId("A7-1234-56"));
}
@Test
@DisplayName("Should return 0 for IDs that exceed Long.MAX_VALUE")
void testExtractNumericIdTooLarge() {
// These IDs are too large for a long (> 19 digits or > Long.MAX_VALUE)
assertEquals(0, ScraperDataAdapter.extractNumericId("856462986966260305674"));
assertEquals(0, ScraperDataAdapter.extractNumericId("28492384530402679688"));
assertEquals(0, ScraperDataAdapter.extractNumericId("A7-856462986966260305674"));
}
@Test
@DisplayName("Should convert scraper auction format to AuctionInfo")
void testFromScraperAuction() throws SQLException {
// Mock ResultSet with scraper format data
ResultSet rs = mock(ResultSet.class);
when(rs.getString("auction_id")).thenReturn("A7-39813");
when(rs.getString("title")).thenReturn("Industrial Equipment Auction");
when(rs.getString("location")).thenReturn("Cluj-Napoca, RO");
when(rs.getString("url")).thenReturn("https://example.com/auction/A7-39813");
when(rs.getInt("lots_count")).thenReturn(150);
when(rs.getString("first_lot_closing_time")).thenReturn("2025-12-15T14:30:00");
AuctionInfo result = ScraperDataAdapter.fromScraperAuction(rs);
assertNotNull(result);
assertEquals(39813, result.auctionId());
assertEquals("Industrial Equipment Auction", result.title());
assertEquals("Cluj-Napoca, RO", result.location());
assertEquals("Cluj-Napoca", result.city());
assertEquals("RO", result.country());
assertEquals("https://example.com/auction/A7-39813", result.url());
assertEquals("A7", result.typePrefix());
assertEquals(150, result.lotCount());
assertNotNull(result.firstLotClosingTime());
}
@Test
@DisplayName("Should handle auction with simple location without country")
void testFromScraperAuctionSimpleLocation() throws SQLException {
ResultSet rs = mock(ResultSet.class);
when(rs.getString("auction_id")).thenReturn("A1-12345");
when(rs.getString("title")).thenReturn("Test Auction");
when(rs.getString("location")).thenReturn("Amsterdam");
when(rs.getString("url")).thenReturn("https://example.com/auction/A1-12345");
when(rs.getInt("lots_count")).thenReturn(50);
when(rs.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo result = ScraperDataAdapter.fromScraperAuction(rs);
assertEquals("Amsterdam", result.city());
assertEquals("", result.country());
assertNull(result.firstLotClosingTime());
}
@Test
@DisplayName("Should convert scraper lot format to Lot")
void testFromScraperLot() throws SQLException {
ResultSet rs = mock(ResultSet.class);
when(rs.getString("lot_id")).thenReturn("A1-28505-5");
when(rs.getString("auction_id")).thenReturn("A7-39813");
when(rs.getString("title")).thenReturn("Forklift Toyota");
when(rs.getString("description")).thenReturn("Electric forklift in good condition");
when(rs.getString("category")).thenReturn("Machinery");
when(rs.getString("current_bid")).thenReturn("€1250.50");
when(rs.getString("closing_time")).thenReturn("2025-12-15T14:30:00");
when(rs.getString("url")).thenReturn("https://example.com/lot/A1-28505-5");
Lot result = ScraperDataAdapter.fromScraperLot(rs);
assertNotNull(result);
assertEquals(285055, result.lotId());
assertEquals(39813, result.saleId());
assertEquals("Forklift Toyota", result.title());
assertEquals("Electric forklift in good condition", result.description());
assertEquals("Machinery", result.category());
assertEquals(1250.50, result.currentBid(), 0.01);
assertEquals("EUR", result.currency());
assertEquals("https://example.com/lot/A1-28505-5", result.url());
assertNotNull(result.closingTime());
assertFalse(result.closingNotified());
}
@Test
@DisplayName("Should parse bid amount from various formats")
void testParseBidAmount() throws SQLException {
// Test €123.45 format
ResultSet rs1 = createLotResultSet("€123.45");
Lot lot1 = ScraperDataAdapter.fromScraperLot(rs1);
assertEquals(123.45, lot1.currentBid(), 0.01);
assertEquals("EUR", lot1.currency());
// Test $50.00 format
ResultSet rs2 = createLotResultSet("$50.00");
Lot lot2 = ScraperDataAdapter.fromScraperLot(rs2);
assertEquals(50.00, lot2.currentBid(), 0.01);
assertEquals("USD", lot2.currency());
// Test "No bids" format
ResultSet rs3 = createLotResultSet("No bids");
Lot lot3 = ScraperDataAdapter.fromScraperLot(rs3);
assertEquals(0.0, lot3.currentBid(), 0.01);
// Test plain number
ResultSet rs4 = createLotResultSet("999.99");
Lot lot4 = ScraperDataAdapter.fromScraperLot(rs4);
assertEquals(999.99, lot4.currentBid(), 0.01);
}
@Test
@DisplayName("Should handle missing or null fields gracefully")
void testHandleNullFields() throws SQLException {
ResultSet rs = mock(ResultSet.class);
when(rs.getString("lot_id")).thenReturn("A1-12345-1");
when(rs.getString("auction_id")).thenReturn("A7-99999");
when(rs.getString("title")).thenReturn("Test Lot");
when(rs.getString("description")).thenReturn(null);
when(rs.getString("category")).thenReturn(null);
when(rs.getString("current_bid")).thenReturn(null);
when(rs.getString("closing_time")).thenReturn(null);
when(rs.getString("url")).thenReturn("https://example.com/lot");
Lot result = ScraperDataAdapter.fromScraperLot(rs);
assertNotNull(result);
assertEquals("", result.description());
assertEquals("", result.category());
assertEquals(0.0, result.currentBid());
assertNull(result.closingTime());
}
@Test
@DisplayName("Should parse various timestamp formats")
void testTimestampParsing() throws SQLException {
// ISO local date time
ResultSet rs1 = mock(ResultSet.class);
setupBasicLotMock(rs1);
when(rs1.getString("closing_time")).thenReturn("2025-12-15T14:30:00");
Lot lot1 = ScraperDataAdapter.fromScraperLot(rs1);
assertNotNull(lot1.closingTime());
assertEquals(LocalDateTime.of(2025, 12, 15, 14, 30, 0), lot1.closingTime());
// SQL timestamp format
ResultSet rs2 = mock(ResultSet.class);
setupBasicLotMock(rs2);
when(rs2.getString("closing_time")).thenReturn("2025-12-15 14:30:00");
Lot lot2 = ScraperDataAdapter.fromScraperLot(rs2);
assertNotNull(lot2.closingTime());
}
@Test
@DisplayName("Should handle invalid timestamp gracefully")
void testInvalidTimestamp() throws SQLException {
ResultSet rs = mock(ResultSet.class);
setupBasicLotMock(rs);
when(rs.getString("closing_time")).thenReturn("invalid-date");
Lot result = ScraperDataAdapter.fromScraperLot(rs);
assertNull(result.closingTime());
}
@Test
@DisplayName("Should extract type prefix from auction ID")
void testTypeExtraction() throws SQLException {
ResultSet rs1 = mock(ResultSet.class);
when(rs1.getString("auction_id")).thenReturn("A7-39813");
when(rs1.getString("title")).thenReturn("Test");
when(rs1.getString("location")).thenReturn("Test, NL");
when(rs1.getString("url")).thenReturn("http://test.com");
when(rs1.getInt("lots_count")).thenReturn(10);
when(rs1.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo auction1 = ScraperDataAdapter.fromScraperAuction(rs1);
assertEquals("A7", auction1.typePrefix());
ResultSet rs2 = mock(ResultSet.class);
when(rs2.getString("auction_id")).thenReturn("B1-12345");
when(rs2.getString("title")).thenReturn("Test");
when(rs2.getString("location")).thenReturn("Test, NL");
when(rs2.getString("url")).thenReturn("http://test.com");
when(rs2.getInt("lots_count")).thenReturn(10);
when(rs2.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo auction2 = ScraperDataAdapter.fromScraperAuction(rs2);
assertEquals("B1", auction2.typePrefix());
}
@Test
@DisplayName("Should handle GBP currency symbol")
void testGBPCurrency() throws SQLException {
ResultSet rs = createLotResultSet("£75.00");
Lot lot = ScraperDataAdapter.fromScraperLot(rs);
assertEquals(75.00, lot.currentBid(), 0.01);
assertEquals("GBP", lot.currency());
}
// Helper methods
private ResultSet createLotResultSet(String bidAmount) throws SQLException {
ResultSet rs = mock(ResultSet.class);
when(rs.getString("lot_id")).thenReturn("A1-12345-1");
when(rs.getString("auction_id")).thenReturn("A7-99999");
when(rs.getString("title")).thenReturn("Test Lot");
when(rs.getString("description")).thenReturn("Test description");
when(rs.getString("category")).thenReturn("Test");
when(rs.getString("current_bid")).thenReturn(bidAmount);
when(rs.getString("closing_time")).thenReturn("2025-12-15T14:30:00");
when(rs.getString("url")).thenReturn("https://example.com/lot");
return rs;
}
private void setupBasicLotMock(ResultSet rs) throws SQLException {
when(rs.getString("lot_id")).thenReturn("A1-12345-1");
when(rs.getString("auction_id")).thenReturn("A7-99999");
when(rs.getString("title")).thenReturn("Test Lot");
when(rs.getString("description")).thenReturn("Test");
when(rs.getString("category")).thenReturn("Test");
when(rs.getString("current_bid")).thenReturn("€100.00");
when(rs.getString("url")).thenReturn("https://example.com/lot");
}
}

380
src/test/java/auctiora/TroostwijkMonitorTest.java Normal file
View File

@@ -0,0 +1,380 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.time.LocalDateTime;
import static org.junit.jupiter.api.Assertions.*;
/**
* Test cases for TroostwijkMonitor.
* Tests monitoring orchestration, bid tracking, and notification triggers.
*/
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TroostwijkMonitorTest {
private String testDbPath;
private TroostwijkMonitor monitor;
@BeforeAll
void setUp() throws SQLException, IOException {
testDbPath = "test_monitor_" + System.currentTimeMillis() + ".db";
monitor = new TroostwijkMonitor(
testDbPath,
"desktop",
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
}
@AfterAll
void tearDown() throws Exception {
Files.deleteIfExists(Paths.get(testDbPath));
}
@Test
@DisplayName("Should initialize monitor successfully")
void testMonitorInitialization() {
assertNotNull(monitor);
assertNotNull(monitor.getDb());
}
@Test
@DisplayName("Should print database stats without error")
void testPrintDatabaseStats() {
assertDoesNotThrow(() -> monitor.printDatabaseStats());
}
@Test
@DisplayName("Should process pending images without error")
void testProcessPendingImages() {
assertDoesNotThrow(() -> monitor.processPendingImages());
}
@Test
@DisplayName("Should handle empty database gracefully")
void testEmptyDatabaseHandling() throws SQLException {
var auctions = monitor.getDb().getAllAuctions();
var lots = monitor.getDb().getAllLots();
assertNotNull(auctions);
assertNotNull(lots);
assertTrue(auctions.isEmpty() || auctions.size() >= 0);
}
@Test
@DisplayName("Should track lots in database")
void testLotTracking() throws SQLException {
// Insert test lot
var lot = Lot.basic(
11111, 22222,
"Test Forklift",
"Electric forklift in good condition",
"Toyota",
"Electric",
2020,
"Machinery",
1500.00,
"EUR",
"https://example.com/lot/22222",
LocalDateTime.now().plusDays(1),
false
);
monitor.getDb().upsertLot(lot);
var lots = monitor.getDb().getAllLots();
assertTrue(lots.stream().anyMatch(l -> l.lotId() == 22222));
}
@Test
@DisplayName("Should monitor lots closing soon")
void testClosingSoonMonitoring() throws SQLException {
// Insert lot closing in 4 minutes
var closingSoon = Lot.basic(
33333, 44444,
"Closing Soon Item",
"Description",
"",
"",
0,
"Category",
100.00,
"EUR",
"https://example.com/lot/44444",
LocalDateTime.now().plusMinutes(4),
false
);
monitor.getDb().upsertLot(closingSoon);
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 44444)
.findFirst()
.orElse(null);
assertNotNull(found);
assertTrue(found.minutesUntilClose() < 30);
}
@Test
@DisplayName("Should identify lots with time remaining")
void testTimeRemainingCalculation() throws SQLException {
var futureLot = Lot.basic(
55555, 66666,
"Future Lot",
"Description",
"",
"",
0,
"Category",
200.00,
"EUR",
"https://example.com/lot/66666",
LocalDateTime.now().plusHours(2),
false
);
monitor.getDb().upsertLot(futureLot);
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 66666)
.findFirst()
.orElse(null);
assertNotNull(found);
assertTrue(found.minutesUntilClose() > 60);
}
@Test
@DisplayName("Should handle lots without closing time")
void testLotsWithoutClosingTime() throws SQLException {
var noClosing = Lot.basic(
77777, 88888,
"No Closing Time",
"Description",
"",
"",
0,
"Category",
150.00,
"EUR",
"https://example.com/lot/88888",
null,
false
);
monitor.getDb().upsertLot(noClosing);
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 88888)
.findFirst()
.orElse(null);
assertNotNull(found);
assertNull(found.closingTime());
}
@Test
@DisplayName("Should track notification status")
void testNotificationStatusTracking() throws SQLException {
var lot = Lot.basic(
99999, 11110,
"Test Notification",
"Description",
"",
"",
0,
"Category",
100.00,
"EUR",
"https://example.com/lot/11110",
LocalDateTime.now().plusMinutes(3),
false
);
monitor.getDb().upsertLot(lot);
// Update notification flag
var notified = Lot.basic(
99999, 11110,
"Test Notification",
"Description",
"",
"",
0,
"Category",
100.00,
"EUR",
"https://example.com/lot/11110",
LocalDateTime.now().plusMinutes(3),
true
);
monitor.getDb().updateLotNotificationFlags(notified);
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 11110)
.findFirst()
.orElse(null);
assertNotNull(found);
assertTrue(found.closingNotified());
}
@Test
@DisplayName("Should update bid amounts")
void testBidAmountUpdates() throws SQLException {
var lot = Lot.basic(
12121, 13131,
"Bid Update Test",
"Description",
"",
"",
0,
"Category",
100.00,
"EUR",
"https://example.com/lot/13131",
LocalDateTime.now().plusDays(1),
false
);
monitor.getDb().upsertLot(lot);
// Simulate bid increase
var higherBid = Lot.basic(
12121, 13131,
"Bid Update Test",
"Description",
"",
"",
0,
"Category",
250.00,
"EUR",
"https://example.com/lot/13131",
LocalDateTime.now().plusDays(1),
false
);
monitor.getDb().updateLotCurrentBid(higherBid);
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 13131)
.findFirst()
.orElse(null);
assertNotNull(found);
assertEquals(250.00, found.currentBid(), 0.01);
}
@Test
@DisplayName("Should handle multiple concurrent lot updates")
void testConcurrentLotUpdates() throws InterruptedException, SQLException {
Thread t1 = new Thread(() -> {
try {
for (int i = 0; i < 5; i++) {
monitor.getDb().upsertLot(Lot.basic(
20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
100.0, "EUR", "https://example.com/" + i, null, false
));
}
} catch (Exception e) {
fail("Thread 1 failed: " + e.getMessage());
}
});
Thread t2 = new Thread(() -> {
try {
for (int i = 5; i < 10; i++) {
monitor.getDb().upsertLot(Lot.basic(
20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
200.0, "EUR", "https://example.com/" + i, null, false
));
}
} catch (Exception e) {
fail("Thread 2 failed: " + e.getMessage());
}
});
t1.start();
t2.start();
t1.join();
t2.join();
var lots = monitor.getDb().getActiveLots();
long count = lots.stream()
.filter(l -> l.lotId() >= 30000 && l.lotId() < 30010)
.count();
assertTrue(count >= 10);
}
@Test
@DisplayName("Should schedule monitoring without error")
void testScheduleMonitoring() {
// This just tests that scheduling doesn't throw
// Actual monitoring would run in background
assertDoesNotThrow(() -> {
// Don't actually start monitoring in test
// Just verify monitor is ready
assertNotNull(monitor);
});
}
@Test
@DisplayName("Should handle database with auctions and lots")
void testDatabaseWithData() throws SQLException {
// Insert auction
var auction = new AuctionInfo(
40000,
"Test Auction",
"Amsterdam, NL",
"Amsterdam",
"NL",
"https://example.com/auction/40000",
"A7",
10,
LocalDateTime.now().plusDays(2)
);
monitor.getDb().upsertAuction(auction);
// Insert related lot
var lot = Lot.basic(
40000, 50000,
"Test Lot",
"Description",
"",
"",
0,
"Category",
500.00,
"EUR",
"https://example.com/lot/50000",
LocalDateTime.now().plusDays(2),
false
);
monitor.getDb().upsertLot(lot);
// Verify
var auctions = monitor.getDb().getAllAuctions();
var lots = monitor.getDb().getAllLots();
assertTrue(auctions.stream().anyMatch(a -> a.auctionId() == 40000));
assertTrue(lots.stream().anyMatch(l -> l.lotId() == 50000));
}
}

20
workflows/maven.yml Normal file
View File

@@ -0,0 +1,20 @@
name: Publish to Gitea Package Registry
on:
push:
tags:
- 'v*'
jobs:
publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up JDK 25
uses: actions/setup-java@v4
with:
java-version: '25'
distribution: 'temurin'
- name: Publish with Maven
run: mvn --batch-mode clean deploy
env:
GITEA_TOKEN: ${{ secrets.EA_PUBLISH_TOKEN }}