tour/auctiora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix-tests-cleanup

Browse Source
This commit is contained in:
Tour
2025-12-08 05:37:29 +01:00
parent 7600cebcbb
commit 62cda5c0cb
22 changed files with 816 additions and 1386 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
SCHEMA_FIX_INSTRUCTIONS.md
View File

@@ -1,113 +0,0 @@
# Database Schema Fix Instructions
## Problem
The server database was created with `BIGINT` primary keys for `auction_id` and `lot_id`, but the scraper uses TEXT IDs like "A7-40063-2". This causes PRIMARY KEY constraint failures.
## Root Cause
- Local DB: `auction_id TEXT PRIMARY KEY`, `lot_id TEXT PRIMARY KEY`
- Server DB (old): `auction_id BIGINT PRIMARY KEY`, `lot_id BIGINT PRIMARY KEY`
- Scraper data: Uses TEXT IDs like "A7-40063-2", "A1-34732-49"
This mismatch prevents the scraper from inserting data, resulting in zero bids showing in the UI.
## Solution
### Step 1: Backup Server Database
```bash
ssh tour@192.168.1.149
cd /mnt/okcomputer/output
cp cache.db cache.db.backup.$(date +%Y%m%d_%H%M%S)
```
### Step 2: Upload Fix Script
From your local machine:
```bash
scp C:\vibe\auctiora\fix-schema.sql tour@192.168.1.149:/tmp/
```
### Step 3: Stop the Application
```bash
ssh tour@192.168.1.149
cd /path/to/docker/compose # wherever docker-compose.yml is located
docker-compose down
```
### Step 4: Apply Schema Fix
```bash
ssh tour@192.168.1.149
cd /mnt/okcomputer/output
sqlite3 cache.db < /tmp/fix-schema.sql
```
### Step 5: Verify Schema
```bash
sqlite3 cache.db "PRAGMA table_info(auctions);"
# Should show: auction_id TEXT PRIMARY KEY
sqlite3 cache.db "PRAGMA table_info(lots);"
# Should show: lot_id TEXT PRIMARY KEY, sale_id TEXT, auction_id TEXT
# Check data integrity
sqlite3 cache.db "SELECT COUNT(*) FROM auctions;"
sqlite3 cache.db "SELECT COUNT(*) FROM lots;"
```
### Step 6: Rebuild Application with Fixed Schema
```bash
# Build new image with fixed DatabaseService.java
cd C:\vibe\auctiora
./mvnw clean package -DskipTests
# Copy new JAR to server
scp target/quarkus-app/quarkus-run.jar tour@192.168.1.149:/path/to/app/
```
### Step 7: Restart Application
```bash
ssh tour@192.168.1.149
cd /path/to/docker/compose
docker-compose up -d
```
### Step 8: Verify Fix
```bash
# Check logs for successful imports
docker-compose logs -f --tail=100
# Should see:
# ✓ Imported XXX auctions
# ✓ Imported XXXXX lots
# No more "PRIMARY KEY constraint failed" errors
# Check UI at http://192.168.1.149:8081/
# Should now show:
# - Lots with Bids: > 0
# - Total Bid Value: > €0.00
# - Average Bid: > €0.00
```
## Alternative: Quick Fix Without Downtime
If you can't afford downtime, delete the corrupted database and let it rebuild:
```bash
ssh tour@192.168.1.149
cd /mnt/okcomputer/output
mv cache.db cache.db.old
docker-compose restart
# The app will create a new database with correct schema
# Wait for scraper to re-populate data (may take 10-15 minutes)
```
## Files Changed
1. `DatabaseService.java` - Fixed schema definitions (auction_id, lot_id, sale_id now TEXT)
2. `fix-schema.sql` - SQL migration script to fix existing database
3. `SCHEMA_FIX_INSTRUCTIONS.md` - This file
## Testing Locally
Before deploying to server, test locally:
```bash
cd C:\vibe\auctiora
./mvnw clean test
# All tests should pass with new schema
```

128
fix-schema.sql
View File

@@ -1,128 +0,0 @@
-- Schema Fix Script for Server Database
-- This script migrates auction_id and lot_id from BIGINT to TEXT to match scraper format
-- The scraper uses TEXT IDs like "A7-40063-2" but DatabaseService.java was creating BIGINT columns
-- Step 1: Backup existing data
CREATE TABLE IF NOT EXISTS auctions_backup AS SELECT * FROM auctions;
CREATE TABLE IF NOT EXISTS lots_backup AS SELECT * FROM lots;
CREATE TABLE IF NOT EXISTS images_backup AS SELECT * FROM images;
-- Step 2: Drop existing tables (CASCADE would drop foreign keys)
DROP TABLE IF EXISTS images;
DROP TABLE IF EXISTS lots;
DROP TABLE IF EXISTS auctions;
-- Step 3: Recreate auctions table with TEXT primary key (matching scraper format)
CREATE TABLE auctions (
auction_id TEXT PRIMARY KEY,
title TEXT NOT NULL,
location TEXT,
city TEXT,
country TEXT,
url TEXT NOT NULL UNIQUE,
type TEXT,
lot_count INTEGER DEFAULT 0,
closing_time TEXT,
discovered_at INTEGER
);
-- Step 4: Recreate lots table with TEXT primary key (matching scraper format)
CREATE TABLE lots (
lot_id TEXT PRIMARY KEY,
sale_id TEXT,
auction_id TEXT,
title TEXT,
description TEXT,
manufacturer TEXT,
type TEXT,
year INTEGER,
category TEXT,
current_bid REAL,
currency TEXT DEFAULT 'EUR',
url TEXT UNIQUE,
closing_time TEXT,
closing_notified INTEGER DEFAULT 0,
starting_bid REAL,
minimum_bid REAL,
status TEXT,
brand TEXT,
model TEXT,
attributes_json TEXT,
first_bid_time TEXT,
last_bid_time TEXT,
bid_velocity REAL,
bid_increment REAL,
year_manufactured INTEGER,
condition_score REAL,
condition_description TEXT,
serial_number TEXT,
damage_description TEXT,
followers_count INTEGER DEFAULT 0,
estimated_min_price REAL,
estimated_max_price REAL,
lot_condition TEXT,
appearance TEXT,
estimated_min REAL,
estimated_max REAL,
next_bid_step_cents INTEGER,
condition TEXT,
category_path TEXT,
city_location TEXT,
country_code TEXT,
bidding_status TEXT,
packaging TEXT,
quantity INTEGER,
vat REAL,
buyer_premium_percentage REAL,
remarks TEXT,
reserve_price REAL,
reserve_met INTEGER,
view_count INTEGER,
bid_count INTEGER,
viewing_time TEXT,
pickup_date TEXT,
location TEXT,
scraped_at TEXT,
FOREIGN KEY (auction_id) REFERENCES auctions(auction_id),
FOREIGN KEY (sale_id) REFERENCES auctions(auction_id)
);
-- Step 5: Recreate images table
CREATE TABLE images (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT,
url TEXT,
local_path TEXT,
labels TEXT,
processed_at INTEGER,
downloaded INTEGER DEFAULT 0,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
);
-- Step 6: Create bid_history table if it doesn't exist
CREATE TABLE IF NOT EXISTS bid_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT,
bid_amount REAL,
bid_time TEXT,
is_autobid INTEGER DEFAULT 0,
bidder_id TEXT,
bidder_number INTEGER,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
);
-- Step 7: Restore data from backup (converting BIGINT to TEXT if needed)
INSERT OR IGNORE INTO auctions SELECT * FROM auctions_backup;
INSERT OR IGNORE INTO lots SELECT * FROM lots_backup;
INSERT OR IGNORE INTO images SELECT * FROM images_backup;
-- Step 8: Create indexes for performance
CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country);
CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id);
CREATE INDEX IF NOT EXISTS idx_lots_auction_id ON lots(auction_id);
CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id);
-- Step 9: Clean up backup tables (optional - comment out if you want to keep backups)
-- DROP TABLE auctions_backup;
-- DROP TABLE lots_backup;
-- DROP TABLE images_backup;

6
pom.xml
View File

@@ -175,6 +175,12 @@
</dependency>
<!-- Mockito for mocking in tests -->
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5-mockito</artifactId>
<version>3.30.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>

18
src/main/java/auctiora/AuctionInfo.java
View File

@@ -7,13 +7,13 @@ import java.time.LocalDateTime;
* Data typically populated by the external scraper process
*/
public record AuctionInfo(
long auctionId, // Unique auction ID (from URL)
String title, // Auction title
String location, // Location (e.g., "Amsterdam, NL")
String city, // City name
String country, // Country code (e.g., "NL")
String url, // Full auction URL
String typePrefix, // Auction type (A1 or A7)
int lotCount, // Number of lots/kavels
LocalDateTime firstLotClosingTime // Closing time if available
long auctionId, // Unique auction ID (from URL)
String title, // Auction title
String location, // Location (e.g., "Amsterdam, NL")
String city, // City name
String country, // Country code (e.g., "NL")
String url, // Full auction URL
String typePrefix, // Auction type (A1 or A7)
int lotCount, // Number of lots/kavels
LocalDateTime firstLotClosingTime // Closing time if available
) { }

151
src/main/java/auctiora/AuctionMonitorHealthCheck.java
View File

@@ -11,93 +11,72 @@ import org.eclipse.microprofile.health.Startup;
import java.nio.file.Files;
import java.nio.file.Paths;
/**
* Health checks for Auction Monitor.
* Provides liveness and readiness probes for Kubernetes/Docker deployment.
*/
@ApplicationScoped
public class AuctionMonitorHealthCheck {
@Inject
DatabaseService db;
/**
* Liveness probe - checks if application is alive
* GET /health/live
*/
@Liveness
public static class LivenessCheck implements HealthCheck {
@Override
public HealthCheckResponse call() {
return HealthCheckResponse.up("Auction Monitor is alive");
}
}
/**
* Readiness probe - checks if application is ready to serve requests
* GET /health/ready
*/
@Readiness
@ApplicationScoped
public static class ReadinessCheck implements HealthCheck {
@Inject
DatabaseService db;
@Override
public HealthCheckResponse call() {
try {
// Check database connection
var auctions = db.getAllAuctions();
// Check database path exists
var dbPath = Paths.get("C:\\mnt\\okcomputer\\output\\cache.db");
if (!Files.exists(dbPath.getParent())) {
return HealthCheckResponse.down("Database directory does not exist");
}
return HealthCheckResponse.named("database")
.up()
.withData("auctions", auctions.size())
.build();
} catch (Exception e) {
return HealthCheckResponse.named("database")
.down()
.withData("error", e.getMessage())
.build();
@Liveness
public static class LivenessCheck
implements HealthCheck {
@Override public HealthCheckResponse call() {
return HealthCheckResponse.up("Auction Monitor is alive");
}
}
@Readiness
@ApplicationScoped
public static class ReadinessCheck
implements HealthCheck {
@Inject DatabaseService db;
@Override
public HealthCheckResponse call() {
try {
var auctions = db.getAllAuctions();
var dbPath = Paths.get("C:\\mnt\\okcomputer\\output\\cache.db");
if (!Files.exists(dbPath.getParent())) {
return HealthCheckResponse.down("Database directory does not exist");
}
}
}
/**
* Startup probe - checks if application has started correctly
* GET /health/started
*/
@Startup
@ApplicationScoped
public static class StartupCheck implements HealthCheck {
@Inject
DatabaseService db;
@Override
public HealthCheckResponse call() {
try {
// Verify database schema
db.ensureSchema();
return HealthCheckResponse.named("startup")
.up()
.withData("message", "Database schema initialized")
.build();
} catch (Exception e) {
return HealthCheckResponse.named("startup")
.down()
.withData("error", e.getMessage())
.build();
}
}
}
return HealthCheckResponse.named("database")
.up()
.withData("auctions", auctions.size())
.build();
} catch (Exception e) {
return HealthCheckResponse.named("database")
.down()
.withData("error", e.getMessage())
.build();
}
}
}
@Startup
@ApplicationScoped
public static class StartupCheck
implements HealthCheck {
@Inject DatabaseService db;
@Override
public HealthCheckResponse call() {
try {
// Verify database schema
db.ensureSchema();
return HealthCheckResponse.named("startup")
.up()
.withData("message", "Database schema initialized")
.build();
} catch (Exception e) {
return HealthCheckResponse.named("startup")
.down()
.withData("error", e.getMessage())
.build();
}
}
}
}

92
src/main/java/auctiora/AuctionMonitorProducer.java
View File

@@ -19,58 +19,42 @@ import java.sql.SQLException;
@Startup
@ApplicationScoped
public class AuctionMonitorProducer {
private static final Logger LOG = Logger.getLogger(AuctionMonitorProducer.class);
@PostConstruct
void init() {
// Load OpenCV native library at startup
try {
nu.pattern.OpenCV.loadLocally();
LOG.info("✓ OpenCV loaded successfully");
} catch (Exception e) {
LOG.warn("⚠️ OpenCV not available - image detection will be disabled: " + e.getMessage());
}
}
@Produces
@Singleton
public DatabaseService produceDatabaseService(
@ConfigProperty(name = "auction.database.path") String dbPath) throws SQLException {
LOG.infof("Initializing DatabaseService with path: %s", dbPath);
var db = new DatabaseService(dbPath);
db.ensureSchema();
return db;
}
@Produces
@Singleton
public NotificationService produceNotificationService(
@ConfigProperty(name = "auction.notification.config") String config) {
LOG.infof("Initializing NotificationService with config: %s", config);
return new NotificationService(config);
}
@Produces
@Singleton
public ObjectDetectionService produceObjectDetectionService(
@ConfigProperty(name = "auction.yolo.config") String cfgPath,
@ConfigProperty(name = "auction.yolo.weights") String weightsPath,
@ConfigProperty(name = "auction.yolo.classes") String classesPath) throws IOException {
LOG.infof("Initializing ObjectDetectionService");
return new ObjectDetectionService(cfgPath, weightsPath, classesPath);
}
@Produces
@Singleton
public ImageProcessingService produceImageProcessingService(
DatabaseService db,
ObjectDetectionService detector) {
LOG.infof("Initializing ImageProcessingService");
return new ImageProcessingService(db, detector);
}
private static final Logger LOG = Logger.getLogger(AuctionMonitorProducer.class);
@PostConstruct void init() {
try {
nu.pattern.OpenCV.loadLocally();
LOG.info("✓ OpenCV loaded successfully");
} catch (Exception e) {
LOG.warn("⚠️ OpenCV not available - image detection will be disabled: " + e.getMessage());
}
}
@Produces @Singleton public DatabaseService produceDatabaseService(
@ConfigProperty(name = "auction.database.path") String dbPath) throws SQLException {
var db = new DatabaseService(dbPath);
db.ensureSchema();
return db;
}
@Produces @Singleton public NotificationService produceNotificationService(
@ConfigProperty(name = "auction.notification.config") String config) {
return new NotificationService(config);
}
@Produces @Singleton public ObjectDetectionService produceObjectDetectionService(
@ConfigProperty(name = "auction.yolo.config") String cfgPath,
@ConfigProperty(name = "auction.yolo.weights") String weightsPath,
@ConfigProperty(name = "auction.yolo.classes") String classesPath) throws IOException {
return new ObjectDetectionService(cfgPath, weightsPath, classesPath);
}
@Produces @Singleton public ImageProcessingService produceImageProcessingService(
DatabaseService db,
ObjectDetectionService detector) {
return new ImageProcessingService(db, detector);
}
}

193
src/main/java/auctiora/AuctionMonitorResource.java
View File

@@ -30,21 +30,12 @@ public class AuctionMonitorResource {
private static final Logger LOG = Logger.getLogger(AuctionMonitorResource.class);
@Inject
DatabaseService db;
@Inject DatabaseService db;
@Inject QuarkusWorkflowScheduler scheduler;
@Inject NotificationService notifier;
@Inject RateLimitedHttpClient httpClient;
@Inject LotEnrichmentService enrichmentService;
@Inject
QuarkusWorkflowScheduler scheduler;
@Inject
NotificationService notifier;
@Inject
RateLimitedHttpClient httpClient;
@Inject
LotEnrichmentService enrichmentService;
/**
* GET /api/monitor/status
* Returns current monitoring status
@@ -99,33 +90,33 @@ public class AuctionMonitorResource {
stats.put("totalImages", db.getImageCount());
// Lot statistics
var activeLots = 0;
var lotsWithBids = 0;
double totalBids = 0;
var hotLots = 0;
var sleeperLots = 0;
var bargainLots = 0;
var lotsClosing1h = 0;
var lotsClosing6h = 0;
var activeLots = 0;
var lotsWithBids = 0;
double totalBids = 0;
var hotLots = 0;
var sleeperLots = 0;
var bargainLots = 0;
var lotsClosing1h = 0;
var lotsClosing6h = 0;
double totalBidVelocity = 0;
int velocityCount = 0;
int velocityCount = 0;
for (var lot : lots) {
long minutesLeft = lot.closingTime() != null ? lot.minutesUntilClose() : Long.MAX_VALUE;
if (lot.closingTime() != null && minutesLeft > 0) {
activeLots++;
// Time-based counts
if (minutesLeft < 60) lotsClosing1h++;
if (minutesLeft < 360) lotsClosing6h++;
}
if (lot.currentBid() > 0) {
lotsWithBids++;
totalBids += lot.currentBid();
}
// Intelligence metrics (require GraphQL enrichment)
if (lot.followersCount() != null && lot.followersCount() > 20) {
hotLots++;
@@ -136,22 +127,22 @@ public class AuctionMonitorResource {
if (lot.isBelowEstimate()) {
bargainLots++;
}
// Bid velocity
if (lot.bidVelocity() != null && lot.bidVelocity() > 0) {
totalBidVelocity += lot.bidVelocity();
velocityCount++;
}
}
// Calculate bids per hour (average velocity across all lots with velocity data)
double bidsPerHour = velocityCount > 0 ? totalBidVelocity / velocityCount : 0;
stats.put("activeLots", activeLots);
stats.put("lotsWithBids", lotsWithBids);
stats.put("totalBidValue", String.format("€%.2f", totalBids));
stats.put("averageBid", lotsWithBids > 0 ? String.format("€%.2f", totalBids / lotsWithBids) : "€0.00");
// Bidding intelligence
stats.put("bidsPerHour", String.format("%.1f", bidsPerHour));
stats.put("hotLots", hotLots);
@@ -159,11 +150,11 @@ public class AuctionMonitorResource {
stats.put("bargainLots", bargainLots);
stats.put("lotsClosing1h", lotsClosing1h);
stats.put("lotsClosing6h", lotsClosing6h);
// Conversion rate
double conversionRate = activeLots > 0 ? (lotsWithBids * 100.0 / activeLots) : 0;
stats.put("conversionRate", String.format("%.1f%%", conversionRate));
return Response.ok(stats).build();
} catch (Exception e) {
@@ -184,12 +175,12 @@ public class AuctionMonitorResource {
try {
var lots = db.getAllLots();
var closingSoon = lots.stream()
.filter(lot -> lot.closingTime() != null)
.filter(lot -> lot.minutesUntilClose() > 0 && lot.minutesUntilClose() <= hours * 60)
.sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose()))
.limit(100)
.toList();
.filter(lot -> lot.closingTime() != null)
.filter(lot -> lot.minutesUntilClose() > 0 && lot.minutesUntilClose() <= hours * 60)
.sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose()))
.limit(100)
.toList();
return Response.ok(closingSoon).build();
} catch (Exception e) {
LOG.error("Failed to get closing soon lots", e);
@@ -198,7 +189,7 @@ public class AuctionMonitorResource {
.build();
}
}
/**
* GET /api/monitor/lots/{lotId}/bid-history
* Returns bid history for a specific lot
@@ -216,7 +207,7 @@ public class AuctionMonitorResource {
.build();
}
}
/**
* POST /api/monitor/trigger/scraper-import
* Manually trigger scraper import workflow
@@ -288,7 +279,7 @@ public class AuctionMonitorResource {
.build();
}
}
/**
* POST /api/monitor/trigger/graphql-enrichment
* Manually trigger GraphQL enrichment for all lots or lots closing soon
@@ -301,15 +292,15 @@ public class AuctionMonitorResource {
if (hours > 0) {
enriched = enrichmentService.enrichClosingSoonLots(hours);
return Response.ok(Map.of(
"message", "GraphQL enrichment triggered for lots closing within " + hours + " hours",
"enrichedCount", enriched
)).build();
"message", "GraphQL enrichment triggered for lots closing within " + hours + " hours",
"enrichedCount", enriched
)).build();
} else {
enriched = enrichmentService.enrichAllActiveLots();
return Response.ok(Map.of(
"message", "GraphQL enrichment triggered for all lots",
"enrichedCount", enriched
)).build();
"message", "GraphQL enrichment triggered for all lots",
"enrichedCount", enriched
)).build();
}
} catch (Exception e) {
LOG.error("Failed to trigger GraphQL enrichment", e);
@@ -318,7 +309,7 @@ public class AuctionMonitorResource {
.build();
}
}
/**
* GET /api/monitor/auctions
* Returns list of all auctions
@@ -375,7 +366,7 @@ public class AuctionMonitorResource {
})
.sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose()))
.toList();
return Response.ok(closingSoon).build();
} catch (Exception e) {
LOG.error("Failed to get closing lots", e);
@@ -530,7 +521,7 @@ public class AuctionMonitorResource {
public Response getCategoryDistribution() {
try {
var lots = db.getAllLots();
// Category distribution
Map<String, Long> distribution = lots.stream()
.filter(l -> l.category() != null && !l.category().isEmpty())
@@ -538,34 +529,34 @@ public class AuctionMonitorResource {
l -> l.category().length() > 20 ? l.category().substring(0, 20) + "..." : l.category(),
Collectors.counting()
));
// Find top category by count
var topCategory = distribution.entrySet().stream()
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse("N/A");
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse("N/A");
// Calculate average bids per category
Map<String, Double> avgBidsByCategory = lots.stream()
.filter(l -> l.category() != null && !l.category().isEmpty() && l.currentBid() > 0)
.collect(Collectors.groupingBy(
l -> l.category().length() > 20 ? l.category().substring(0, 20) + "..." : l.category(),
Collectors.averagingDouble(Lot::currentBid)
));
.filter(l -> l.category() != null && !l.category().isEmpty() && l.currentBid() > 0)
.collect(Collectors.groupingBy(
l -> l.category().length() > 20 ? l.category().substring(0, 20) + "..." : l.category(),
Collectors.averagingDouble(Lot::currentBid)
));
double overallAvgBid = lots.stream()
.filter(l -> l.currentBid() > 0)
.mapToDouble(Lot::currentBid)
.average()
.orElse(0.0);
.filter(l -> l.currentBid() > 0)
.mapToDouble(Lot::currentBid)
.average()
.orElse(0.0);
Map<String, Object> response = new HashMap<>();
response.put("distribution", distribution);
response.put("topCategory", topCategory);
response.put("categoryCount", distribution.size());
response.put("averageBidOverall", String.format("€%.2f", overallAvgBid));
response.put("avgBidsByCategory", avgBidsByCategory);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get category distribution", e);
@@ -663,7 +654,7 @@ public class AuctionMonitorResource {
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse("N/A");
if (!"N/A".equals(topCountry)) {
insights.add(Map.of(
"icon", "fa-globe",
@@ -671,7 +662,7 @@ public class AuctionMonitorResource {
"description", "Top performing country"
));
}
// Add sleeper lots insight
long sleeperCount = lots.stream().filter(Lot::isSleeperLot).count();
if (sleeperCount > 0) {
@@ -681,7 +672,7 @@ public class AuctionMonitorResource {
"description", "High interest, low bids - opportunity?"
));
}
// Add bargain insight
long bargainCount = lots.stream().filter(Lot::isBelowEstimate).count();
if (bargainCount > 5) {
@@ -691,7 +682,7 @@ public class AuctionMonitorResource {
"description", "Priced below auction house estimates"
));
}
// Add watch/followers insight
long highWatchCount = lots.stream()
.filter(l -> l.followersCount() != null && l.followersCount() > 20)
@@ -703,7 +694,7 @@ public class AuctionMonitorResource {
"description", "High follower count, strong competition"
));
}
return Response.ok(insights).build();
} catch (Exception e) {
LOG.error("Failed to get insights", e);
@@ -725,12 +716,12 @@ public class AuctionMonitorResource {
var sleepers = allLots.stream()
.filter(Lot::isSleeperLot)
.toList();
Map<String, Object> response = Map.of(
"count", sleepers.size(),
"lots", sleepers
);
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get sleeper lots", e);
@@ -739,7 +730,7 @@ public class AuctionMonitorResource {
.build();
}
}
/**
* GET /api/monitor/intelligence/bargains
* Returns lots priced below auction house estimates
@@ -759,12 +750,12 @@ public class AuctionMonitorResource {
return ratioA.compareTo(ratioB);
})
.toList();
Map<String, Object> response = Map.of(
"count", bargains.size(),
"lots", bargains
);
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get bargains", e);
@@ -773,7 +764,7 @@ public class AuctionMonitorResource {
.build();
}
}
/**
* GET /api/monitor/intelligence/popular
* Returns lots by popularity level
@@ -791,13 +782,13 @@ public class AuctionMonitorResource {
return followersB.compareTo(followersA);
})
.toList();
Map<String, Object> response = Map.of(
"count", popular.size(),
"level", level,
"lots", popular
);
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get popular lots", e);
@@ -806,7 +797,7 @@ public class AuctionMonitorResource {
.build();
}
}
/**
* GET /api/monitor/intelligence/price-analysis
* Returns price vs estimate analysis
@@ -816,28 +807,28 @@ public class AuctionMonitorResource {
public Response getPriceAnalysis() {
try {
var allLots = db.getAllLots();
long belowEstimate = allLots.stream().filter(Lot::isBelowEstimate).count();
long aboveEstimate = allLots.stream().filter(Lot::isAboveEstimate).count();
long withEstimates = allLots.stream()
.filter(lot -> lot.estimatedMin() != null && lot.estimatedMax() != null)
.count();
double avgPriceVsEstimate = allLots.stream()
.map(Lot::getPriceVsEstimateRatio)
.filter(ratio -> ratio != null)
.mapToDouble(Double::doubleValue)
.average()
.orElse(0.0);
Map<String, Object> response = Map.of(
"totalLotsWithEstimates", withEstimates,
"belowEstimate", belowEstimate,
"aboveEstimate", aboveEstimate,
"averagePriceVsEstimatePercent", Math.round(avgPriceVsEstimate),
"bargainOpportunities", belowEstimate
);
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get price analysis", e);
@@ -846,7 +837,7 @@ public class AuctionMonitorResource {
.build();
}
}
/**
* GET /api/monitor/lots/{lotId}/intelligence
* Returns detailed intelligence for a specific lot
@@ -859,13 +850,13 @@ public class AuctionMonitorResource {
.filter(l -> l.lotId() == lotId)
.findFirst()
.orElse(null);
if (lot == null) {
return Response.status(Response.Status.NOT_FOUND)
.entity(Map.of("error", "Lot not found"))
.build();
}
Map<String, Object> intelligence = new HashMap<>();
intelligence.put("lotId", lot.lotId());
intelligence.put("followersCount", lot.followersCount());
@@ -882,7 +873,7 @@ public class AuctionMonitorResource {
intelligence.put("condition", lot.condition());
intelligence.put("vat", lot.vat());
intelligence.put("buyerPremium", lot.buyerPremiumPercentage());
return Response.ok(intelligence).build();
} catch (Exception e) {
LOG.error("Failed to get lot intelligence", e);
@@ -891,7 +882,7 @@ public class AuctionMonitorResource {
.build();
}
}
/**
* GET /api/monitor/charts/watch-distribution
* Returns follower/watch count distribution
@@ -901,14 +892,14 @@ public class AuctionMonitorResource {
public Response getWatchDistribution() {
try {
var lots = db.getAllLots();
Map<String, Long> distribution = new HashMap<>();
distribution.put("0 watchers", lots.stream().filter(l -> l.followersCount() == null || l.followersCount() == 0).count());
distribution.put("1-5 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 1 && l.followersCount() <= 5).count());
distribution.put("6-20 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 6 && l.followersCount() <= 20).count());
distribution.put("21-50 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 21 && l.followersCount() <= 50).count());
distribution.put("50+ watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() > 50).count());
return Response.ok(distribution).build();
} catch (Exception e) {
LOG.error("Failed to get watch distribution", e);
@@ -917,14 +908,14 @@ public class AuctionMonitorResource {
.build();
}
}
// Helper class for trend data
public static class TrendHour {
public int hour;
public int lots;
public int bids;
public TrendHour(int hour, int lots, int bids) {
this.hour = hour;
this.lots = lots;

160
src/main/java/auctiora/DatabaseService.java
View File

@@ -20,9 +20,9 @@ import java.util.List;
*/
@Slf4j
public class DatabaseService {
private final String url;
DatabaseService(String dbPath) {
// Enable WAL mode and busy timeout for concurrent access
this.url = "jdbc:sqlite:" + dbPath + "?journal_mode=WAL&busy_timeout=10000";
@@ -38,7 +38,7 @@ public class DatabaseService {
stmt.execute("PRAGMA journal_mode=WAL");
stmt.execute("PRAGMA busy_timeout=10000");
stmt.execute("PRAGMA synchronous=NORMAL");
// Cache table (for HTTP caching)
stmt.execute("""
CREATE TABLE IF NOT EXISTS cache (
@@ -47,7 +47,7 @@ public class DatabaseService {
timestamp REAL,
status_code INTEGER
)""");
// Auctions table (populated by external scraper)
stmt.execute("""
CREATE TABLE IF NOT EXISTS auctions (
@@ -65,7 +65,7 @@ public class DatabaseService {
closing_time TEXT,
discovered_at INTEGER
)""");
// Lots table (populated by external scraper)
stmt.execute("""
CREATE TABLE IF NOT EXISTS lots (
@@ -126,7 +126,7 @@ public class DatabaseService {
view_count INTEGER,
FOREIGN KEY (auction_id) REFERENCES auctions(auction_id)
)""");
// Images table (populated by external scraper with URLs and local_path)
stmt.execute("""
CREATE TABLE IF NOT EXISTS images (
@@ -139,7 +139,7 @@ public class DatabaseService {
processed_at INTEGER,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)""");
// Bid history table
stmt.execute("""
CREATE TABLE IF NOT EXISTS bid_history (
@@ -153,7 +153,7 @@ public class DatabaseService {
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)""");
// Indexes for performance
stmt.execute("CREATE INDEX IF NOT EXISTS idx_timestamp ON cache(timestamp)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)");
@@ -164,9 +164,8 @@ public class DatabaseService {
stmt.execute("CREATE INDEX IF NOT EXISTS idx_bid_history_bidder ON bid_history(bidder_id)");
}
}
/**
/**
* Inserts or updates an auction record (typically called by external scraper)
* Handles both auction_id conflicts and url uniqueness constraints
*/
@@ -185,7 +184,7 @@ public class DatabaseService {
lot_count = excluded.lot_count,
closing_time = excluded.closing_time
""";
try (var conn = DriverManager.getConnection(url)) {
try (var ps = conn.prepareStatement(insertSql)) {
ps.setLong(1, auction.auctionId());
@@ -205,7 +204,7 @@ public class DatabaseService {
if (errMsg.contains("UNIQUE constraint failed: auctions.auction_id") ||
errMsg.contains("UNIQUE constraint failed: auctions.url") ||
errMsg.contains("PRIMARY KEY constraint failed")) {
// Try updating by URL as fallback (most reliable unique identifier)
var updateByUrlSql = """
UPDATE auctions SET
@@ -229,12 +228,12 @@ public class DatabaseService {
ps.setInt(7, auction.lotCount());
ps.setString(8, auction.firstLotClosingTime() != null ? auction.firstLotClosingTime().toString() : null);
ps.setString(9, auction.url());
int updated = ps.executeUpdate();
if (updated == 0) {
// Auction doesn't exist by URL either - this is unexpected
log.warn("Could not insert or update auction with url={}, auction_id={} - constraint violation but no existing record found",
auction.url(), auction.auctionId());
auction.url(), auction.auctionId());
} else {
log.debug("Updated existing auction by URL: {}", auction.url());
}
@@ -255,12 +254,12 @@ public class DatabaseService {
synchronized List<AuctionInfo> getAllAuctions() throws SQLException {
List<AuctionInfo> auctions = new ArrayList<>();
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time FROM auctions";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
var closingStr = rs.getString("closing_time");
LocalDateTime closing = null;
var closingStr = rs.getString("closing_time");
LocalDateTime closing = null;
if (closingStr != null && !closingStr.isBlank()) {
try {
closing = LocalDateTime.parse(closingStr);
@@ -268,7 +267,7 @@ public class DatabaseService {
log.debug("Invalid closing_time format for auction {}: {}", rs.getLong("auction_id"), closingStr);
}
}
auctions.add(new AuctionInfo(
rs.getLong("auction_id"),
rs.getString("title"),
@@ -292,13 +291,13 @@ public class DatabaseService {
List<AuctionInfo> auctions = new ArrayList<>();
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time "
+ "FROM auctions WHERE country = ?";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setString(1, countryCode);
var rs = ps.executeQuery();
while (rs.next()) {
var closingStr = rs.getString("closing_time");
LocalDateTime closing = null;
var closingStr = rs.getString("closing_time");
LocalDateTime closing = null;
if (closingStr != null && !closingStr.isBlank()) {
try {
closing = LocalDateTime.parse(closingStr);
@@ -306,7 +305,7 @@ public class DatabaseService {
log.debug("Invalid closing_time format for auction {}: {}", rs.getLong("auction_id"), closingStr);
}
}
auctions.add(new AuctionInfo(
rs.getLong("auction_id"),
rs.getString("title"),
@@ -343,16 +342,16 @@ public class DatabaseService {
closing_time = ?
WHERE lot_id = ?
""";
var insertSql = """
INSERT OR IGNORE INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""";
try (var conn = DriverManager.getConnection(url)) {
// Try UPDATE first
try (var ps = conn.prepareStatement(updateSql)) {
ps.setLong(1, lot.saleId());
ps.setString(1, String.valueOf(lot.saleId()));
ps.setString(2, lot.title());
ps.setString(3, lot.description());
ps.setString(4, lot.manufacturer());
@@ -363,18 +362,18 @@ public class DatabaseService {
ps.setString(9, lot.currency());
ps.setString(10, lot.url());
ps.setString(11, lot.closingTime() != null ? lot.closingTime().toString() : null);
ps.setLong(12, lot.lotId());
ps.setString(12, String.valueOf(lot.lotId()));
int updated = ps.executeUpdate();
if (updated > 0) {
return; // Successfully updated existing record
}
}
// If no rows updated, try INSERT (ignore if conflicts with UNIQUE constraints)
try (var ps = conn.prepareStatement(insertSql)) {
ps.setLong(1, lot.lotId());
ps.setLong(2, lot.saleId());
ps.setString(1, String.valueOf(lot.lotId()));
ps.setString(2, String.valueOf(lot.saleId()));
ps.setString(3, lot.title());
ps.setString(4, lot.description());
ps.setString(5, lot.manufacturer());
@@ -390,7 +389,7 @@ public class DatabaseService {
}
}
}
/**
* Updates a lot with full intelligence data from GraphQL enrichment.
* This is a comprehensive update that includes all 24 intelligence fields.
@@ -434,7 +433,7 @@ public class DatabaseService {
bid_velocity = ?
WHERE lot_id = ?
""";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setLong(1, lot.saleId());
ps.setString(2, lot.title());
@@ -447,12 +446,16 @@ public class DatabaseService {
ps.setString(9, lot.currency());
ps.setString(10, lot.url());
ps.setString(11, lot.closingTime() != null ? lot.closingTime().toString() : null);
// Intelligence fields
if (lot.followersCount() != null) ps.setInt(12, lot.followersCount()); else ps.setNull(12, java.sql.Types.INTEGER);
if (lot.estimatedMin() != null) ps.setDouble(13, lot.estimatedMin()); else ps.setNull(13, java.sql.Types.REAL);
if (lot.estimatedMax() != null) ps.setDouble(14, lot.estimatedMax()); else ps.setNull(14, java.sql.Types.REAL);
if (lot.nextBidStepInCents() != null) ps.setLong(15, lot.nextBidStepInCents()); else ps.setNull(15, java.sql.Types.BIGINT);
if (lot.followersCount() != null) ps.setInt(12, lot.followersCount());
else ps.setNull(12, java.sql.Types.INTEGER);
if (lot.estimatedMin() != null) ps.setDouble(13, lot.estimatedMin());
else ps.setNull(13, java.sql.Types.REAL);
if (lot.estimatedMax() != null) ps.setDouble(14, lot.estimatedMax());
else ps.setNull(14, java.sql.Types.REAL);
if (lot.nextBidStepInCents() != null) ps.setLong(15, lot.nextBidStepInCents());
else ps.setNull(15, java.sql.Types.BIGINT);
ps.setString(16, lot.condition());
ps.setString(17, lot.categoryPath());
ps.setString(18, lot.cityLocation());
@@ -460,28 +463,37 @@ public class DatabaseService {
ps.setString(20, lot.biddingStatus());
ps.setString(21, lot.appearance());
ps.setString(22, lot.packaging());
if (lot.quantity() != null) ps.setLong(23, lot.quantity()); else ps.setNull(23, java.sql.Types.BIGINT);
if (lot.vat() != null) ps.setDouble(24, lot.vat()); else ps.setNull(24, java.sql.Types.REAL);
if (lot.buyerPremiumPercentage() != null) ps.setDouble(25, lot.buyerPremiumPercentage()); else ps.setNull(25, java.sql.Types.REAL);
if (lot.quantity() != null) ps.setLong(23, lot.quantity());
else ps.setNull(23, java.sql.Types.BIGINT);
if (lot.vat() != null) ps.setDouble(24, lot.vat());
else ps.setNull(24, java.sql.Types.REAL);
if (lot.buyerPremiumPercentage() != null) ps.setDouble(25, lot.buyerPremiumPercentage());
else ps.setNull(25, java.sql.Types.REAL);
ps.setString(26, lot.remarks());
if (lot.startingBid() != null) ps.setDouble(27, lot.startingBid()); else ps.setNull(27, java.sql.Types.REAL);
if (lot.reservePrice() != null) ps.setDouble(28, lot.reservePrice()); else ps.setNull(28, java.sql.Types.REAL);
if (lot.reserveMet() != null) ps.setInt(29, lot.reserveMet() ? 1 : 0); else ps.setNull(29, java.sql.Types.INTEGER);
if (lot.bidIncrement() != null) ps.setDouble(30, lot.bidIncrement()); else ps.setNull(30, java.sql.Types.REAL);
if (lot.viewCount() != null) ps.setInt(31, lot.viewCount()); else ps.setNull(31, java.sql.Types.INTEGER);
if (lot.startingBid() != null) ps.setDouble(27, lot.startingBid());
else ps.setNull(27, java.sql.Types.REAL);
if (lot.reservePrice() != null) ps.setDouble(28, lot.reservePrice());
else ps.setNull(28, java.sql.Types.REAL);
if (lot.reserveMet() != null) ps.setInt(29, lot.reserveMet() ? 1 : 0);
else ps.setNull(29, java.sql.Types.INTEGER);
if (lot.bidIncrement() != null) ps.setDouble(30, lot.bidIncrement());
else ps.setNull(30, java.sql.Types.REAL);
if (lot.viewCount() != null) ps.setInt(31, lot.viewCount());
else ps.setNull(31, java.sql.Types.INTEGER);
ps.setString(32, lot.firstBidTime() != null ? lot.firstBidTime().toString() : null);
ps.setString(33, lot.lastBidTime() != null ? lot.lastBidTime().toString() : null);
if (lot.bidVelocity() != null) ps.setDouble(34, lot.bidVelocity()); else ps.setNull(34, java.sql.Types.REAL);
if (lot.bidVelocity() != null) ps.setDouble(34, lot.bidVelocity());
else ps.setNull(34, java.sql.Types.REAL);
ps.setLong(35, lot.lotId());
int updated = ps.executeUpdate();
if (updated == 0) {
log.warn("Failed to update lot {} - lot not found in database", lot.lotId());
}
}
}
/**
* Inserts a complete image record (for testing/legacy compatibility).
* In production, scraper inserts with local_path, monitor updates labels via updateImageLabels.
@@ -497,7 +509,7 @@ public class DatabaseService {
ps.executeUpdate();
}
}
/**
* Updates the labels field for an image after object detection
*/
@@ -510,7 +522,7 @@ public class DatabaseService {
ps.executeUpdate();
}
}
/**
* Gets the labels for a specific image
*/
@@ -559,7 +571,7 @@ public class DatabaseService {
List<Lot> list = new ArrayList<>();
var sql = "SELECT lot_id, sale_id as auction_id, title, description, manufacturer, type, year, category, " +
"current_bid, currency, url, closing_time, closing_notified FROM lots";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
@@ -603,7 +615,7 @@ public class DatabaseService {
try (var conn = DriverManager.getConnection(url);
var ps = conn.prepareStatement("UPDATE lots SET current_bid = ? WHERE lot_id = ?")) {
ps.setDouble(1, lot.currentBid());
ps.setLong(2, lot.lotId());
ps.setString(2, String.valueOf(lot.lotId()));
ps.executeUpdate();
}
}
@@ -615,11 +627,11 @@ public class DatabaseService {
try (var conn = DriverManager.getConnection(url);
var ps = conn.prepareStatement("UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) {
ps.setInt(1, lot.closingNotified() ? 1 : 0);
ps.setLong(2, lot.lotId());
ps.setString(2, String.valueOf(lot.lotId()));
ps.executeUpdate();
}
}
/**
* Retrieves bid history for a specific lot
*/
@@ -627,15 +639,15 @@ public class DatabaseService {
List<BidHistory> history = new ArrayList<>();
var sql = "SELECT id, lot_id, bid_amount, bid_time, is_autobid, bidder_id, bidder_number " +
"FROM bid_history WHERE lot_id = ? ORDER BY bid_time DESC LIMIT 100";
try (var conn = DriverManager.getConnection(url);
var ps = conn.prepareStatement(sql)) {
ps.setString(1, lotId);
var rs = ps.executeQuery();
while (rs.next()) {
LocalDateTime bidTime = null;
var bidTimeStr = rs.getString("bid_time");
LocalDateTime bidTime = null;
var bidTimeStr = rs.getString("bid_time");
if (bidTimeStr != null && !bidTimeStr.isBlank()) {
try {
bidTime = LocalDateTime.parse(bidTimeStr);
@@ -643,7 +655,7 @@ public class DatabaseService {
log.debug("Invalid bid_time format: {}", bidTimeStr);
}
}
history.add(new BidHistory(
rs.getInt("id"),
rs.getString("lot_id"),
@@ -667,7 +679,7 @@ public class DatabaseService {
*/
synchronized List<AuctionInfo> importAuctionsFromScraper() throws SQLException {
List<AuctionInfo> imported = new ArrayList<>();
// Derive auctions from lots table (scraper doesn't populate auctions table)
var sql = """
SELECT
@@ -682,7 +694,7 @@ public class DatabaseService {
WHERE l.auction_id IS NOT NULL
GROUP BY l.auction_id
""";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
@@ -707,7 +719,7 @@ public class DatabaseService {
// Table might not exist in scraper format - that's ok
log.info("ℹ️ Scraper lots table not found or incompatible schema: {}", e.getMessage());
}
return imported;
}
@@ -722,7 +734,7 @@ public class DatabaseService {
var sql = "SELECT lot_id, auction_id, title, description, category, " +
"current_bid, closing_time, url " +
"FROM lots";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
@@ -743,7 +755,7 @@ public class DatabaseService {
// Table might not exist in scraper format - that's ok
log.info("ℹ️ Scraper lots table not found or incompatible schema");
}
return imported;
}
@@ -762,14 +774,14 @@ public class DatabaseService {
AND i.local_path != ''
AND (i.labels IS NULL OR i.labels = '')
""";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
// Extract numeric lot ID from TEXT field (e.g., "A1-34732-49" -> 3473249)
String lotIdStr = rs.getString("lot_id");
long lotId = ScraperDataAdapter.extractNumericId(lotIdStr);
long lotId = ScraperDataAdapter.extractNumericId(lotIdStr);
images.add(new ImageDetectionRecord(
rs.getInt("id"),
lotId,
@@ -779,17 +791,11 @@ public class DatabaseService {
} catch (SQLException e) {
log.info("ℹ️ No images needing detection found");
}
return images;
}
/**
* Simple record for image data from database
*/
record ImageRecord(int id, long lotId, String url, String filePath, String labels) { }
/**
* Record for images that need object detection processing
*/
record ImageDetectionRecord(int id, long lotId, String filePath) { }
}

113
src/main/java/auctiora/ImageProcessingService.java
View File

@@ -1,106 +1,55 @@
package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.util.List;
/**
* Service responsible for processing images from the IMAGES table.
* Performs object detection on already-downloaded images and updates the database.
*
* NOTE: Image downloading is handled by the external scraper process.
* This service only performs object detection on images that already have local_path set.
*/
@Slf4j
class ImageProcessingService {
private final DatabaseService db;
private final ObjectDetectionService detector;
ImageProcessingService(DatabaseService db, ObjectDetectionService detector) {
this.db = db;
this.detector = detector;
}
/**
* Processes a single image: runs object detection and updates labels in database.
*
* @param imageId database ID of the image record
* @param localPath local file path to the downloaded image
* @param lotId lot identifier (for logging)
* @return true if processing succeeded
*/
boolean processImage(int imageId, String localPath, long lotId) {
public record ImageProcessingService(DatabaseService db, ObjectDetectionService detector) {
boolean processImage(int id, String path, long lot) {
try {
// Normalize path separators (convert Windows backslashes to forward slashes)
localPath = localPath.replace('\\', '/');
// Check if file exists before processing
var file = new java.io.File(localPath);
if (!file.exists() || !file.canRead()) {
log.warn(" Image file not accessible: {}", localPath);
path = path.replace('\\', '/');
var f = new java.io.File(path);
if (!f.exists() || !f.canRead()) {
log.warn("Image not accessible: {}", path);
return false;
}
// Check file size (skip very large files that might cause issues)
long fileSizeBytes = file.length();
if (fileSizeBytes > 50 * 1024 * 1024) { // 50 MB limit
log.warn(" Image file too large ({}MB): {}", fileSizeBytes / (1024 * 1024), localPath);
if (f.length() > 50L * 1024 * 1024) {
log.warn("Image too large: {}", path);
return false;
}
// Run object detection on the local file
var labels = detector.detectObjects(localPath);
// Update the database with detected labels
db.updateImageLabels(imageId, labels);
if (!labels.isEmpty()) {
log.info(" Lot {}: Detected {}", lotId, String.join(", ", labels));
}
var labels = detector.detectObjects(path);
db.updateImageLabels(id, labels);
if (!labels.isEmpty())
log.info("Lot {}: {}", lot, String.join(", ", labels));
return true;
} catch (Exception e) {
log.error(" Failed to process image {}: {}", imageId, e.getMessage());
log.error("Process fail {}: {}", id, e.getMessage());
return false;
}
}
/**
* Batch processes all pending images in the database.
* Only processes images that have been downloaded by the scraper but haven't had object detection run yet.
*/
void processPendingImages() {
log.info("Processing pending images...");
try {
var pendingImages = db.getImagesNeedingDetection();
log.info("Found {} images needing object detection", pendingImages.size());
var processed = 0;
var detected = 0;
for (var image : pendingImages) {
if (processImage(image.id(), image.filePath(), image.lotId())) {
var images = db.getImagesNeedingDetection();
log.info("Pending {}", images.size());
int processed = 0, detected = 0;
for (var i : images) {
if (processImage(i.id(), i.filePath(), i.lotId())) {
processed++;
// Re-fetch to check if labels were found
var labels = db.getImageLabels(image.id());
if (labels != null && !labels.isEmpty()) {
detected++;
}
var lbl = db.getImageLabels(i.id());
if (lbl != null && !lbl.isEmpty()) detected++;
}
}
log.info("Processed {} images, detected objects in {}", processed, detected);
} catch (SQLException e) {
log.error("Error processing pending images: {}", e.getMessage());
log.info("Processed {}, detected {}", processed, detected);
} catch (Exception e) {
log.error("Batch fail: {}", e.getMessage());
}
}
}

131
src/main/java/auctiora/LotEnrichmentScheduler.java
View File

@@ -16,73 +16,66 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@ApplicationScoped
public class LotEnrichmentScheduler {
@Inject
LotEnrichmentService enrichmentService;
/**
* Enriches lots closing within 1 hour - HIGH PRIORITY
* Runs every 5 minutes
*/
@Scheduled(cron = "0 */5 * * * ?")
public void enrichCriticalLots() {
try {
log.debug("Enriching critical lots (closing < 1 hour)");
int enriched = enrichmentService.enrichClosingSoonLots(1);
if (enriched > 0) {
log.info("Enriched {} critical lots", enriched);
}
} catch (Exception e) {
log.error("Failed to enrich critical lots", e);
}
}
/**
* Enriches lots closing within 6 hours - MEDIUM PRIORITY
* Runs every 30 minutes
*/
@Scheduled(cron = "0 */30 * * * ?")
public void enrichUrgentLots() {
try {
log.debug("Enriching urgent lots (closing < 6 hours)");
int enriched = enrichmentService.enrichClosingSoonLots(6);
if (enriched > 0) {
log.info("Enriched {} urgent lots", enriched);
}
} catch (Exception e) {
log.error("Failed to enrich urgent lots", e);
}
}
/**
* Enriches lots closing within 24 hours - NORMAL PRIORITY
* Runs every 2 hours
*/
@Scheduled(cron = "0 0 */2 * * ?")
public void enrichDailyLots() {
try {
log.debug("Enriching daily lots (closing < 24 hours)");
int enriched = enrichmentService.enrichClosingSoonLots(24);
if (enriched > 0) {
log.info("Enriched {} daily lots", enriched);
}
} catch (Exception e) {
log.error("Failed to enrich daily lots", e);
}
}
/**
* Enriches all active lots - LOW PRIORITY
* Runs every 6 hours to keep all data fresh
*/
@Scheduled(cron = "0 0 */6 * * ?")
public void enrichAllLots() {
try {
log.info("Starting full enrichment of all lots");
int enriched = enrichmentService.enrichAllActiveLots();
log.info("Full enrichment complete: {} lots updated", enriched);
} catch (Exception e) {
log.error("Failed to enrich all lots", e);
}
}
@Inject LotEnrichmentService enrichmentService;
/**
* Enriches lots closing within 1 hour - HIGH PRIORITY
* Runs every 5 minutes
*/
@Scheduled(cron = "0 */5 * * * ?")
public void enrichCriticalLots() {
try {
log.debug("Enriching critical lots (closing < 1 hour)");
int enriched = enrichmentService.enrichClosingSoonLots(1);
if (enriched > 0) log.info("Enriched {} critical lots", enriched);
} catch (Exception e) {
log.error("Failed to enrich critical lots", e);
}
}
/**
* Enriches lots closing within 6 hours - MEDIUM PRIORITY
* Runs every 30 minutes
*/
@Scheduled(cron = "0 */30 * * * ?")
public void enrichUrgentLots() {
try {
log.debug("Enriching urgent lots (closing < 6 hours)");
int enriched = enrichmentService.enrichClosingSoonLots(6);
if (enriched > 0) log.info("Enriched {} urgent lots", enriched);
} catch (Exception e) {
log.error("Failed to enrich urgent lots", e);
}
}
/**
* Enriches lots closing within 24 hours - NORMAL PRIORITY
* Runs every 2 hours
*/
@Scheduled(cron = "0 0 */2 * * ?")
public void enrichDailyLots() {
try {
log.debug("Enriching daily lots (closing < 24 hours)");
int enriched = enrichmentService.enrichClosingSoonLots(24);
if (enriched > 0) log.info("Enriched {} daily lots", enriched);
} catch (Exception e) {
log.error("Failed to enrich daily lots", e);
}
}
/**
* Enriches all active lots - LOW PRIORITY
* Runs every 6 hours to keep all data fresh
*/
@Scheduled(cron = "0 0 */6 * * ?")
public void enrichAllLots() {
try {
log.info("Starting full enrichment of all lots");
int enriched = enrichmentService.enrichAllActiveLots();
log.info("Full enrichment complete: {} lots updated", enriched);
} catch (Exception e) {
log.error("Failed to enrich all lots", e);
}
}
}

358
src/main/java/auctiora/LotEnrichmentService.java
View File

@@ -16,190 +16,186 @@ import java.util.stream.Collectors;
@Slf4j
@ApplicationScoped
public class LotEnrichmentService {
@Inject
TroostwijkGraphQLClient graphQLClient;
@Inject
DatabaseService db;
/**
* Enriches a single lot with GraphQL intelligence data
*/
public boolean enrichLot(Lot lot) {
if (lot.displayId() == null || lot.displayId().isBlank()) {
log.debug("Cannot enrich lot {} - missing displayId", lot.lotId());
@Inject TroostwijkGraphQLClient graphQLClient;
@Inject DatabaseService db;
/**
* Enriches a single lot with GraphQL intelligence data
*/
public boolean enrichLot(Lot lot) {
if (lot.displayId() == null || lot.displayId().isBlank()) {
log.debug("Cannot enrich lot {} - missing displayId", lot.lotId());
return false;
}
try {
var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId());
if (intelligence == null) {
log.debug("No intelligence data for lot {}", lot.displayId());
return false;
}
try {
}
// Merge intelligence with existing lot data
var enrichedLot = mergeLotWithIntelligence(lot, intelligence);
db.upsertLotWithIntelligence(enrichedLot);
log.debug("Enriched lot {} with GraphQL data", lot.lotId());
return true;
} catch (Exception e) {
log.warn("Failed to enrich lot {}: {}", lot.lotId(), e.getMessage());
return false;
}
}
/**
* Enriches multiple lots sequentially
* @param lots List of lots to enrich
* @return Number of successfully enriched lots
*/
public int enrichLotsBatch(List<Lot> lots) {
if (lots.isEmpty()) {
return 0;
}
log.info("Enriching {} lots via GraphQL", lots.size());
int enrichedCount = 0;
for (var lot : lots) {
if (lot.displayId() == null || lot.displayId().isBlank()) {
log.debug("Skipping lot {} - missing displayId", lot.lotId());
continue;
}
try {
var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId());
if (intelligence == null) {
log.debug("No intelligence data for lot {}", lot.displayId());
return false;
if (intelligence != null) {
var enrichedLot = mergeLotWithIntelligence(lot, intelligence);
db.upsertLotWithIntelligence(enrichedLot);
enrichedCount++;
} else {
log.debug("No intelligence data for lot {}", lot.displayId());
}
// Merge intelligence with existing lot data
var enrichedLot = mergeLotWithIntelligence(lot, intelligence);
db.upsertLotWithIntelligence(enrichedLot);
log.debug("Enriched lot {} with GraphQL data", lot.lotId());
return true;
} catch (Exception e) {
log.warn("Failed to enrich lot {}: {}", lot.lotId(), e.getMessage());
return false;
}
}
/**
* Enriches multiple lots sequentially
* @param lots List of lots to enrich
* @return Number of successfully enriched lots
*/
public int enrichLotsBatch(List<Lot> lots) {
if (lots.isEmpty()) {
} catch (Exception e) {
log.warn("Failed to enrich lot {}: {}", lot.displayId(), e.getMessage());
}
// Small delay to respect rate limits (handled by RateLimitedHttpClient)
}
log.info("Successfully enriched {}/{} lots", enrichedCount, lots.size());
return enrichedCount;
}
/**
* Enriches lots closing soon (within specified hours) with higher priority
*/
public int enrichClosingSoonLots(int hoursUntilClose) {
try {
var allLots = db.getAllLots();
var closingSoon = allLots.stream()
.filter(lot -> lot.closingTime() != null)
.filter(lot -> {
long minutes = lot.minutesUntilClose();
return minutes > 0 && minutes <= hoursUntilClose * 60;
})
.toList();
if (closingSoon.isEmpty()) {
log.debug("No lots closing within {} hours", hoursUntilClose);
return 0;
}
log.info("Enriching {} lots via GraphQL", lots.size());
int enrichedCount = 0;
for (var lot : lots) {
if (lot.displayId() == null || lot.displayId().isBlank()) {
log.debug("Skipping lot {} - missing displayId", lot.lotId());
continue;
}
log.info("Enriching {} lots closing within {} hours", closingSoon.size(), hoursUntilClose);
return enrichLotsBatch(closingSoon);
} catch (Exception e) {
log.error("Failed to enrich closing soon lots: {}", e.getMessage());
return 0;
}
}
/**
* Enriches all active lots (can be slow for large datasets)
*/
public int enrichAllActiveLots() {
try {
var allLots = db.getAllLots();
log.info("Enriching all {} active lots", allLots.size());
// Process in batches to avoid overwhelming the API
int batchSize = 50;
int totalEnriched = 0;
for (int i = 0; i < allLots.size(); i += batchSize) {
int end = Math.min(i + batchSize, allLots.size());
List<Lot> batch = allLots.subList(i, end);
int enriched = enrichLotsBatch(batch);
totalEnriched += enriched;
// Small delay between batches to respect rate limits
if (end < allLots.size()) {
Thread.sleep(1000);
}
try {
var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId());
if (intelligence != null) {
var enrichedLot = mergeLotWithIntelligence(lot, intelligence);
db.upsertLotWithIntelligence(enrichedLot);
enrichedCount++;
} else {
log.debug("No intelligence data for lot {}", lot.displayId());
}
} catch (Exception e) {
log.warn("Failed to enrich lot {}: {}", lot.displayId(), e.getMessage());
}
// Small delay to respect rate limits (handled by RateLimitedHttpClient)
}
log.info("Successfully enriched {}/{} lots", enrichedCount, lots.size());
return enrichedCount;
}
/**
* Enriches lots closing soon (within specified hours) with higher priority
*/
public int enrichClosingSoonLots(int hoursUntilClose) {
try {
var allLots = db.getAllLots();
var closingSoon = allLots.stream()
.filter(lot -> lot.closingTime() != null)
.filter(lot -> {
long minutes = lot.minutesUntilClose();
return minutes > 0 && minutes <= hoursUntilClose * 60;
})
.toList();
if (closingSoon.isEmpty()) {
log.debug("No lots closing within {} hours", hoursUntilClose);
return 0;
}
log.info("Enriching {} lots closing within {} hours", closingSoon.size(), hoursUntilClose);
return enrichLotsBatch(closingSoon);
} catch (Exception e) {
log.error("Failed to enrich closing soon lots: {}", e.getMessage());
return 0;
}
}
/**
* Enriches all active lots (can be slow for large datasets)
*/
public int enrichAllActiveLots() {
try {
var allLots = db.getAllLots();
log.info("Enriching all {} active lots", allLots.size());
// Process in batches to avoid overwhelming the API
int batchSize = 50;
int totalEnriched = 0;
for (int i = 0; i < allLots.size(); i += batchSize) {
int end = Math.min(i + batchSize, allLots.size());
List<Lot> batch = allLots.subList(i, end);
int enriched = enrichLotsBatch(batch);
totalEnriched += enriched;
// Small delay between batches to respect rate limits
if (end < allLots.size()) {
Thread.sleep(1000);
}
}
log.info("Finished enriching all lots. Total enriched: {}/{}", totalEnriched, allLots.size());
return totalEnriched;
} catch (Exception e) {
log.error("Failed to enrich all lots: {}", e.getMessage());
return 0;
}
}
/**
* Merges existing lot data with GraphQL intelligence
*/
private Lot mergeLotWithIntelligence(Lot lot, LotIntelligence intel) {
return new Lot(
lot.saleId(),
lot.lotId(),
lot.displayId(), // Preserve displayId
lot.title(),
lot.description(),
lot.manufacturer(),
lot.type(),
lot.year(),
lot.category(),
lot.currentBid(),
lot.currency(),
lot.url(),
lot.closingTime(),
lot.closingNotified(),
// HIGH PRIORITY FIELDS from GraphQL
intel.followersCount(),
intel.estimatedMin(),
intel.estimatedMax(),
intel.nextBidStepInCents(),
intel.condition(),
intel.categoryPath(),
intel.cityLocation(),
intel.countryCode(),
// MEDIUM PRIORITY FIELDS
intel.biddingStatus(),
intel.appearance(),
intel.packaging(),
intel.quantity(),
intel.vat(),
intel.buyerPremiumPercentage(),
intel.remarks(),
// BID INTELLIGENCE FIELDS
intel.startingBid(),
intel.reservePrice(),
intel.reserveMet(),
intel.bidIncrement(),
intel.viewCount(),
intel.firstBidTime(),
intel.lastBidTime(),
intel.bidVelocity(),
null, // condition_score (computed separately)
null // provenance_docs (computed separately)
);
}
}
log.info("Finished enriching all lots. Total enriched: {}/{}", totalEnriched, allLots.size());
return totalEnriched;
} catch (Exception e) {
log.error("Failed to enrich all lots: {}", e.getMessage());
return 0;
}
}
/**
* Merges existing lot data with GraphQL intelligence
*/
private Lot mergeLotWithIntelligence(Lot lot, LotIntelligence intel) {
return new Lot(
lot.saleId(),
lot.lotId(),
lot.displayId(), // Preserve displayId
lot.title(),
lot.description(),
lot.manufacturer(),
lot.type(),
lot.year(),
lot.category(),
lot.currentBid(),
lot.currency(),
lot.url(),
lot.closingTime(),
lot.closingNotified(),
// HIGH PRIORITY FIELDS from GraphQL
intel.followersCount(),
intel.estimatedMin(),
intel.estimatedMax(),
intel.nextBidStepInCents(),
intel.condition(),
intel.categoryPath(),
intel.cityLocation(),
intel.countryCode(),
// MEDIUM PRIORITY FIELDS
intel.biddingStatus(),
intel.appearance(),
intel.packaging(),
intel.quantity(),
intel.vat(),
intel.buyerPremiumPercentage(),
intel.remarks(),
// BID INTELLIGENCE FIELDS
intel.startingBid(),
intel.reservePrice(),
intel.reserveMet(),
intel.bidIncrement(),
intel.viewCount(),
intel.firstBidTime(),
intel.lastBidTime(),
intel.bidVelocity(),
null, // condition_score (computed separately)
null // provenance_docs (computed separately)
);
}
}

231
src/main/java/auctiora/Main.java
View File

@@ -1,231 +0,0 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import org.opencv.core.Core;
/**
* Main entry point for Troostwijk Auction Monitor.
*
* ARCHITECTURE:
* This project focuses on:
* 1. Image processing and object detection
* 2. Bid monitoring and notifications
* 3. Data enrichment
*
* Auction/Lot scraping is handled by the external ARCHITECTURE-TROOSTWIJK-SCRAPER process.
* That process populates the auctions and lots tables in the shared database.
* This process reads from those tables and enriches them with:
* - Downloaded images
* - Object detection labels
* - Bid monitoring
* - Notifications
*/
@Slf4j
public class Main {
@SuppressWarnings("restricted")
private static Object loadOpenCV() {
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
return null;
}
public static void main(String[] args) throws Exception {
log.info("=== Troostwijk Auction Monitor ===\n");
// Parse command line arguments
var mode = args.length > 0 ? args[0] : "workflow";
// Configuration - Windows paths
var databaseFile = System.getenv().getOrDefault("DATABASE_FILE", "C:\\mnt\\okcomputer\\output\\cache.db");
var notificationConfig = System.getenv().getOrDefault("NOTIFICATION_CONFIG", "desktop");
// YOLO model paths (optional - monitor works without object detection)
var yoloCfg = "models/yolov4.cfg";
var yoloWeights = "models/yolov4.weights";
var yoloClasses = "models/coco.names";
// Load native OpenCV library (only if models exist)
try {
loadOpenCV();
log.info("✓ OpenCV loaded");
} catch (UnsatisfiedLinkError e) {
log.info("⚠️ OpenCV not available - image detection disabled");
}
switch (mode.toLowerCase()) {
case "workflow":
runWorkflowMode(databaseFile, notificationConfig, yoloCfg, yoloWeights, yoloClasses);
break;
case "once":
runOnceMode(databaseFile, notificationConfig, yoloCfg, yoloWeights, yoloClasses);
break;
case "legacy":
runLegacyMode(databaseFile, notificationConfig, yoloCfg, yoloWeights, yoloClasses);
break;
case "status":
showStatus(databaseFile, notificationConfig, yoloCfg, yoloWeights, yoloClasses);
break;
default:
showUsage();
break;
}
}
/**
* WORKFLOW MODE: Run orchestrated scheduled workflows (default)
* This is the recommended mode for production use.
*/
private static void runWorkflowMode(String dbPath, String notifConfig,
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
log.info("🚀 Starting in WORKFLOW MODE (Orchestrated Scheduling)\n");
var orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
);
// Show initial status
orchestrator.printStatus();
// Start all scheduled workflows
orchestrator.startScheduledWorkflows();
log.info("✓ All workflows are running");
log.info(" - Scraper import: every 30 min");
log.info(" - Image processing: every 1 hour");
log.info(" - Bid monitoring: every 15 min");
log.info(" - Closing alerts: every 5 min");
log.info("\nPress Ctrl+C to stop.\n");
// Add shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
log.info("\n🛑 Shutdown signal received...");
orchestrator.shutdown();
}));
// Keep application alive
try {
Thread.sleep(Long.MAX_VALUE);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
orchestrator.shutdown();
}
}
/**
* ONCE MODE: Run complete workflow once and exit
* Useful for cron jobs or scheduled tasks.
*/
private static void runOnceMode(String dbPath, String notifConfig,
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
log.info("🔄 Starting in ONCE MODE (Single Execution)\n");
var orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
);
orchestrator.runCompleteWorkflowOnce();
log.info("✓ Workflow execution completed. Exiting.\n");
}
/**
* LEGACY MODE: Original monitoring approach
* Kept for backward compatibility.
*/
private static void runLegacyMode(String dbPath, String notifConfig,
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
log.info("⚙️ Starting in LEGACY MODE\n");
var monitor = new TroostwijkMonitor(dbPath, notifConfig,
yoloCfg, yoloWeights, yoloClasses);
log.info("\n📊 Current Database State:");
monitor.printDatabaseStats();
log.info("\n[1/2] Processing images...");
monitor.processPendingImages();
log.info("\n[2/2] Starting bid monitoring...");
monitor.scheduleMonitoring();
log.info("\n✓ Monitor is running. Press Ctrl+C to stop.\n");
log.info("NOTE: This process expects auction/lot data from the external scraper.");
log.info(" Make sure ARCHITECTURE-TROOSTWIJK-SCRAPER is running and populating the database.\n");
try {
Thread.sleep(Long.MAX_VALUE);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.info("Monitor interrupted, exiting.");
}
}
/**
* STATUS MODE: Show current status and exit
*/
private static void showStatus(String dbPath, String notifConfig,
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
log.info("📊 Checking Status...\n");
var orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
);
orchestrator.printStatus();
}
/**
* Show usage information
*/
private static void showUsage() {
log.info("Usage: java -jar troostwijk-monitor.jar [mode]\n");
log.info("Modes:");
log.info(" workflow - Run orchestrated scheduled workflows (default)");
log.info(" once - Run complete workflow once and exit (for cron)");
log.info(" legacy - Run original monitoring approach");
log.info(" status - Show current status and exit");
log.info("\nEnvironment Variables:");
log.info(" DATABASE_FILE - Path to SQLite database");
log.info(" (default: C:\\mnt\\okcomputer\\output\\cache.db)");
log.info(" NOTIFICATION_CONFIG - 'desktop' or 'smtp:user:pass:email'");
log.info(" (default: desktop)");
log.info("\nExamples:");
log.info(" java -jar troostwijk-monitor.jar workflow");
log.info(" java -jar troostwijk-monitor.jar once");
log.info(" java -jar troostwijk-monitor.jar status");
IO.println();
}
/**
* Alternative entry point for container environments.
* Simply keeps the container alive for manual commands.
*/
public static void main2(String[] args) {
if (args.length > 0) {
log.info("Command mode - exiting to allow shell commands");
return;
}
log.info("Troostwijk Monitor container is running and healthy.");
log.info("Use 'docker exec' or 'dokku run' to execute commands.");
try {
Thread.sleep(Long.MAX_VALUE);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.info("Container interrupted, exiting.");
}
}
}

82
src/main/java/auctiora/NotificationService.java
View File

@@ -1,48 +1,49 @@
package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import lombok.extern.slf4j.Slf4j;
import javax.mail.*;
import javax.mail.internet.*;
import lombok.extern.slf4j.Slf4j;
import java.awt.*;
import java.util.Date;
import java.util.Properties;
@Slf4j
public class NotificationService {
private final Config config;
public NotificationService(String cfg) {
this.config = Config.parse(cfg);
public record NotificationService(Config cfg) {
// Extra convenience constructor: raw string → Config
public NotificationService(String raw) {
this(Config.parse(raw));
}
public void sendNotification(String message, String title, int priority) {
if (config.useDesktop()) sendDesktop(title, message, priority);
if (config.useEmail()) sendEmail(title, message, priority);
public void sendNotification(String msg, String title, int prio) {
if (cfg.useDesktop()) sendDesktop(title, msg, prio);
if (cfg.useEmail()) sendEmail(title, msg, prio);
}
private void sendDesktop(String title, String msg, int prio) {
try {
if (!SystemTray.isSupported()) {
log.info("Desktop notifications not supported — " + title + " / " + msg);
log.info("Desktop not supported: {}", title);
return;
}
var tray = SystemTray.getSystemTray();
var image = Toolkit.getDefaultToolkit().createImage(new byte[0]);
var trayIcon = new TrayIcon(image, "NotificationService");
trayIcon.setImageAutoSize(true);
var tray = SystemTray.getSystemTray();
var icon = new TrayIcon(
Toolkit.getDefaultToolkit().createImage(new byte[0]),
"notify"
);
icon.setImageAutoSize(true);
tray.add(icon);
var type = prio > 0 ? TrayIcon.MessageType.WARNING : TrayIcon.MessageType.INFO;
tray.add(trayIcon);
trayIcon.displayMessage(title, msg, type);
icon.displayMessage(title, msg, type);
Thread.sleep(2000);
tray.remove(trayIcon);
log.info("Desktop notification sent: " + title);
tray.remove(icon);
log.info("Desktop notification: {}", title);
} catch (Exception e) {
System.err.println("Desktop notification failed: " + e);
log.warn("Desktop failed: {}", e.getMessage());
}
}
@@ -57,29 +58,32 @@ public class NotificationService {
var session = Session.getInstance(props, new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(config.smtpUsername(), config.smtpPassword());
return new PasswordAuthentication(cfg.smtpUsername(), cfg.smtpPassword());
}
});
var m = new MimeMessage(session);
m.setFrom(new InternetAddress(config.smtpUsername()));
m.setRecipients(Message.RecipientType.TO, InternetAddress.parse(config.toEmail()));
m.setFrom(new InternetAddress(cfg.smtpUsername()));
m.setRecipients(Message.RecipientType.TO, InternetAddress.parse(cfg.toEmail()));
m.setSubject("[Troostwijk] " + title);
m.setText(msg);
m.setSentDate(new Date());
if (prio > 0) {
m.setHeader("X-Priority", "1");
m.setHeader("Importance", "High");
}
Transport.send(m);
log.info("Email notification sent: " + title);
log.info("Email notification: {}", title);
} catch (Exception e) {
log.info("Email notification failed: " + e);
log.warn("Email failed: {}", e.getMessage());
}
}
private record Config(
public record Config(
boolean useDesktop,
boolean useEmail,
String smtpUsername,
@@ -87,16 +91,20 @@ public class NotificationService {
String toEmail
) {
static Config parse(String cfg) {
if ("desktop".equalsIgnoreCase(cfg)) {
public static Config parse(String raw) {
if ("desktop".equalsIgnoreCase(raw)) {
return new Config(true, false, null, null, null);
} else if (cfg.startsWith("smtp:")) {
var parts = cfg.split(":", -1); // Use -1 to include trailing empty strings
if (parts.length < 4)
throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'");
return new Config(true, true, parts[1], parts[2], parts[3]);
}
throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'");
if (raw != null && raw.startsWith("smtp:")) {
var p = raw.split(":", -1);
if (p.length < 4) {
throw new IllegalArgumentException("Format: smtp:username:password:toEmail");
}
return new Config(true, true, p[1], p[2], p[3]);
}
throw new IllegalArgumentException("Use 'desktop' or 'smtp:username:password:toEmail'");
}
}
}

62
src/main/java/auctiora/ObjectDetectionService.java
View File

@@ -33,13 +33,13 @@ import static org.opencv.dnn.Dnn.DNN_TARGET_CPU;
*/
@Slf4j
public class ObjectDetectionService {
private final Net net;
private final List<String> classNames;
private final boolean enabled;
private int warnCount = 0;
private static final int MAX_WARNINGS = 5;
private final Net net;
private final List<String> classNames;
private final boolean enabled;
private int warnCount = 0;
private static final int MAX_WARNINGS = 5;
ObjectDetectionService(String cfgPath, String weightsPath, String classNamesPath) throws IOException {
// Check if model files exist
var cfgFile = Paths.get(cfgPath);
@@ -53,38 +53,38 @@ public class ObjectDetectionService {
log.info(" - {}", weightsPath);
log.info(" - {}", classNamesPath);
log.info(" Scraper will continue without image analysis.");
this.enabled = false;
this.net = null;
this.classNames = new ArrayList<>();
enabled = false;
net = null;
classNames = new ArrayList<>();
return;
}
try {
// Load network
this.net = Dnn.readNetFromDarknet(cfgPath, weightsPath);
net = Dnn.readNetFromDarknet(cfgPath, weightsPath);
// Try to use GPU/CUDA if available, fallback to CPU
try {
this.net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA);
this.net.setPreferableTarget(Dnn.DNN_TARGET_CUDA);
net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA);
net.setPreferableTarget(Dnn.DNN_TARGET_CUDA);
log.info("✓ Object detection enabled with YOLO (CUDA/GPU acceleration)");
} catch (Exception e) {
// CUDA not available, try Vulkan for AMD GPUs
try {
this.net.setPreferableBackend(Dnn.DNN_BACKEND_VKCOM);
this.net.setPreferableTarget(Dnn.DNN_TARGET_VULKAN);
net.setPreferableBackend(Dnn.DNN_BACKEND_VKCOM);
net.setPreferableTarget(Dnn.DNN_TARGET_VULKAN);
log.info("✓ Object detection enabled with YOLO (Vulkan/GPU acceleration)");
} catch (Exception e2) {
// GPU not available, fallback to CPU
this.net.setPreferableBackend(DNN_BACKEND_OPENCV);
this.net.setPreferableTarget(DNN_TARGET_CPU);
net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(DNN_TARGET_CPU);
log.info("✓ Object detection enabled with YOLO (CPU only)");
}
}
// Load class names (one per line)
this.classNames = Files.readAllLines(classNamesFile);
this.enabled = true;
classNames = Files.readAllLines(classNamesFile);
enabled = true;
} catch (UnsatisfiedLinkError e) {
System.err.println("⚠️ Object detection disabled: OpenCV native libraries not loaded");
throw new IOException("Failed to initialize object detection: OpenCV native libraries not loaded", e);
@@ -121,15 +121,15 @@ public class ObjectDetectionService {
var confThreshold = 0.5f;
for (var out : outs) {
// YOLO output shape: [num_detections, 85] where 85 = 4 (bbox) + 1 (objectness) + 80 (classes)
int numDetections = out.rows();
int numElements = out.cols();
int numDetections = out.rows();
int numElements = out.cols();
int expectedLength = 5 + classNames.size();
if (numElements < expectedLength) {
// Rate-limit warnings to prevent thread blocking from excessive logging
if (warnCount < MAX_WARNINGS) {
log.warn("Output matrix has wrong dimensions: expected {} columns, got {}. Output shape: [{}, {}]",
expectedLength, numElements, numDetections, numElements);
expectedLength, numElements, numDetections, numElements);
warnCount++;
if (warnCount == MAX_WARNINGS) {
log.warn("Suppressing further dimension warnings (reached {} warnings)", MAX_WARNINGS);
@@ -137,27 +137,27 @@ public class ObjectDetectionService {
}
continue;
}
for (var i = 0; i < numDetections; i++) {
// Get entire row (all 85 elements)
var data = new double[numElements];
for (int j = 0; j < numElements; j++) {
data[j] = out.get(i, j)[0];
}
// Extract objectness score (index 4) and class scores (index 5+)
double objectness = data[4];
if (objectness < confThreshold) {
continue; // Skip low-confidence detections
}
// Extract class scores
var scores = new double[classNames.size()];
System.arraycopy(data, 5, scores, 0, Math.min(scores.length, data.length - 5));
var classId = argMax(scores);
var confidence = scores[classId] * objectness; // Combine objectness with class confidence
if (confidence > confThreshold) {
var label = classNames.get(classId);
if (!labels.contains(label)) {
@@ -166,7 +166,7 @@ public class ObjectDetectionService {
}
}
}
// Release resources
image.release();
blob.release();

51
src/main/java/auctiora/QuarkusWorkflowScheduler.java
View File

@@ -5,6 +5,7 @@ import io.quarkus.scheduler.Scheduled;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
@@ -22,24 +23,14 @@ public class QuarkusWorkflowScheduler {
private static final Logger LOG = Logger.getLogger(QuarkusWorkflowScheduler.class);
@Inject
DatabaseService db;
@Inject DatabaseService db;
@Inject NotificationService notifier;
@Inject ObjectDetectionService detector;
@Inject ImageProcessingService imageProcessor;
@Inject LotEnrichmentService enrichmentService;
@Inject
NotificationService notifier;
@ConfigProperty(name = "auction.database.path") String databasePath;
@Inject
ObjectDetectionService detector;
@Inject
ImageProcessingService imageProcessor;
@Inject
LotEnrichmentService enrichmentService;
@ConfigProperty(name = "auction.database.path")
String databasePath;
/**
* Triggered on application startup to enrich existing lots with bid intelligence
*/
@@ -108,41 +99,41 @@ public class QuarkusWorkflowScheduler {
try {
LOG.info("🖼️ [WORKFLOW 2] Processing pending images...");
var start = System.currentTimeMillis();
// Get images that have been downloaded but need object detection
var pendingImages = db.getImagesNeedingDetection();
if (pendingImages.isEmpty()) {
LOG.info(" → No pending images to process");
return;
}
// Limit batch size to prevent thread blocking (max 100 images per run)
final int MAX_BATCH_SIZE = 100;
int totalPending = pendingImages.size();
int totalPending = pendingImages.size();
if (totalPending > MAX_BATCH_SIZE) {
LOG.infof(" → Found %d pending images, processing first %d (batch limit)",
totalPending, MAX_BATCH_SIZE);
totalPending, MAX_BATCH_SIZE);
pendingImages = pendingImages.subList(0, MAX_BATCH_SIZE);
} else {
LOG.infof(" → Processing %d images", totalPending);
}
var processed = 0;
var detected = 0;
var failed = 0;
for (var image : pendingImages) {
try {
// Run object detection on already-downloaded image
if (imageProcessor.processImage(image.id(), image.filePath(), image.lotId())) {
processed++;
// Check if objects were detected
var labels = db.getImageLabels(image.id());
if (labels != null && !labels.isEmpty()) {
detected++;
// Send notification for interesting detections
if (labels.size() >= 3) {
notifier.sendNotification(
@@ -151,16 +142,16 @@ public class QuarkusWorkflowScheduler {
String.join(", ", labels)),
"Objects Detected",
0
);
);
}
}
} else {
failed++;
}
// Rate limiting (lighter since no network I/O)
Thread.sleep(100);
} catch (Exception e) {
failed++;
LOG.warnf(" ⚠️ Failed to process image: %s", e.getMessage());
@@ -170,7 +161,7 @@ public class QuarkusWorkflowScheduler {
var duration = System.currentTimeMillis() - start;
LOG.infof(" ✓ Processed %d/%d images, detected objects in %d, failed %d (%.1fs)",
processed, totalPending, detected, failed, duration / 1000.0);
if (totalPending > MAX_BATCH_SIZE) {
LOG.infof(" → %d images remaining for next run", totalPending - MAX_BATCH_SIZE);
}
@@ -238,7 +229,7 @@ public class QuarkusWorkflowScheduler {
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
lot.currentBid(), lot.currency(), lot.url(),
lot.closingTime(), true
);
);
db.updateLotNotificationFlags(updated);
alertsSent++;

38
src/main/java/auctiora/RateLimitedHttpClient.java
View File

@@ -142,29 +142,15 @@ public class RateLimitedHttpClient {
* Determines max requests per second for a given host.
*/
private int getMaxRequestsPerSecond(String host) {
if (host.contains("troostwijk")) {
return troostwijkMaxRequestsPerSecond;
}
return defaultMaxRequestsPerSecond;
return host.contains("troostwijk") ? troostwijkMaxRequestsPerSecond : defaultMaxRequestsPerSecond;
}
/**
* Extracts host from URI (e.g., "api.troostwijkauctions.com").
*/
private String extractHost(URI uri) {
return uri.getHost() != null ? uri.getHost() : uri.toString();
}
/**
* Gets statistics for all hosts.
*/
public Map<String, RequestStats> getAllStats() {
return Map.copyOf(requestStats);
}
/**
* Gets statistics for a specific host.
*/
public RequestStats getStats(String host) {
return requestStats.get(host);
}
@@ -218,10 +204,7 @@ public class RateLimitedHttpClient {
}
}
/**
* Statistics tracker for HTTP requests per host.
*/
public static class RequestStats {
public static final class RequestStats {
private final String host;
private final AtomicLong totalRequests = new AtomicLong(0);
@@ -234,25 +217,16 @@ public class RateLimitedHttpClient {
this.host = host;
}
void incrementTotal() {
totalRequests.incrementAndGet();
}
void incrementTotal() { totalRequests.incrementAndGet(); }
void recordSuccess(long durationMs) {
successfulRequests.incrementAndGet();
totalDurationMs.addAndGet(durationMs);
}
void incrementFailed() {
failedRequests.incrementAndGet();
}
void incrementRateLimited() {
rateLimitedRequests.incrementAndGet();
}
// Getters
public String getHost() { return host; }
void incrementFailed() { failedRequests.incrementAndGet(); }
void incrementRateLimited() { rateLimitedRequests.incrementAndGet(); }
public String getHost() { return host; }
public long getTotalRequests() { return totalRequests.get(); }
public long getSuccessfulRequests() { return successfulRequests.get(); }
public long getFailedRequests() { return failedRequests.get(); }

11
src/main/java/auctiora/ScraperDataAdapter.java
View File

@@ -63,13 +63,15 @@ public class ScraperDataAdapter {
lotIdStr, // Store full displayId for GraphQL queries
rs.getString("title"),
getStringOrDefault(rs, "description", ""),
"", "", 0,
getStringOrDefault(rs, "manufacturer", ""),
getStringOrDefault(rs, "type", ""),
getIntOrDefault(rs, "year", 0),
getStringOrDefault(rs, "category", ""),
bid,
currency,
rs.getString("url"),
closing,
false,
getBooleanOrDefault(rs, "closing_notified", false),
// New intelligence fields - set to null for now
null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null,
@@ -166,4 +168,9 @@ public class ScraperDataAdapter {
var v = rs.getInt(col);
return rs.wasNull() ? def : v;
}
private static boolean getBooleanOrDefault(ResultSet rs, String col, boolean def) throws SQLException {
var v = rs.getInt(col);
return rs.wasNull() ? def : v != 0;
}
}

26
src/main/java/auctiora/StatusResource.java
View File

@@ -5,7 +5,9 @@ import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import lombok.extern.slf4j.Slf4j;
import nu.pattern.OpenCV;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.opencv.core.Core;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
@@ -19,18 +21,11 @@ public class StatusResource {
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z")
.withZone(ZoneId.systemDefault());
@ConfigProperty(name = "application.version", defaultValue = "1.0-SNAPSHOT")
String appVersion;
@ConfigProperty(name = "application.groupId")
String groupId;
@ConfigProperty(name = "application.version", defaultValue = "1.0-SNAPSHOT") String appVersion;
@ConfigProperty(name = "application.groupId") String groupId;
@ConfigProperty(name = "application.artifactId") String artifactId;
@ConfigProperty(name = "application.version") String version;
@ConfigProperty(name = "application.artifactId")
String artifactId;
@ConfigProperty(name = "application.version")
String version;
// Java 16+ Record for structured response
public record StatusResponse(
String groupId,
String artifactId,
@@ -47,8 +42,6 @@ public class StatusResource {
@Path("/status")
@Produces(MediaType.APPLICATION_JSON)
public StatusResponse getStatus() {
log.info("Status endpoint called");
return new StatusResponse(groupId, artifactId, version,
"running",
FORMATTER.format(Instant.now()),
@@ -63,8 +56,6 @@ public class StatusResource {
@Path("/hello")
@Produces(MediaType.APPLICATION_JSON)
public Map<String, String> sayHello() {
log.info("hello endpoint called");
return Map.of(
"message", "Hello from Scrape-UI!",
"timestamp", FORMATTER.format(Instant.now()),
@@ -74,9 +65,8 @@ public class StatusResource {
private String getOpenCvVersion() {
try {
// Load OpenCV if not already loaded (safe to call multiple times)
nu.pattern.OpenCV.loadLocally();
return org.opencv.core.Core.VERSION;
OpenCV.loadLocally();
return Core.VERSION;
} catch (Exception e) {
return "4.9.0 (default)";
}

80
src/main/java/auctiora/TroostwijkGraphQLClient.java
View File

@@ -44,15 +44,15 @@ public class TroostwijkGraphQLClient {
}
try {
String query = buildLotQuery();
String variables = buildVariables(displayId);
var query = buildLotQuery();
var variables = buildVariables(displayId);
// Proper GraphQL request format with query and variables
String requestBody = String.format(
var requestBody = String.format(
"{\"query\":\"%s\",\"variables\":%s}",
escapeJson(query),
variables
);
);
var request = java.net.http.HttpRequest.newBuilder()
.uri(java.net.URI.create(GRAPHQL_ENDPOINT))
@@ -87,15 +87,15 @@ public class TroostwijkGraphQLClient {
List<LotIntelligence> results = new ArrayList<>();
// Split into batches of 50 to avoid query size limits
int batchSize = 50;
for (int i = 0; i < lotIds.size(); i += batchSize) {
int end = Math.min(i + batchSize, lotIds.size());
List<Long> batch = lotIds.subList(i, end);
var batchSize = 50;
for (var i = 0; i < lotIds.size(); i += batchSize) {
var end = Math.min(i + batchSize, lotIds.size());
var batch = lotIds.subList(i, end);
try {
String query = buildBatchLotQuery(batch);
String requestBody = String.format("{\"query\":\"%s\"}",
escapeJson(query));
var query = buildBatchLotQuery(batch);
var requestBody = String.format("{\"query\":\"%s\"}",
escapeJson(query));
var request = java.net.http.HttpRequest.newBuilder()
.uri(java.net.URI.create(GRAPHQL_ENDPOINT))
@@ -162,9 +162,9 @@ public class TroostwijkGraphQLClient {
}
private String buildBatchLotQuery(List<Long> lotIds) {
StringBuilder query = new StringBuilder("query {");
var query = new StringBuilder("query {");
for (int i = 0; i < lotIds.size(); i++) {
for (var i = 0; i < lotIds.size(); i++) {
query.append(String.format("""
lot%d: lot(id: %d) {
id
@@ -196,9 +196,9 @@ public class TroostwijkGraphQLClient {
log.debug("GraphQL API returned HTML instead of JSON - likely auth required or wrong endpoint");
return null;
}
JsonNode root = objectMapper.readTree(json);
JsonNode lotNode = root.path("data").path("lotDetails");
var root = objectMapper.readTree(json);
var lotNode = root.path("data").path("lotDetails");
if (lotNode.isMissingNode()) {
log.debug("No lotDetails in GraphQL response");
@@ -206,19 +206,19 @@ public class TroostwijkGraphQLClient {
}
// Extract location from nested object
JsonNode locationNode = lotNode.path("location");
String city = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "city");
String countryCode = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "country");
var locationNode = lotNode.path("location");
var city = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "city");
var countryCode = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "country");
// Extract bids count from nested biddingStatistics
JsonNode statsNode = lotNode.path("biddingStatistics");
Integer bidsCount = statsNode.isMissingNode() ? null : getIntOrNull(statsNode, "numberOfBids");
var statsNode = lotNode.path("biddingStatistics");
var bidsCount = statsNode.isMissingNode() ? null : getIntOrNull(statsNode, "numberOfBids");
// Convert cents to euros for estimates
Long estimatedMinCents = getLongOrNull(lotNode, "estimatedValueInCentsMin");
Long estimatedMaxCents = getLongOrNull(lotNode, "estimatedValueInCentsMax");
Double estimatedMin = estimatedMinCents != null ? estimatedMinCents.doubleValue() : null;
Double estimatedMax = estimatedMaxCents != null ? estimatedMaxCents.doubleValue() : null;
var estimatedMinCents = getLongOrNull(lotNode, "estimatedValueInCentsMin");
var estimatedMaxCents = getLongOrNull(lotNode, "estimatedValueInCentsMax");
var estimatedMin = estimatedMinCents != null ? estimatedMinCents.doubleValue() : null;
var estimatedMax = estimatedMaxCents != null ? estimatedMaxCents.doubleValue() : null;
return new LotIntelligence(
lotId,
@@ -257,11 +257,11 @@ public class TroostwijkGraphQLClient {
List<LotIntelligence> results = new ArrayList<>();
try {
JsonNode root = objectMapper.readTree(json);
JsonNode data = root.path("data");
var root = objectMapper.readTree(json);
var data = root.path("data");
for (int i = 0; i < lotIds.size(); i++) {
JsonNode lotNode = data.path("lot" + i);
for (var i = 0; i < lotIds.size(); i++) {
var lotNode = data.path("lot" + i);
if (!lotNode.isMissingNode()) {
var intelligence = parseLotIntelligenceFromNode(lotNode, lotIds.get(i));
if (intelligence != null) {
@@ -313,17 +313,17 @@ public class TroostwijkGraphQLClient {
private Double calculateBidVelocity(JsonNode lotNode) {
try {
Integer bidsCount = getIntOrNull(lotNode, "bidsCount");
String firstBidStr = getStringOrNull(lotNode, "firstBidTime");
var bidsCount = getIntOrNull(lotNode, "bidsCount");
var firstBidStr = getStringOrNull(lotNode, "firstBidTime");
if (bidsCount == null || firstBidStr == null || bidsCount == 0) {
return null;
}
LocalDateTime firstBid = parseDateTime(firstBidStr);
var firstBid = parseDateTime(firstBidStr);
if (firstBid == null) return null;
long hoursElapsed = java.time.Duration.between(firstBid, LocalDateTime.now()).toHours();
var hoursElapsed = java.time.Duration.between(firstBid, LocalDateTime.now()).toHours();
if (hoursElapsed == 0) return (double) bidsCount;
return (double) bidsCount / hoursElapsed;
@@ -352,27 +352,27 @@ public class TroostwijkGraphQLClient {
}
private Integer getIntOrNull(JsonNode node, String field) {
JsonNode fieldNode = node.path(field);
var fieldNode = node.path(field);
return fieldNode.isNumber() ? fieldNode.asInt() : null;
}
private Long getLongOrNull(JsonNode node, String field) {
JsonNode fieldNode = node.path(field);
var fieldNode = node.path(field);
return fieldNode.isNumber() ? fieldNode.asLong() : null;
}
private Double getDoubleOrNull(JsonNode node, String field) {
JsonNode fieldNode = node.path(field);
var fieldNode = node.path(field);
return fieldNode.isNumber() ? fieldNode.asDouble() : null;
}
private String getStringOrNull(JsonNode node, String field) {
JsonNode fieldNode = node.path(field);
var fieldNode = node.path(field);
return fieldNode.isTextual() ? fieldNode.asText() : null;
}
private Boolean getBooleanOrNull(JsonNode node, String field) {
JsonNode fieldNode = node.path(field);
var fieldNode = node.path(field);
return fieldNode.isBoolean() ? fieldNode.asBoolean() : null;
}
}

86
src/main/java/auctiora/ValuationAnalyticsResource.java
View File

@@ -50,25 +50,25 @@ public class ValuationAnalyticsResource {
public Response calculateValuation(ValuationRequest request) {
try {
LOG.infof("Valuation request for lot: %s", request.lotId);
long startTime = System.currentTimeMillis();
var startTime = System.currentTimeMillis();
// Step 1: Fetch comparable sales from database
List<ComparableLot> comparables = fetchComparables(request);
var comparables = fetchComparables(request);
// Step 2: Calculate Fair Market Value (FMV)
FairMarketValue fmv = calculateFairMarketValue(request, comparables);
var fmv = calculateFairMarketValue(request, comparables);
// Step 3: Calculate undervaluation score
double undervaluationScore = calculateUndervaluationScore(request, fmv.value);
var undervaluationScore = calculateUndervaluationScore(request, fmv.value);
// Step 4: Predict final price
PricePrediction prediction = calculateFinalPrice(request, fmv.value);
var prediction = calculateFinalPrice(request, fmv.value);
// Step 5: Generate bidding strategy
BiddingStrategy strategy = generateBiddingStrategy(request, fmv, prediction);
var strategy = generateBiddingStrategy(request, fmv, prediction);
// Step 6: Compile response
ValuationResponse response = new ValuationResponse();
var response = new ValuationResponse();
response.lotId = request.lotId;
response.timestamp = LocalDateTime.now().toString();
response.fairMarketValue = fmv;
@@ -76,8 +76,8 @@ public class ValuationAnalyticsResource {
response.pricePrediction = prediction;
response.biddingStrategy = strategy;
response.parameters = request;
long duration = System.currentTimeMillis() - startTime;
var duration = System.currentTimeMillis() - startTime;
LOG.infof("Valuation completed in %d ms", duration);
return Response.ok(response).build();
@@ -115,24 +115,24 @@ public class ValuationAnalyticsResource {
* Where weights are exponential/logistic functions of similarity
*/
private FairMarketValue calculateFairMarketValue(ValuationRequest req, List<ComparableLot> comparables) {
double weightedSum = 0.0;
double weightSum = 0.0;
var weightedSum = 0.0;
var weightSum = 0.0;
List<WeightedComparable> weightedComps = new ArrayList<>();
for (ComparableLot comp : comparables) {
for (var comp : comparables) {
// Condition weight: ω_c = exp(-λ_c · |C_target - C_i|)
double omegaC = Math.exp(-0.693 * Math.abs(req.conditionScore - comp.conditionScore));
var omegaC = Math.exp(-0.693 * Math.abs(req.conditionScore - comp.conditionScore));
// Time weight: ω_t = exp(-λ_t · |T_target - T_i|)
double omegaT = Math.exp(-0.048 * Math.abs(req.manufacturingYear - comp.manufacturingYear));
var omegaT = Math.exp(-0.048 * Math.abs(req.manufacturingYear - comp.manufacturingYear));
// Provenance weight: ω_p = 1 + δ_p · (P_target - P_i)
double omegaP = 1 + 0.15 * ((req.provenanceDocs > 0 ? 1 : 0) - comp.hasProvenance);
var omegaP = 1 + 0.15 * ((req.provenanceDocs > 0 ? 1 : 0) - comp.hasProvenance);
// Historical weight: ω_h = 1 / (1 + e^(-kh · (D_i - D_median)))
double omegaH = 1.0 / (1 + Math.exp(-0.01 * (comp.daysAgo - 40)));
double totalWeight = omegaC * omegaT * omegaP * omegaH;
var omegaH = 1.0 / (1 + Math.exp(-0.01 * (comp.daysAgo - 40)));
var totalWeight = omegaC * omegaT * omegaP * omegaH;
weightedSum += comp.finalPrice * totalWeight;
weightSum += totalWeight;
@@ -140,20 +140,20 @@ public class ValuationAnalyticsResource {
// Store for transparency
weightedComps.add(new WeightedComparable(comp, totalWeight, omegaC, omegaT, omegaP, omegaH));
}
double baseFMV = weightSum > 0 ? weightedSum / weightSum : (req.estimatedMin + req.estimatedMax) / 2;
var baseFMV = weightSum > 0 ? weightedSum / weightSum : (req.estimatedMin + req.estimatedMax) / 2;
// Apply condition multiplier: M_cond = exp(α_c · √C_target - β_c)
double conditionMultiplier = Math.exp(0.15 * Math.sqrt(req.conditionScore) - 0.40);
var conditionMultiplier = Math.exp(0.15 * Math.sqrt(req.conditionScore) - 0.40);
baseFMV *= conditionMultiplier;
// Apply provenance premium: Δ_prov = V_base · (η_0 + η_1 · ln(1 + N_docs))
if (req.provenanceDocs > 0) {
double provenancePremium = 0.08 + 0.035 * Math.log(1 + req.provenanceDocs);
var provenancePremium = 0.08 + 0.035 * Math.log(1 + req.provenanceDocs);
baseFMV *= (1 + provenancePremium);
}
FairMarketValue fmv = new FairMarketValue();
var fmv = new FairMarketValue();
fmv.value = Math.round(baseFMV * 100.0) / 100.0;
fmv.conditionMultiplier = Math.round(conditionMultiplier * 1000.0) / 1000.0;
fmv.provenancePremium = req.provenanceDocs > 0 ? 0.08 + 0.035 * Math.log(1 + req.provenanceDocs) : 0.0;
@@ -170,12 +170,12 @@ public class ValuationAnalyticsResource {
*/
private double calculateUndervaluationScore(ValuationRequest req, double fmv) {
if (fmv <= 0) return 0.0;
double priceGap = (fmv - req.currentBid) / fmv;
double velocityFactor = 1 + req.bidVelocity / 10.0;
double watchRatio = Math.log(1 + req.watchCount / Math.max(req.bidCount, 1));
double uScore = priceGap * req.marketVolatility * velocityFactor * watchRatio;
var priceGap = (fmv - req.currentBid) / fmv;
var velocityFactor = 1 + req.bidVelocity / 10.0;
var watchRatio = Math.log(1 + req.watchCount / Math.max(req.bidCount, 1));
var uScore = priceGap * req.marketVolatility * velocityFactor * watchRatio;
return Math.max(0.0, Math.round(uScore * 1000.0) / 1000.0);
}
@@ -186,22 +186,22 @@ public class ValuationAnalyticsResource {
*/
private PricePrediction calculateFinalPrice(ValuationRequest req, double fmv) {
// Bid momentum error: ε_bid = tanh(φ_1 · Λ_b - φ_2 · P_current/FMV)
double epsilonBid = Math.tanh(0.15 * req.bidVelocity - 0.10 * (req.currentBid / fmv));
var epsilonBid = Math.tanh(0.15 * req.bidVelocity - 0.10 * (req.currentBid / fmv));
// Time pressure error: ε_time = ψ · exp(-t_close/30)
double epsilonTime = 0.20 * Math.exp(-req.minutesUntilClose / 30.0);
var epsilonTime = 0.20 * Math.exp(-req.minutesUntilClose / 30.0);
// Competition error: ε_comp = ρ · ln(1 + W_watch/50)
double epsilonComp = 0.08 * Math.log(1 + req.watchCount / 50.0);
double predictedPrice = fmv * (1 + epsilonBid + epsilonTime + epsilonComp);
var epsilonComp = 0.08 * Math.log(1 + req.watchCount / 50.0);
var predictedPrice = fmv * (1 + epsilonBid + epsilonTime + epsilonComp);
// 95% confidence interval: ± 1.96 · σ_residual
double residualStdDev = fmv * 0.08; // Mock residual standard deviation
double ciLower = predictedPrice - 1.96 * residualStdDev;
double ciUpper = predictedPrice + 1.96 * residualStdDev;
PricePrediction pred = new PricePrediction();
var residualStdDev = fmv * 0.08; // Mock residual standard deviation
var ciLower = predictedPrice - 1.96 * residualStdDev;
var ciUpper = predictedPrice + 1.96 * residualStdDev;
var pred = new PricePrediction();
pred.predictedPrice = Math.round(predictedPrice * 100.0) / 100.0;
pred.confidenceIntervalLower = Math.round(ciLower * 100.0) / 100.0;
pred.confidenceIntervalUpper = Math.round(ciUpper * 100.0) / 100.0;
@@ -218,7 +218,7 @@ public class ValuationAnalyticsResource {
* Generates optimal bidding strategy based on market conditions
*/
private BiddingStrategy generateBiddingStrategy(ValuationRequest req, FairMarketValue fmv, PricePrediction pred) {
BiddingStrategy strategy = new BiddingStrategy();
var strategy = new BiddingStrategy();
// Determine competition level
if (req.bidVelocity > 5.0) {
@@ -236,7 +236,7 @@ public class ValuationAnalyticsResource {
strategy.recommendedTiming = "FINAL_10_MINUTES";
// Adjust max bid based on undervaluation
double undervaluationScore = calculateUndervaluationScore(req, fmv.value);
var undervaluationScore = calculateUndervaluationScore(req, fmv.value);
if (undervaluationScore > 0.25) {
// Aggressive strategy for undervalued lots
strategy.maxBid = fmv.value * (1 + 0.05); // Conservative overbid
@@ -270,7 +270,7 @@ public class ValuationAnalyticsResource {
* Calculates confidence score based on number and quality of comparables
*/
private double calculateFMVConfidence(int comparableCount, double totalWeight) {
double confidence = 0.5; // Base confidence
var confidence = 0.5; // Base confidence
// Boost for more comparables
confidence += Math.min(comparableCount * 0.05, 0.3);

7
src/test/java/auctiora/DatabaseServiceTest.java
View File

@@ -22,6 +22,13 @@ class DatabaseServiceTest {
@BeforeAll
void setUp() throws SQLException {
// Load SQLite JDBC driver
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
throw new SQLException("SQLite JDBC driver not found", e);
}
testDbPath = "test_database_" + System.currentTimeMillis() + ".db";
db = new DatabaseService(testDbPath);
db.ensureSchema();

65
src/test/java/auctiora/ImageProcessingServiceTest.java
View File

@@ -20,37 +20,51 @@ class ImageProcessingServiceTest {
private DatabaseService mockDb;
private ObjectDetectionService mockDetector;
private ImageProcessingService service;
private java.io.File testImage;
@BeforeEach
void setUp() {
void setUp() throws Exception {
mockDb = mock(DatabaseService.class);
mockDetector = mock(ObjectDetectionService.class);
service = new ImageProcessingService(mockDb, mockDetector);
// Create a temporary test image file
testImage = java.io.File.createTempFile("test_image_", ".jpg");
testImage.deleteOnExit();
// Write minimal JPEG header to make it a valid file
try (var out = new java.io.FileOutputStream(testImage)) {
out.write(new byte[]{(byte)0xFF, (byte)0xD8, (byte)0xFF, (byte)0xE0});
}
}
@Test
@DisplayName("Should process single image and update labels")
void testProcessImage() throws SQLException {
// Mock object detection
when(mockDetector.detectObjects("/path/to/image.jpg"))
// Normalize path (convert backslashes to forward slashes)
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
// Mock object detection with normalized path
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("car", "vehicle"));
// Process image
boolean result = service.processImage(1, "/path/to/image.jpg", 12345);
boolean result = service.processImage(1, testImage.getAbsolutePath(), 12345);
// Verify success
assertTrue(result);
verify(mockDetector).detectObjects("/path/to/image.jpg");
verify(mockDetector).detectObjects(normalizedPath);
verify(mockDb).updateImageLabels(1, List.of("car", "vehicle"));
}
@Test
@DisplayName("Should handle empty detection results")
void testProcessImageWithNoDetections() throws SQLException {
when(mockDetector.detectObjects(anyString()))
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of());
boolean result = service.processImage(2, "/path/to/image2.jpg", 12346);
boolean result = service.processImage(2, testImage.getAbsolutePath(), 12346);
assertTrue(result);
verify(mockDb).updateImageLabels(2, List.of());
@@ -59,14 +73,16 @@ class ImageProcessingServiceTest {
@Test
@DisplayName("Should handle database error gracefully")
void testProcessImageDatabaseError() throws SQLException {
when(mockDetector.detectObjects(anyString()))
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("object"));
doThrow(new SQLException("Database error"))
.when(mockDb).updateImageLabels(anyInt(), anyList());
// Should return false on error
boolean result = service.processImage(3, "/path/to/image3.jpg", 12347);
boolean result = service.processImage(3, testImage.getAbsolutePath(), 12347);
assertFalse(result);
}
@@ -84,13 +100,15 @@ class ImageProcessingServiceTest {
@Test
@DisplayName("Should process pending images batch")
void testProcessPendingImages() throws SQLException {
// Mock pending images from database
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
// Mock pending images from database - use real test image path
when(mockDb.getImagesNeedingDetection()).thenReturn(List.of(
new DatabaseService.ImageDetectionRecord(1, 100L, "/images/100/001.jpg"),
new DatabaseService.ImageDetectionRecord(2, 101L, "/images/101/001.jpg")
new DatabaseService.ImageDetectionRecord(1, 100L, testImage.getAbsolutePath()),
new DatabaseService.ImageDetectionRecord(2, 101L, testImage.getAbsolutePath())
));
when(mockDetector.detectObjects(anyString()))
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("item1"))
.thenReturn(List.of("item2"));
@@ -103,7 +121,7 @@ class ImageProcessingServiceTest {
// Verify all images were processed
verify(mockDb).getImagesNeedingDetection();
verify(mockDetector, times(2)).detectObjects(anyString());
verify(mockDetector, times(2)).detectObjects(normalizedPath);
verify(mockDb, times(2)).updateImageLabels(anyInt(), anyList());
}
@@ -121,15 +139,16 @@ class ImageProcessingServiceTest {
@Test
@DisplayName("Should continue processing after single image failure")
void testProcessPendingImagesWithFailure() throws SQLException {
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDb.getImagesNeedingDetection()).thenReturn(List.of(
new DatabaseService.ImageDetectionRecord(1, 100L, "/images/100/001.jpg"),
new DatabaseService.ImageDetectionRecord(2, 101L, "/images/101/001.jpg")
new DatabaseService.ImageDetectionRecord(1, 100L, testImage.getAbsolutePath()),
new DatabaseService.ImageDetectionRecord(2, 101L, testImage.getAbsolutePath())
));
// First image fails, second succeeds
when(mockDetector.detectObjects("/images/100/001.jpg"))
.thenThrow(new RuntimeException("Detection error"));
when(mockDetector.detectObjects("/images/101/001.jpg"))
when(mockDetector.detectObjects(normalizedPath))
.thenThrow(new RuntimeException("Detection error"))
.thenReturn(List.of("item"));
when(mockDb.getImageLabels(2))
@@ -138,7 +157,7 @@ class ImageProcessingServiceTest {
service.processPendingImages();
// Verify second image was still processed
verify(mockDetector, times(2)).detectObjects(anyString());
verify(mockDetector, times(2)).detectObjects(normalizedPath);
}
@Test
@@ -154,10 +173,12 @@ class ImageProcessingServiceTest {
@Test
@DisplayName("Should process images with multiple detected objects")
void testProcessImageMultipleDetections() throws SQLException {
when(mockDetector.detectObjects(anyString()))
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("car", "truck", "vehicle", "road"));
boolean result = service.processImage(5, "/path/to/image5.jpg", 12349);
boolean result = service.processImage(5, testImage.getAbsolutePath(), 12349);
assertTrue(result);
verify(mockDb).updateImageLabels(5, List.of("car", "truck", "vehicle", "road"));