932 lines
38 KiB
Java
932 lines
38 KiB
Java
package auctiora;
|
||
|
||
import jakarta.enterprise.context.ApplicationScoped;
|
||
import jakarta.inject.Inject;
|
||
import lombok.extern.slf4j.Slf4j;
|
||
import org.eclipse.microprofile.config.inject.ConfigProperty;
|
||
|
||
import java.io.Console;
|
||
import java.sql.DriverManager;
|
||
import java.sql.SQLException;
|
||
import java.time.Instant;
|
||
import java.time.LocalDateTime;
|
||
import java.util.ArrayList;
|
||
import java.util.List;
|
||
|
||
/**
|
||
* Service for persisting auctions, lots, and images into a SQLite database.
|
||
* Data is typically populated by an external scraper process;
|
||
* this service enriches it with image processing and monitoring.
|
||
*/
|
||
@Slf4j
|
||
public class DatabaseService {
|
||
|
||
private final String url;
|
||
|
||
DatabaseService(String dbPath) {
|
||
// Enable WAL mode and busy timeout for concurrent access
|
||
this.url = "jdbc:sqlite:" + dbPath + "?journal_mode=WAL&busy_timeout=10000";
|
||
}
|
||
|
||
/**
|
||
* Creates tables if they do not already exist.
|
||
* Schema supports data from external scraper and adds image processing results.
|
||
*/
|
||
void ensureSchema() throws SQLException {
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
// Enable WAL mode for better concurrent access
|
||
stmt.execute("PRAGMA journal_mode=WAL");
|
||
stmt.execute("PRAGMA busy_timeout=10000");
|
||
stmt.execute("PRAGMA synchronous=NORMAL");
|
||
|
||
// Auctions table (populated by external scraper)
|
||
stmt.execute("""
|
||
CREATE TABLE IF NOT EXISTS auctions (
|
||
auction_id BIGINT PRIMARY KEY,
|
||
title TEXT NOT NULL,
|
||
location TEXT,
|
||
city TEXT,
|
||
country TEXT,
|
||
url TEXT NOT NULL,
|
||
type TEXT,
|
||
lot_count INTEGER DEFAULT 0,
|
||
closing_time TEXT,
|
||
discovered_at INTEGER
|
||
)""");
|
||
|
||
// Lots table (populated by external scraper)
|
||
stmt.execute("""
|
||
CREATE TABLE IF NOT EXISTS lots (
|
||
lot_id BIGINT PRIMARY KEY,
|
||
sale_id BIGINT,
|
||
title TEXT,
|
||
description TEXT,
|
||
manufacturer TEXT,
|
||
type TEXT,
|
||
year INTEGER,
|
||
category TEXT,
|
||
current_bid REAL,
|
||
currency TEXT,
|
||
url TEXT,
|
||
closing_time TEXT,
|
||
closing_notified INTEGER DEFAULT 0,
|
||
FOREIGN KEY (sale_id) REFERENCES auctions(auction_id)
|
||
)""");
|
||
|
||
// Images table (populated by external scraper with URLs and local_path)
|
||
// This process only adds labels via object detection
|
||
stmt.execute("""
|
||
CREATE TABLE IF NOT EXISTS images (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
lot_id INTEGER,
|
||
url TEXT,
|
||
local_path TEXT,
|
||
labels TEXT,
|
||
processed_at INTEGER,
|
||
downloaded INTEGER DEFAULT 0,
|
||
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
|
||
)""");
|
||
|
||
// Migrate existing tables to add missing columns
|
||
migrateSchema(stmt);
|
||
|
||
// Indexes for performance
|
||
stmt.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)");
|
||
stmt.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)");
|
||
stmt.execute("CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id)");
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Migrates existing database schema to add new columns.
|
||
* SQLite doesn't support DROP COLUMN, so we add columns with ALTER TABLE ADD COLUMN.
|
||
*/
|
||
private void migrateSchema(java.sql.Statement stmt) throws SQLException {
|
||
// Check auctions table for missing columns
|
||
try (var rs = stmt.executeQuery("PRAGMA table_info(auctions)")) {
|
||
var hasLocation = false;
|
||
var hasCity = false;
|
||
var hasCountry = false;
|
||
var hasType = false;
|
||
var hasLotCount = false;
|
||
var hasClosingTime = false;
|
||
var hasDiscoveredAt = false;
|
||
|
||
while (rs.next()) {
|
||
var colName = rs.getString("name");
|
||
switch (colName) {
|
||
case "location" -> hasLocation = true;
|
||
case "city" -> hasCity = true;
|
||
case "country" -> hasCountry = true;
|
||
case "type" -> hasType = true;
|
||
case "lot_count" -> hasLotCount = true;
|
||
case "closing_time" -> hasClosingTime = true;
|
||
case "discovered_at" -> hasDiscoveredAt = true;
|
||
}
|
||
}
|
||
|
||
if (!hasLocation) {
|
||
log.info("Migrating schema: Adding 'location' column to auctions table");
|
||
stmt.execute("ALTER TABLE auctions ADD COLUMN location TEXT");
|
||
}
|
||
if (!hasCity) {
|
||
log.info("Migrating schema: Adding 'city' column to auctions table");
|
||
stmt.execute("ALTER TABLE auctions ADD COLUMN city TEXT");
|
||
}
|
||
if (!hasCountry) {
|
||
log.info("Migrating schema: Adding 'country' column to auctions table");
|
||
stmt.execute("ALTER TABLE auctions ADD COLUMN country TEXT");
|
||
}
|
||
if (!hasType) {
|
||
log.info("Migrating schema: Adding 'type' column to auctions table");
|
||
stmt.execute("ALTER TABLE auctions ADD COLUMN type TEXT");
|
||
}
|
||
if (!hasLotCount) {
|
||
log.info("Migrating schema: Adding 'lot_count' column to auctions table");
|
||
stmt.execute("ALTER TABLE auctions ADD COLUMN lot_count INTEGER DEFAULT 0");
|
||
}
|
||
if (!hasClosingTime) {
|
||
log.info("Migrating schema: Adding 'closing_time' column to auctions table");
|
||
stmt.execute("ALTER TABLE auctions ADD COLUMN closing_time TEXT");
|
||
}
|
||
if (!hasDiscoveredAt) {
|
||
log.info("Migrating schema: Adding 'discovered_at' column to auctions table");
|
||
stmt.execute("ALTER TABLE auctions ADD COLUMN discovered_at INTEGER");
|
||
}
|
||
} catch (SQLException e) {
|
||
// Table might not exist yet, which is fine
|
||
log.debug("Could not check auctions table schema: " + e.getMessage());
|
||
}
|
||
|
||
// Check lots table for missing columns and migrate
|
||
try (var rs = stmt.executeQuery("PRAGMA table_info(lots)")) {
|
||
var hasSaleId = false;
|
||
var hasAuctionId = false;
|
||
var hasTitle = false;
|
||
var hasDescription = false;
|
||
var hasManufacturer = false;
|
||
var hasType = false;
|
||
var hasYear = false;
|
||
var hasCategory = false;
|
||
var hasCurrentBid = false;
|
||
var hasCurrency = false;
|
||
var hasUrl = false;
|
||
var hasClosingTime = false;
|
||
var hasClosingNotified = false;
|
||
|
||
while (rs.next()) {
|
||
var colName = rs.getString("name");
|
||
switch (colName) {
|
||
case "sale_id" -> hasSaleId = true;
|
||
case "auction_id" -> hasAuctionId = true;
|
||
case "title" -> hasTitle = true;
|
||
case "description" -> hasDescription = true;
|
||
case "manufacturer" -> hasManufacturer = true;
|
||
case "type" -> hasType = true;
|
||
case "year" -> hasYear = true;
|
||
case "category" -> hasCategory = true;
|
||
case "current_bid" -> hasCurrentBid = true;
|
||
case "currency" -> hasCurrency = true;
|
||
case "url" -> hasUrl = true;
|
||
case "closing_time" -> hasClosingTime = true;
|
||
case "closing_notified" -> hasClosingNotified = true;
|
||
}
|
||
}
|
||
|
||
// If we have auction_id but not sale_id, we need to rename the column
|
||
// SQLite doesn't support RENAME COLUMN before 3.25.0, so we add sale_id and copy data
|
||
if (hasAuctionId && !hasSaleId) {
|
||
log.info("Migrating schema: Adding 'sale_id' column to lots table and copying data from auction_id");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN sale_id INTEGER");
|
||
stmt.execute("UPDATE lots SET sale_id = auction_id");
|
||
} else if (!hasSaleId && !hasAuctionId) {
|
||
// New table, add sale_id
|
||
log.info("Migrating schema: Adding 'sale_id' column to new lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN sale_id INTEGER");
|
||
}
|
||
|
||
// Add missing columns for lot details
|
||
if (!hasTitle) {
|
||
log.info("Migrating schema: Adding 'title' column to lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN title TEXT");
|
||
}
|
||
if (!hasDescription) {
|
||
log.info("Migrating schema: Adding 'description' column to lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN description TEXT");
|
||
}
|
||
if (!hasManufacturer) {
|
||
log.info("Migrating schema: Adding 'manufacturer' column to lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN manufacturer TEXT");
|
||
}
|
||
if (!hasType) {
|
||
log.info("Migrating schema: Adding 'type' column to lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN type TEXT");
|
||
}
|
||
if (!hasYear) {
|
||
log.info("Migrating schema: Adding 'year' column to lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN year INTEGER");
|
||
}
|
||
if (!hasCategory) {
|
||
log.info("Migrating schema: Adding 'category' column to lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN category TEXT");
|
||
}
|
||
if (!hasCurrentBid) {
|
||
log.info("Migrating schema: Adding 'current_bid' column to lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN current_bid REAL");
|
||
}
|
||
if (!hasCurrency) {
|
||
log.info("Migrating schema: Adding 'currency' column to lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN currency TEXT DEFAULT 'EUR'");
|
||
}
|
||
if (!hasUrl) {
|
||
log.info("Migrating schema: Adding 'url' column to lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN url TEXT");
|
||
}
|
||
if (!hasClosingTime) {
|
||
log.info("Migrating schema: Adding 'closing_time' column to lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN closing_time TEXT");
|
||
}
|
||
if (!hasClosingNotified) {
|
||
log.info("Migrating schema: Adding 'closing_notified' column to lots table");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN closing_notified INTEGER DEFAULT 0");
|
||
}
|
||
|
||
// Migrate intelligence fields from GraphQL API
|
||
migrateIntelligenceFields(stmt);
|
||
} catch (SQLException e) {
|
||
// Table might not exist yet, which is fine
|
||
log.debug("Could not check lots table schema: " + e.getMessage());
|
||
}
|
||
|
||
// Check images table for missing columns and migrate
|
||
try (var rs = stmt.executeQuery("PRAGMA table_info(images)")) {
|
||
var hasLabels = false;
|
||
var hasLocalPath = false;
|
||
var hasProcessedAt = false;
|
||
var hasDownloaded = false;
|
||
|
||
while (rs.next()) {
|
||
var colName = rs.getString("name");
|
||
switch (colName) {
|
||
case "labels" -> hasLabels = true;
|
||
case "local_path" -> hasLocalPath = true;
|
||
case "processed_at" -> hasProcessedAt = true;
|
||
case "downloaded" -> hasDownloaded = true;
|
||
}
|
||
}
|
||
|
||
if (!hasLabels) {
|
||
log.info("Migrating schema: Adding 'labels' column to images table");
|
||
stmt.execute("ALTER TABLE images ADD COLUMN labels TEXT");
|
||
}
|
||
if (!hasLocalPath) {
|
||
log.info("Migrating schema: Adding 'local_path' column to images table");
|
||
stmt.execute("ALTER TABLE images ADD COLUMN local_path TEXT");
|
||
}
|
||
if (!hasProcessedAt) {
|
||
log.info("Migrating schema: Adding 'processed_at' column to images table");
|
||
stmt.execute("ALTER TABLE images ADD COLUMN processed_at INTEGER");
|
||
}
|
||
if (!hasDownloaded) {
|
||
log.info("Migrating schema: Adding 'downloaded' column to images table");
|
||
stmt.execute("ALTER TABLE images ADD COLUMN downloaded INTEGER DEFAULT 0");
|
||
}
|
||
} catch (SQLException e) {
|
||
// Table might not exist yet, which is fine
|
||
log.debug("Could not check images table schema: " + e.getMessage());
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Migrates intelligence fields to lots table (GraphQL enrichment data)
|
||
*/
|
||
private void migrateIntelligenceFields(java.sql.Statement stmt) throws SQLException {
|
||
try (var rs = stmt.executeQuery("PRAGMA table_info(lots)")) {
|
||
var columns = new java.util.HashSet<String>();
|
||
while (rs.next()) {
|
||
columns.add(rs.getString("name"));
|
||
}
|
||
|
||
// HIGH PRIORITY FIELDS
|
||
if (!columns.contains("followers_count")) {
|
||
log.info("Adding followers_count column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN followers_count INTEGER");
|
||
}
|
||
if (!columns.contains("estimated_min")) {
|
||
log.info("Adding estimated_min column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN estimated_min REAL");
|
||
}
|
||
if (!columns.contains("estimated_max")) {
|
||
log.info("Adding estimated_max column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN estimated_max REAL");
|
||
}
|
||
if (!columns.contains("next_bid_step_cents")) {
|
||
log.info("Adding next_bid_step_cents column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN next_bid_step_cents INTEGER");
|
||
}
|
||
if (!columns.contains("condition")) {
|
||
log.info("Adding condition column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN condition TEXT");
|
||
}
|
||
if (!columns.contains("category_path")) {
|
||
log.info("Adding category_path column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN category_path TEXT");
|
||
}
|
||
if (!columns.contains("city_location")) {
|
||
log.info("Adding city_location column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN city_location TEXT");
|
||
}
|
||
if (!columns.contains("country_code")) {
|
||
log.info("Adding country_code column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN country_code TEXT");
|
||
}
|
||
|
||
// MEDIUM PRIORITY FIELDS
|
||
if (!columns.contains("bidding_status")) {
|
||
log.info("Adding bidding_status column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN bidding_status TEXT");
|
||
}
|
||
if (!columns.contains("appearance")) {
|
||
log.info("Adding appearance column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN appearance TEXT");
|
||
}
|
||
if (!columns.contains("packaging")) {
|
||
log.info("Adding packaging column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN packaging TEXT");
|
||
}
|
||
if (!columns.contains("quantity")) {
|
||
log.info("Adding quantity column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN quantity INTEGER");
|
||
}
|
||
if (!columns.contains("vat")) {
|
||
log.info("Adding vat column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN vat REAL");
|
||
}
|
||
if (!columns.contains("buyer_premium_percentage")) {
|
||
log.info("Adding buyer_premium_percentage column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN buyer_premium_percentage REAL");
|
||
}
|
||
if (!columns.contains("remarks")) {
|
||
log.info("Adding remarks column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN remarks TEXT");
|
||
}
|
||
|
||
// BID INTELLIGENCE FIELDS
|
||
if (!columns.contains("starting_bid")) {
|
||
log.info("Adding starting_bid column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN starting_bid REAL");
|
||
}
|
||
if (!columns.contains("reserve_price")) {
|
||
log.info("Adding reserve_price column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN reserve_price REAL");
|
||
}
|
||
if (!columns.contains("reserve_met")) {
|
||
log.info("Adding reserve_met column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN reserve_met INTEGER");
|
||
}
|
||
if (!columns.contains("bid_increment")) {
|
||
log.info("Adding bid_increment column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN bid_increment REAL");
|
||
}
|
||
if (!columns.contains("view_count")) {
|
||
log.info("Adding view_count column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN view_count INTEGER");
|
||
}
|
||
if (!columns.contains("first_bid_time")) {
|
||
log.info("Adding first_bid_time column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN first_bid_time TEXT");
|
||
}
|
||
if (!columns.contains("last_bid_time")) {
|
||
log.info("Adding last_bid_time column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN last_bid_time TEXT");
|
||
}
|
||
if (!columns.contains("bid_velocity")) {
|
||
log.info("Adding bid_velocity column");
|
||
stmt.execute("ALTER TABLE lots ADD COLUMN bid_velocity REAL");
|
||
}
|
||
} catch (SQLException e) {
|
||
log.warn("Could not migrate intelligence fields: {}", e.getMessage());
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Inserts or updates an auction record (typically called by external scraper)
|
||
* Handles both auction_id conflicts and url uniqueness constraints
|
||
*/
|
||
synchronized void upsertAuction(AuctionInfo auction) throws SQLException {
|
||
// First try to INSERT with ON CONFLICT on auction_id
|
||
var insertSql = """
|
||
INSERT INTO auctions (auction_id, title, location, city, country, url, type, lot_count, closing_time, discovered_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(auction_id) DO UPDATE SET
|
||
title = excluded.title,
|
||
location = excluded.location,
|
||
city = excluded.city,
|
||
country = excluded.country,
|
||
url = excluded.url,
|
||
type = excluded.type,
|
||
lot_count = excluded.lot_count,
|
||
closing_time = excluded.closing_time
|
||
""";
|
||
|
||
try (var conn = DriverManager.getConnection(url)) {
|
||
try (var ps = conn.prepareStatement(insertSql)) {
|
||
ps.setLong(1, auction.auctionId());
|
||
ps.setString(2, auction.title());
|
||
ps.setString(3, auction.location());
|
||
ps.setString(4, auction.city());
|
||
ps.setString(5, auction.country());
|
||
ps.setString(6, auction.url());
|
||
ps.setString(7, auction.typePrefix());
|
||
ps.setInt(8, auction.lotCount());
|
||
ps.setString(9, auction.firstLotClosingTime() != null ? auction.firstLotClosingTime().toString() : null);
|
||
ps.setLong(10, Instant.now().getEpochSecond());
|
||
ps.executeUpdate();
|
||
} catch (SQLException e) {
|
||
// If it fails due to UNIQUE constraint on url, try updating by url instead
|
||
if (e.getMessage().contains("UNIQUE constraint failed: auctions.url")) {
|
||
var updateByUrlSql = """
|
||
UPDATE auctions SET
|
||
auction_id = ?,
|
||
title = ?,
|
||
location = ?,
|
||
city = ?,
|
||
country = ?,
|
||
type = ?,
|
||
lot_count = ?,
|
||
closing_time = ?
|
||
WHERE url = ?
|
||
""";
|
||
try (var ps = conn.prepareStatement(updateByUrlSql)) {
|
||
ps.setLong(1, auction.auctionId());
|
||
ps.setString(2, auction.title());
|
||
ps.setString(3, auction.location());
|
||
ps.setString(4, auction.city());
|
||
ps.setString(5, auction.country());
|
||
ps.setString(6, auction.typePrefix());
|
||
ps.setInt(7, auction.lotCount());
|
||
ps.setString(8, auction.firstLotClosingTime() != null ? auction.firstLotClosingTime().toString() : null);
|
||
ps.setString(9, auction.url());
|
||
|
||
int updated = ps.executeUpdate();
|
||
if (updated == 0) {
|
||
log.warn("Could not insert or update auction with url={}, auction_id={}", auction.url(), auction.auctionId());
|
||
}
|
||
}
|
||
} else {
|
||
throw e;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Retrieves all auctions from the database
|
||
*/
|
||
synchronized List<AuctionInfo> getAllAuctions() throws SQLException {
|
||
List<AuctionInfo> auctions = new ArrayList<>();
|
||
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time FROM auctions";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
while (rs.next()) {
|
||
var closingStr = rs.getString("closing_time");
|
||
LocalDateTime closing = null;
|
||
if (closingStr != null && !closingStr.isBlank()) {
|
||
try {
|
||
closing = LocalDateTime.parse(closingStr);
|
||
} catch (Exception e) {
|
||
log.debug("Invalid closing_time format for auction {}: {}", rs.getLong("auction_id"), closingStr);
|
||
}
|
||
}
|
||
|
||
auctions.add(new AuctionInfo(
|
||
rs.getLong("auction_id"),
|
||
rs.getString("title"),
|
||
rs.getString("location"),
|
||
rs.getString("city"),
|
||
rs.getString("country"),
|
||
rs.getString("url"),
|
||
rs.getString("type"),
|
||
rs.getInt("lot_count"),
|
||
closing
|
||
));
|
||
}
|
||
}
|
||
return auctions;
|
||
}
|
||
|
||
/**
|
||
* Retrieves auctions by country code
|
||
*/
|
||
synchronized List<AuctionInfo> getAuctionsByCountry(String countryCode) throws SQLException {
|
||
List<AuctionInfo> auctions = new ArrayList<>();
|
||
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time "
|
||
+ "FROM auctions WHERE country = ?";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
|
||
ps.setString(1, countryCode);
|
||
var rs = ps.executeQuery();
|
||
while (rs.next()) {
|
||
var closingStr = rs.getString("closing_time");
|
||
LocalDateTime closing = null;
|
||
if (closingStr != null && !closingStr.isBlank()) {
|
||
try {
|
||
closing = LocalDateTime.parse(closingStr);
|
||
} catch (Exception e) {
|
||
log.debug("Invalid closing_time format for auction {}: {}", rs.getLong("auction_id"), closingStr);
|
||
}
|
||
}
|
||
|
||
auctions.add(new AuctionInfo(
|
||
rs.getLong("auction_id"),
|
||
rs.getString("title"),
|
||
rs.getString("location"),
|
||
rs.getString("city"),
|
||
rs.getString("country"),
|
||
rs.getString("url"),
|
||
rs.getString("type"),
|
||
rs.getInt("lot_count"),
|
||
closing
|
||
));
|
||
}
|
||
}
|
||
return auctions;
|
||
}
|
||
|
||
/**
|
||
* Inserts or updates a lot record (typically called by external scraper)
|
||
*/
|
||
synchronized void upsertLot(Lot lot) throws SQLException {
|
||
// First try to update existing lot by lot_id
|
||
var updateSql = """
|
||
UPDATE lots SET
|
||
sale_id = ?,
|
||
title = ?,
|
||
description = ?,
|
||
manufacturer = ?,
|
||
type = ?,
|
||
year = ?,
|
||
category = ?,
|
||
current_bid = ?,
|
||
currency = ?,
|
||
url = ?,
|
||
closing_time = ?
|
||
WHERE lot_id = ?
|
||
""";
|
||
|
||
var insertSql = """
|
||
INSERT OR IGNORE INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
""";
|
||
|
||
try (var conn = DriverManager.getConnection(url)) {
|
||
// Try UPDATE first
|
||
try (var ps = conn.prepareStatement(updateSql)) {
|
||
ps.setLong(1, lot.saleId());
|
||
ps.setString(2, lot.title());
|
||
ps.setString(3, lot.description());
|
||
ps.setString(4, lot.manufacturer());
|
||
ps.setString(5, lot.type());
|
||
ps.setInt(6, lot.year());
|
||
ps.setString(7, lot.category());
|
||
ps.setDouble(8, lot.currentBid());
|
||
ps.setString(9, lot.currency());
|
||
ps.setString(10, lot.url());
|
||
ps.setString(11, lot.closingTime() != null ? lot.closingTime().toString() : null);
|
||
ps.setLong(12, lot.lotId());
|
||
|
||
int updated = ps.executeUpdate();
|
||
if (updated > 0) {
|
||
return; // Successfully updated existing record
|
||
}
|
||
}
|
||
|
||
// If no rows updated, try INSERT (ignore if conflicts with UNIQUE constraints)
|
||
try (var ps = conn.prepareStatement(insertSql)) {
|
||
ps.setLong(1, lot.lotId());
|
||
ps.setLong(2, lot.saleId());
|
||
ps.setString(3, lot.title());
|
||
ps.setString(4, lot.description());
|
||
ps.setString(5, lot.manufacturer());
|
||
ps.setString(6, lot.type());
|
||
ps.setInt(7, lot.year());
|
||
ps.setString(8, lot.category());
|
||
ps.setDouble(9, lot.currentBid());
|
||
ps.setString(10, lot.currency());
|
||
ps.setString(11, lot.url());
|
||
ps.setString(12, lot.closingTime() != null ? lot.closingTime().toString() : null);
|
||
ps.setInt(13, lot.closingNotified() ? 1 : 0);
|
||
ps.executeUpdate();
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Inserts a complete image record (for testing/legacy compatibility).
|
||
* In production, scraper inserts with local_path, monitor updates labels via updateImageLabels.
|
||
*/
|
||
synchronized void insertImage(long lotId, String url, String filePath, List<String> labels) throws SQLException {
|
||
var sql = "INSERT INTO images (lot_id, url, local_path, labels, processed_at, downloaded) VALUES (?, ?, ?, ?, ?, 1)";
|
||
try (var conn = DriverManager.getConnection(this.url); var ps = conn.prepareStatement(sql)) {
|
||
ps.setLong(1, lotId);
|
||
ps.setString(2, url);
|
||
ps.setString(3, filePath);
|
||
ps.setString(4, String.join(",", labels));
|
||
ps.setLong(5, Instant.now().getEpochSecond());
|
||
ps.executeUpdate();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Updates the labels field for an image after object detection
|
||
*/
|
||
synchronized void updateImageLabels(int imageId, List<String> labels) throws SQLException {
|
||
var sql = "UPDATE images SET labels = ?, processed_at = ? WHERE id = ?";
|
||
try (var conn = DriverManager.getConnection(this.url); var ps = conn.prepareStatement(sql)) {
|
||
ps.setString(1, String.join(",", labels));
|
||
ps.setLong(2, Instant.now().getEpochSecond());
|
||
ps.setInt(3, imageId);
|
||
ps.executeUpdate();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Gets the labels for a specific image
|
||
*/
|
||
synchronized List<String> getImageLabels(int imageId) throws SQLException {
|
||
var sql = "SELECT labels FROM images WHERE id = ?";
|
||
try (var conn = DriverManager.getConnection(this.url); var ps = conn.prepareStatement(sql)) {
|
||
ps.setInt(1, imageId);
|
||
var rs = ps.executeQuery();
|
||
if (rs.next()) {
|
||
var labelsStr = rs.getString("labels");
|
||
if (labelsStr != null && !labelsStr.isEmpty()) {
|
||
return List.of(labelsStr.split(","));
|
||
}
|
||
}
|
||
}
|
||
return List.of();
|
||
}
|
||
|
||
/**
|
||
* Retrieves images for a specific lot
|
||
*/
|
||
synchronized List<ImageRecord> getImagesForLot(long lotId) throws SQLException {
|
||
List<ImageRecord> images = new ArrayList<>();
|
||
var sql = "SELECT id, lot_id, url, local_path, labels FROM images WHERE lot_id = ?";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
|
||
ps.setLong(1, lotId);
|
||
var rs = ps.executeQuery();
|
||
while (rs.next()) {
|
||
images.add(new ImageRecord(
|
||
rs.getInt("id"),
|
||
rs.getLong("lot_id"),
|
||
rs.getString("url"),
|
||
rs.getString("local_path"),
|
||
rs.getString("labels")
|
||
));
|
||
}
|
||
}
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* Retrieves all lots that are active and need monitoring
|
||
*/
|
||
synchronized List<Lot> getActiveLots() throws SQLException {
|
||
List<Lot> list = new ArrayList<>();
|
||
var sql = "SELECT lot_id, sale_id as auction_id, title, description, manufacturer, type, year, category, " +
|
||
"current_bid, currency, url, closing_time, closing_notified FROM lots";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
while (rs.next()) {
|
||
try {
|
||
// Use ScraperDataAdapter to handle TEXT parsing from legacy database
|
||
var lot = ScraperDataAdapter.fromScraperLot(rs);
|
||
list.add(lot);
|
||
} catch (Exception e) {
|
||
log.warn("Failed to parse lot {}: {}", rs.getString("lot_id"), e.getMessage());
|
||
}
|
||
}
|
||
}
|
||
return list;
|
||
}
|
||
|
||
/**
|
||
* Retrieves all lots from the database
|
||
*/
|
||
synchronized List<Lot> getAllLots() throws SQLException {
|
||
return getActiveLots();
|
||
}
|
||
|
||
/**
|
||
* Gets the total number of images in the database
|
||
*/
|
||
synchronized int getImageCount() throws SQLException {
|
||
var sql = "SELECT COUNT(*) as count FROM images";
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
if (rs.next()) {
|
||
return rs.getInt("count");
|
||
}
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/**
|
||
* Updates the current bid of a lot (used by monitoring service)
|
||
*/
|
||
synchronized void updateLotCurrentBid(Lot lot) throws SQLException {
|
||
try (var conn = DriverManager.getConnection(url);
|
||
var ps = conn.prepareStatement("UPDATE lots SET current_bid = ? WHERE lot_id = ?")) {
|
||
ps.setDouble(1, lot.currentBid());
|
||
ps.setLong(2, lot.lotId());
|
||
ps.executeUpdate();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Updates the closingNotified flag of a lot
|
||
*/
|
||
synchronized void updateLotNotificationFlags(Lot lot) throws SQLException {
|
||
try (var conn = DriverManager.getConnection(url);
|
||
var ps = conn.prepareStatement("UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) {
|
||
ps.setInt(1, lot.closingNotified() ? 1 : 0);
|
||
ps.setLong(2, lot.lotId());
|
||
ps.executeUpdate();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Retrieves bid history for a specific lot
|
||
*/
|
||
synchronized List<BidHistory> getBidHistory(String lotId) throws SQLException {
|
||
List<BidHistory> history = new ArrayList<>();
|
||
var sql = "SELECT id, lot_id, bid_amount, bid_time, is_autobid, bidder_id, bidder_number " +
|
||
"FROM bid_history WHERE lot_id = ? ORDER BY bid_time DESC LIMIT 100";
|
||
|
||
try (var conn = DriverManager.getConnection(url);
|
||
var ps = conn.prepareStatement(sql)) {
|
||
ps.setString(1, lotId);
|
||
var rs = ps.executeQuery();
|
||
|
||
while (rs.next()) {
|
||
LocalDateTime bidTime = null;
|
||
var bidTimeStr = rs.getString("bid_time");
|
||
if (bidTimeStr != null && !bidTimeStr.isBlank()) {
|
||
try {
|
||
bidTime = LocalDateTime.parse(bidTimeStr);
|
||
} catch (Exception e) {
|
||
log.debug("Invalid bid_time format: {}", bidTimeStr);
|
||
}
|
||
}
|
||
|
||
history.add(new BidHistory(
|
||
rs.getInt("id"),
|
||
rs.getString("lot_id"),
|
||
rs.getDouble("bid_amount"),
|
||
bidTime,
|
||
rs.getInt("is_autobid") != 0,
|
||
rs.getString("bidder_id"),
|
||
rs.getInt("bidder_number")
|
||
));
|
||
}
|
||
}
|
||
return history;
|
||
}
|
||
|
||
/**
|
||
* Imports auctions from scraper's schema format.
|
||
* Since the scraper doesn't populate a separate auctions table,
|
||
* we derive auction metadata by aggregating lots data.
|
||
*
|
||
* @return List of imported auctions
|
||
*/
|
||
synchronized List<AuctionInfo> importAuctionsFromScraper() throws SQLException {
|
||
List<AuctionInfo> imported = new ArrayList<>();
|
||
|
||
// Derive auctions from lots table (scraper doesn't populate auctions table)
|
||
var sql = """
|
||
SELECT
|
||
l.auction_id,
|
||
MIN(l.title) as title,
|
||
MIN(l.location) as location,
|
||
MIN(l.url) as url,
|
||
COUNT(*) as lots_count,
|
||
MIN(l.closing_time) as first_lot_closing_time,
|
||
MIN(l.scraped_at) as scraped_at
|
||
FROM lots l
|
||
WHERE l.auction_id IS NOT NULL
|
||
GROUP BY l.auction_id
|
||
""";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
while (rs.next()) {
|
||
try {
|
||
var auction = ScraperDataAdapter.fromScraperAuction(rs);
|
||
// Skip auctions with invalid IDs (0 indicates parsing failed)
|
||
if (auction.auctionId() == 0L) {
|
||
log.debug("Skipping auction with invalid ID: auction_id={}", auction.auctionId());
|
||
continue;
|
||
}
|
||
upsertAuction(auction);
|
||
imported.add(auction);
|
||
} catch (Exception e) {
|
||
System.err.println("Failed to import auction: " + e.getMessage());
|
||
}
|
||
}
|
||
} catch (SQLException e) {
|
||
// Table might not exist in scraper format - that's ok
|
||
log.info("ℹ️ Scraper lots table not found or incompatible schema: {}", e.getMessage());
|
||
}
|
||
|
||
return imported;
|
||
}
|
||
|
||
/**
|
||
* Imports lots from scraper's schema format.
|
||
* Reads from scraper's tables and converts to monitor format using adapter.
|
||
*
|
||
* @return List of imported lots
|
||
*/
|
||
synchronized List<Lot> importLotsFromScraper() throws SQLException {
|
||
List<Lot> imported = new ArrayList<>();
|
||
var sql = "SELECT lot_id, auction_id, title, description, category, " +
|
||
"current_bid, closing_time, url " +
|
||
"FROM lots";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
while (rs.next()) {
|
||
try {
|
||
var lot = ScraperDataAdapter.fromScraperLot(rs);
|
||
// Skip lots with invalid IDs (0 indicates parsing failed)
|
||
if (lot.lotId() == 0L || lot.saleId() == 0L) {
|
||
log.debug("Skipping lot with invalid ID: lot_id={}, sale_id={}", lot.lotId(), lot.saleId());
|
||
continue;
|
||
}
|
||
upsertLot(lot);
|
||
imported.add(lot);
|
||
} catch (Exception e) {
|
||
System.err.println("Failed to import lot: " + e.getMessage());
|
||
}
|
||
}
|
||
} catch (SQLException e) {
|
||
// Table might not exist in scraper format - that's ok
|
||
log.info("ℹ️ Scraper lots table not found or incompatible schema");
|
||
}
|
||
|
||
return imported;
|
||
}
|
||
|
||
/**
|
||
* Gets images that have been downloaded by the scraper but need object detection.
|
||
* Only returns images that have local_path set but no labels yet.
|
||
*
|
||
* @return List of images needing object detection
|
||
*/
|
||
synchronized List<ImageDetectionRecord> getImagesNeedingDetection() throws SQLException {
|
||
List<ImageDetectionRecord> images = new ArrayList<>();
|
||
var sql = """
|
||
SELECT i.id, i.lot_id, i.local_path
|
||
FROM images i
|
||
WHERE i.local_path IS NOT NULL
|
||
AND i.local_path != ''
|
||
AND (i.labels IS NULL OR i.labels = '')
|
||
""";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
while (rs.next()) {
|
||
// Extract numeric lot ID from TEXT field (e.g., "A1-34732-49" -> 3473249)
|
||
String lotIdStr = rs.getString("lot_id");
|
||
long lotId = ScraperDataAdapter.extractNumericId(lotIdStr);
|
||
|
||
images.add(new ImageDetectionRecord(
|
||
rs.getInt("id"),
|
||
lotId,
|
||
rs.getString("local_path")
|
||
));
|
||
}
|
||
} catch (SQLException e) {
|
||
log.info("ℹ️ No images needing detection found");
|
||
}
|
||
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* Simple record for image data from database
|
||
*/
|
||
record ImageRecord(int id, long lotId, String url, String filePath, String labels) { }
|
||
|
||
/**
|
||
* Record for images that need object detection processing
|
||
*/
|
||
record ImageDetectionRecord(int id, long lotId, String filePath) { }
|
||
}
|