440 lines
17 KiB
Java
440 lines
17 KiB
Java
package com.auction;
|
||
|
||
import java.sql.DriverManager;
|
||
import java.sql.SQLException;
|
||
import java.time.Instant;
|
||
import java.time.LocalDateTime;
|
||
import java.util.ArrayList;
|
||
import java.util.List;
|
||
|
||
/**
|
||
* Service for persisting auctions, lots, and images into a SQLite database.
|
||
* Data is typically populated by an external scraper process;
|
||
* this service enriches it with image processing and monitoring.
|
||
*/
|
||
public class DatabaseService {
|
||
|
||
private final String url;
|
||
|
||
DatabaseService(String dbPath) {
|
||
this.url = "jdbc:sqlite:" + dbPath;
|
||
}
|
||
|
||
/**
|
||
* Creates tables if they do not already exist.
|
||
* Schema supports data from external scraper and adds image processing results.
|
||
*/
|
||
void ensureSchema() throws SQLException {
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
// Auctions table (populated by external scraper)
|
||
stmt.execute("""
|
||
CREATE TABLE IF NOT EXISTS auctions (
|
||
auction_id INTEGER PRIMARY KEY,
|
||
title TEXT NOT NULL,
|
||
location TEXT,
|
||
city TEXT,
|
||
country TEXT,
|
||
url TEXT NOT NULL,
|
||
type TEXT,
|
||
lot_count INTEGER DEFAULT 0,
|
||
closing_time TEXT,
|
||
discovered_at INTEGER
|
||
)""");
|
||
|
||
// Lots table (populated by external scraper)
|
||
stmt.execute("""
|
||
CREATE TABLE IF NOT EXISTS lots (
|
||
lot_id INTEGER PRIMARY KEY,
|
||
sale_id INTEGER,
|
||
title TEXT,
|
||
description TEXT,
|
||
manufacturer TEXT,
|
||
type TEXT,
|
||
year INTEGER,
|
||
category TEXT,
|
||
current_bid REAL,
|
||
currency TEXT,
|
||
url TEXT,
|
||
closing_time TEXT,
|
||
closing_notified INTEGER DEFAULT 0,
|
||
FOREIGN KEY (sale_id) REFERENCES auctions(auction_id)
|
||
)""");
|
||
|
||
// Images table (populated by this process)
|
||
stmt.execute("""
|
||
CREATE TABLE IF NOT EXISTS images (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
lot_id INTEGER,
|
||
url TEXT,
|
||
file_path TEXT,
|
||
labels TEXT,
|
||
processed_at INTEGER,
|
||
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
|
||
)""");
|
||
|
||
// Indexes for performance
|
||
stmt.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)");
|
||
stmt.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)");
|
||
stmt.execute("CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id)");
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Inserts or updates an auction record (typically called by external scraper)
|
||
*/
|
||
synchronized void upsertAuction(AuctionInfo auction) throws SQLException {
|
||
var sql = """
|
||
INSERT INTO auctions (auction_id, title, location, city, country, url, type, lot_count, closing_time, discovered_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(auction_id) DO UPDATE SET
|
||
title = excluded.title,
|
||
location = excluded.location,
|
||
city = excluded.city,
|
||
country = excluded.country,
|
||
url = excluded.url,
|
||
type = excluded.type,
|
||
lot_count = excluded.lot_count,
|
||
closing_time = excluded.closing_time
|
||
""";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
|
||
ps.setInt(1, auction.auctionId());
|
||
ps.setString(2, auction.title());
|
||
ps.setString(3, auction.location());
|
||
ps.setString(4, auction.city());
|
||
ps.setString(5, auction.country());
|
||
ps.setString(6, auction.url());
|
||
ps.setString(7, auction.type());
|
||
ps.setInt(8, auction.lotCount());
|
||
ps.setString(9, auction.closingTime() != null ? auction.closingTime().toString() : null);
|
||
ps.setLong(10, Instant.now().getEpochSecond());
|
||
ps.executeUpdate();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Retrieves all auctions from the database
|
||
*/
|
||
synchronized List<AuctionInfo> getAllAuctions() throws SQLException {
|
||
List<AuctionInfo> auctions = new ArrayList<>();
|
||
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time FROM auctions";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
while (rs.next()) {
|
||
var closingStr = rs.getString("closing_time");
|
||
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
|
||
|
||
auctions.add(new AuctionInfo(
|
||
rs.getInt("auction_id"),
|
||
rs.getString("title"),
|
||
rs.getString("location"),
|
||
rs.getString("city"),
|
||
rs.getString("country"),
|
||
rs.getString("url"),
|
||
rs.getString("type"),
|
||
rs.getInt("lot_count"),
|
||
closing
|
||
));
|
||
}
|
||
}
|
||
return auctions;
|
||
}
|
||
|
||
/**
|
||
* Retrieves auctions by country code
|
||
*/
|
||
synchronized List<AuctionInfo> getAuctionsByCountry(String countryCode) throws SQLException {
|
||
List<AuctionInfo> auctions = new ArrayList<>();
|
||
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time "
|
||
+ "FROM auctions WHERE country = ?";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
|
||
ps.setString(1, countryCode);
|
||
var rs = ps.executeQuery();
|
||
while (rs.next()) {
|
||
var closingStr = rs.getString("closing_time");
|
||
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
|
||
|
||
auctions.add(new AuctionInfo(
|
||
rs.getInt("auction_id"),
|
||
rs.getString("title"),
|
||
rs.getString("location"),
|
||
rs.getString("city"),
|
||
rs.getString("country"),
|
||
rs.getString("url"),
|
||
rs.getString("type"),
|
||
rs.getInt("lot_count"),
|
||
closing
|
||
));
|
||
}
|
||
}
|
||
return auctions;
|
||
}
|
||
|
||
/**
|
||
* Inserts or updates a lot record (typically called by external scraper)
|
||
*/
|
||
synchronized void upsertLot(Lot lot) throws SQLException {
|
||
var sql = """
|
||
INSERT INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(lot_id) DO UPDATE SET
|
||
sale_id = excluded.sale_id,
|
||
title = excluded.title,
|
||
description = excluded.description,
|
||
manufacturer = excluded.manufacturer,
|
||
type = excluded.type,
|
||
year = excluded.year,
|
||
category = excluded.category,
|
||
current_bid = excluded.current_bid,
|
||
currency = excluded.currency,
|
||
url = excluded.url,
|
||
closing_time = excluded.closing_time
|
||
""";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
|
||
ps.setInt(1, lot.lotId());
|
||
ps.setInt(2, lot.saleId());
|
||
ps.setString(3, lot.title());
|
||
ps.setString(4, lot.description());
|
||
ps.setString(5, lot.manufacturer());
|
||
ps.setString(6, lot.type());
|
||
ps.setInt(7, lot.year());
|
||
ps.setString(8, lot.category());
|
||
ps.setDouble(9, lot.currentBid());
|
||
ps.setString(10, lot.currency());
|
||
ps.setString(11, lot.url());
|
||
ps.setString(12, lot.closingTime() != null ? lot.closingTime().toString() : null);
|
||
ps.setInt(13, lot.closingNotified() ? 1 : 0);
|
||
ps.executeUpdate();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Inserts a new image record with object detection labels
|
||
*/
|
||
synchronized void insertImage(int lotId, String url, String filePath, List<String> labels) throws SQLException {
|
||
var sql = "INSERT INTO images (lot_id, url, file_path, labels, processed_at) VALUES (?, ?, ?, ?, ?)";
|
||
try (var conn = DriverManager.getConnection(this.url); var ps = conn.prepareStatement(sql)) {
|
||
ps.setInt(1, lotId);
|
||
ps.setString(2, url);
|
||
ps.setString(3, filePath);
|
||
ps.setString(4, String.join(",", labels));
|
||
ps.setLong(5, Instant.now().getEpochSecond());
|
||
ps.executeUpdate();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Retrieves images for a specific lot
|
||
*/
|
||
synchronized List<ImageRecord> getImagesForLot(int lotId) throws SQLException {
|
||
List<ImageRecord> images = new ArrayList<>();
|
||
var sql = "SELECT id, lot_id, url, file_path, labels FROM images WHERE lot_id = ?";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
|
||
ps.setInt(1, lotId);
|
||
var rs = ps.executeQuery();
|
||
while (rs.next()) {
|
||
images.add(new ImageRecord(
|
||
rs.getInt("id"),
|
||
rs.getInt("lot_id"),
|
||
rs.getString("url"),
|
||
rs.getString("file_path"),
|
||
rs.getString("labels")
|
||
));
|
||
}
|
||
}
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* Retrieves all lots that are active and need monitoring
|
||
*/
|
||
synchronized List<Lot> getActiveLots() throws SQLException {
|
||
List<Lot> list = new ArrayList<>();
|
||
var sql = "SELECT lot_id, sale_id, title, description, manufacturer, type, year, category, " +
|
||
"current_bid, currency, url, closing_time, closing_notified FROM lots";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
while (rs.next()) {
|
||
var closingStr = rs.getString("closing_time");
|
||
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
|
||
|
||
list.add(new Lot(
|
||
rs.getInt("sale_id"),
|
||
rs.getInt("lot_id"),
|
||
rs.getString("title"),
|
||
rs.getString("description"),
|
||
rs.getString("manufacturer"),
|
||
rs.getString("type"),
|
||
rs.getInt("year"),
|
||
rs.getString("category"),
|
||
rs.getDouble("current_bid"),
|
||
rs.getString("currency"),
|
||
rs.getString("url"),
|
||
closing,
|
||
rs.getInt("closing_notified") != 0
|
||
));
|
||
}
|
||
}
|
||
return list;
|
||
}
|
||
|
||
/**
|
||
* Retrieves all lots from the database
|
||
*/
|
||
synchronized List<Lot> getAllLots() throws SQLException {
|
||
return getActiveLots();
|
||
}
|
||
|
||
/**
|
||
* Gets the total number of images in the database
|
||
*/
|
||
synchronized int getImageCount() throws SQLException {
|
||
var sql = "SELECT COUNT(*) as count FROM images";
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
if (rs.next()) {
|
||
return rs.getInt("count");
|
||
}
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/**
|
||
* Updates the current bid of a lot (used by monitoring service)
|
||
*/
|
||
synchronized void updateLotCurrentBid(Lot lot) throws SQLException {
|
||
try (var conn = DriverManager.getConnection(url);
|
||
var ps = conn.prepareStatement("UPDATE lots SET current_bid = ? WHERE lot_id = ?")) {
|
||
ps.setDouble(1, lot.currentBid());
|
||
ps.setInt(2, lot.lotId());
|
||
ps.executeUpdate();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Updates the closingNotified flag of a lot
|
||
*/
|
||
synchronized void updateLotNotificationFlags(Lot lot) throws SQLException {
|
||
try (var conn = DriverManager.getConnection(url);
|
||
var ps = conn.prepareStatement("UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) {
|
||
ps.setInt(1, lot.closingNotified() ? 1 : 0);
|
||
ps.setInt(2, lot.lotId());
|
||
ps.executeUpdate();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Imports auctions from scraper's schema format.
|
||
* Reads from scraper's tables and converts to monitor format using adapter.
|
||
*
|
||
* @return List of imported auctions
|
||
*/
|
||
synchronized List<AuctionInfo> importAuctionsFromScraper() throws SQLException {
|
||
List<AuctionInfo> imported = new ArrayList<>();
|
||
var sql = "SELECT auction_id, title, location, url, lots_count, first_lot_closing_time, scraped_at " +
|
||
"FROM auctions WHERE location LIKE '%NL%'";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
while (rs.next()) {
|
||
try {
|
||
var auction = ScraperDataAdapter.fromScraperAuction(rs);
|
||
upsertAuction(auction);
|
||
imported.add(auction);
|
||
} catch (Exception e) {
|
||
System.err.println("Failed to import auction: " + e.getMessage());
|
||
}
|
||
}
|
||
} catch (SQLException e) {
|
||
// Table might not exist in scraper format - that's ok
|
||
Console.println("ℹ️ Scraper auction table not found or incompatible schema");
|
||
}
|
||
|
||
return imported;
|
||
}
|
||
|
||
/**
|
||
* Imports lots from scraper's schema format.
|
||
* Reads from scraper's tables and converts to monitor format using adapter.
|
||
*
|
||
* @return List of imported lots
|
||
*/
|
||
synchronized List<Lot> importLotsFromScraper() throws SQLException {
|
||
List<Lot> imported = new ArrayList<>();
|
||
var sql = "SELECT lot_id, auction_id, title, description, category, " +
|
||
"current_bid, closing_time, url " +
|
||
"FROM lots";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
while (rs.next()) {
|
||
try {
|
||
var lot = ScraperDataAdapter.fromScraperLot(rs);
|
||
upsertLot(lot);
|
||
imported.add(lot);
|
||
} catch (Exception e) {
|
||
System.err.println("Failed to import lot: " + e.getMessage());
|
||
}
|
||
}
|
||
} catch (SQLException e) {
|
||
// Table might not exist in scraper format - that's ok
|
||
Console.println("ℹ️ Scraper lots table not found or incompatible schema");
|
||
}
|
||
|
||
return imported;
|
||
}
|
||
|
||
/**
|
||
* Imports image URLs from scraper's schema.
|
||
* The scraper populates the images table with URLs but doesn't download them.
|
||
* This method retrieves undownloaded images for processing.
|
||
*
|
||
* @return List of image URLs that need to be downloaded
|
||
*/
|
||
synchronized List<ImageImportRecord> getUnprocessedImagesFromScraper() throws SQLException {
|
||
List<ImageImportRecord> images = new ArrayList<>();
|
||
var sql = """
|
||
SELECT i.lot_id, i.url, l.auction_id
|
||
FROM images i
|
||
LEFT JOIN lots l ON i.lot_id = l.lot_id
|
||
WHERE i.downloaded = 0 OR i.local_path IS NULL
|
||
""";
|
||
|
||
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
|
||
var rs = stmt.executeQuery(sql);
|
||
while (rs.next()) {
|
||
String lotIdStr = rs.getString("lot_id");
|
||
String auctionIdStr = rs.getString("auction_id");
|
||
|
||
int lotId = ScraperDataAdapter.extractNumericId(lotIdStr);
|
||
int saleId = ScraperDataAdapter.extractNumericId(auctionIdStr);
|
||
|
||
images.add(new ImageImportRecord(
|
||
lotId,
|
||
saleId,
|
||
rs.getString("url")
|
||
));
|
||
}
|
||
} catch (SQLException e) {
|
||
Console.println("ℹ️ No unprocessed images found in scraper format");
|
||
}
|
||
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* Simple record for image data from database
|
||
*/
|
||
record ImageRecord(int id, int lotId, String url, String filePath, String labels) {}
|
||
|
||
/**
|
||
* Record for importing images from scraper format
|
||
*/
|
||
record ImageImportRecord(int lotId, int saleId, String url) {}
|
||
}
|