Tour/auctiora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

start

Browse Source
This commit is contained in:
Tour
2025-12-03 15:32:34 +01:00
parent 815d6a9a4a
commit aef7a3aa30
10 changed files with 533 additions and 1350 deletions
Download Patch File Download Diff File Expand all files Collapse all files

606
src/main/java/com/auction/DatabaseService.java
View File

@@ -1,303 +1,335 @@
package com.auction;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
/**
* Service for persisting auctions, lots, images, and object labels into
* a SQLite database. Uses the Xerial JDBC driver which connects to
* SQLite via a URL of the form "jdbc:sqlite:path_to_file"【329850066306528†L40-L63】.
* Service for persisting auctions, lots, and images into a SQLite database.
* Data is typically populated by an external scraper process;
* this service enriches it with image processing and monitoring.
*/
public class DatabaseService {
private final String url;
DatabaseService(String dbPath) {
this.url = "jdbc:sqlite:" + dbPath;
}
/**
* Creates tables if they do not already exist. The schema includes
* tables for auctions, lots, images, and object labels. This method is
* idempotent; it can be called multiple times.
*/
void ensureSchema() throws SQLException {
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
// Auctions table (veilingen)
stmt.execute("CREATE TABLE IF NOT EXISTS auctions ("
+ "auction_id INTEGER PRIMARY KEY,"
+ "title TEXT NOT NULL,"
+ "location TEXT,"
+ "city TEXT,"
+ "country TEXT,"
+ "url TEXT NOT NULL,"
+ "type TEXT,"
+ "lot_count INTEGER DEFAULT 0,"
+ "closing_time TEXT,"
+ "discovered_at INTEGER" // Unix timestamp
+ ")");
// Sales table (legacy - keep for compatibility)
stmt.execute("CREATE TABLE IF NOT EXISTS sales ("
+ "sale_id INTEGER PRIMARY KEY,"
+ "title TEXT,"
+ "location TEXT,"
+ "closing_time TEXT"
+ ")");
// Lots table
stmt.execute("CREATE TABLE IF NOT EXISTS lots ("
+ "lot_id INTEGER PRIMARY KEY,"
+ "sale_id INTEGER,"
+ "title TEXT,"
+ "description TEXT,"
+ "manufacturer TEXT,"
+ "type TEXT,"
+ "year INTEGER,"
+ "category TEXT,"
+ "current_bid REAL,"
+ "currency TEXT,"
+ "url TEXT,"
+ "closing_time TEXT,"
+ "closing_notified INTEGER DEFAULT 0,"
+ "FOREIGN KEY (sale_id) REFERENCES auctions(auction_id)"
+ ")");
// Images table
stmt.execute("CREATE TABLE IF NOT EXISTS images ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "lot_id INTEGER,"
+ "url TEXT,"
+ "file_path TEXT,"
+ "labels TEXT,"
+ "FOREIGN KEY (lot_id) REFERENCES lots(lot_id)"
+ ")");
// Create indexes for better query performance
stmt.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)");
}
}
/**
* Inserts or updates an auction record
*/
synchronized void upsertAuction(AuctionInfo auction) throws SQLException {
var sql = "INSERT INTO auctions (auction_id, title, location, city, country, url, type, lot_count, closing_time, discovered_at)"
+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+ " ON CONFLICT(auction_id) DO UPDATE SET "
+ "title = excluded.title, location = excluded.location, city = excluded.city, "
+ "country = excluded.country, url = excluded.url, type = excluded.type, "
+ "lot_count = excluded.lot_count, closing_time = excluded.closing_time";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, auction.auctionId);
ps.setString(2, auction.title);
ps.setString(3, auction.location);
ps.setString(4, auction.city);
ps.setString(5, auction.country);
ps.setString(6, auction.url);
ps.setString(7, auction.type);
ps.setInt(8, auction.lotCount);
ps.setString(9, auction.closingTime != null ? auction.closingTime.toString() : null);
ps.setLong(10, Instant.now().getEpochSecond());
ps.executeUpdate();
}
}
/**
* Retrieves all auctions from the database
*/
synchronized List<AuctionInfo> getAllAuctions() throws SQLException {
List<AuctionInfo> auctions = new ArrayList<>();
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time FROM auctions";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
var auction = new AuctionInfo();
auction.auctionId = rs.getInt("auction_id");
auction.title = rs.getString("title");
auction.location = rs.getString("location");
auction.city = rs.getString("city");
auction.country = rs.getString("country");
auction.url = rs.getString("url");
auction.type = rs.getString("type");
auction.lotCount = rs.getInt("lot_count");
var closing = rs.getString("closing_time");
if (closing != null) {
auction.closingTime = LocalDateTime.parse(closing);
private final String url;
DatabaseService(String dbPath) {
this.url = "jdbc:sqlite:" + dbPath;
}
/**
* Creates tables if they do not already exist.
* Schema supports data from external scraper and adds image processing results.
*/
void ensureSchema() throws SQLException {
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
// Auctions table (populated by external scraper)
stmt.execute("""
CREATE TABLE IF NOT EXISTS auctions (
auction_id INTEGER PRIMARY KEY,
title TEXT NOT NULL,
location TEXT,
city TEXT,
country TEXT,
url TEXT NOT NULL,
type TEXT,
lot_count INTEGER DEFAULT 0,
closing_time TEXT,
discovered_at INTEGER
)""");
// Lots table (populated by external scraper)
stmt.execute("""
CREATE TABLE IF NOT EXISTS lots (
lot_id INTEGER PRIMARY KEY,
sale_id INTEGER,
title TEXT,
description TEXT,
manufacturer TEXT,
type TEXT,
year INTEGER,
category TEXT,
current_bid REAL,
currency TEXT,
url TEXT,
closing_time TEXT,
closing_notified INTEGER DEFAULT 0,
FOREIGN KEY (sale_id) REFERENCES auctions(auction_id)
)""");
// Images table (populated by this process)
stmt.execute("""
CREATE TABLE IF NOT EXISTS images (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id INTEGER,
url TEXT,
file_path TEXT,
labels TEXT,
processed_at INTEGER,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)""");
// Indexes for performance
stmt.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id)");
}
}
/**
* Inserts or updates an auction record (typically called by external scraper)
*/
synchronized void upsertAuction(AuctionInfo auction) throws SQLException {
var sql = """
INSERT INTO auctions (auction_id, title, location, city, country, url, type, lot_count, closing_time, discovered_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(auction_id) DO UPDATE SET
title = excluded.title,
location = excluded.location,
city = excluded.city,
country = excluded.country,
url = excluded.url,
type = excluded.type,
lot_count = excluded.lot_count,
closing_time = excluded.closing_time
""";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, auction.auctionId());
ps.setString(2, auction.title());
ps.setString(3, auction.location());
ps.setString(4, auction.city());
ps.setString(5, auction.country());
ps.setString(6, auction.url());
ps.setString(7, auction.type());
ps.setInt(8, auction.lotCount());
ps.setString(9, auction.closingTime() != null ? auction.closingTime().toString() : null);
ps.setLong(10, Instant.now().getEpochSecond());
ps.executeUpdate();
}
}
/**
* Retrieves all auctions from the database
*/
synchronized List<AuctionInfo> getAllAuctions() throws SQLException {
List<AuctionInfo> auctions = new ArrayList<>();
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time FROM auctions";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
var closingStr = rs.getString("closing_time");
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
auctions.add(new AuctionInfo(
rs.getInt("auction_id"),
rs.getString("title"),
rs.getString("location"),
rs.getString("city"),
rs.getString("country"),
rs.getString("url"),
rs.getString("type"),
rs.getInt("lot_count"),
closing
));
}
auctions.add(auction);
}
}
return auctions;
}
/**
* Retrieves auctions by country code
*/
synchronized List<AuctionInfo> getAuctionsByCountry(String countryCode) throws SQLException {
List<AuctionInfo> auctions = new ArrayList<>();
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time "
}
return auctions;
}
/**
* Retrieves auctions by country code
*/
synchronized List<AuctionInfo> getAuctionsByCountry(String countryCode) throws SQLException {
List<AuctionInfo> auctions = new ArrayList<>();
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time "
+ "FROM auctions WHERE country = ?";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setString(1, countryCode);
var rs = ps.executeQuery();
while (rs.next()) {
var auction = new AuctionInfo();
auction.auctionId = rs.getInt("auction_id");
auction.title = rs.getString("title");
auction.location = rs.getString("location");
auction.city = rs.getString("city");
auction.country = rs.getString("country");
auction.url = rs.getString("url");
auction.type = rs.getString("type");
auction.lotCount = rs.getInt("lot_count");
var closing = rs.getString("closing_time");
if (closing != null) {
auction.closingTime = LocalDateTime.parse(closing);
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setString(1, countryCode);
var rs = ps.executeQuery();
while (rs.next()) {
var closingStr = rs.getString("closing_time");
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
auctions.add(new AuctionInfo(
rs.getInt("auction_id"),
rs.getString("title"),
rs.getString("location"),
rs.getString("city"),
rs.getString("country"),
rs.getString("url"),
rs.getString("type"),
rs.getInt("lot_count"),
closing
));
}
auctions.add(auction);
}
}
return auctions;
}
/**
* Inserts or updates a lot record. Uses INSERT OR REPLACE to
* implement upsert semantics so that existing rows are replaced.
*/
synchronized void upsertLot(Lot lot) throws SQLException {
var sql = "INSERT INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)"
+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+ " ON CONFLICT(lot_id) DO UPDATE SET "
+ "sale_id = excluded.sale_id, title = excluded.title, description = excluded.description, "
+ "manufacturer = excluded.manufacturer, type = excluded.type, year = excluded.year, category = excluded.category, "
+ "current_bid = excluded.current_bid, currency = excluded.currency, url = excluded.url, closing_time = excluded.closing_time";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, lot.lotId);
ps.setInt(2, lot.saleId);
ps.setString(3, lot.title);
ps.setString(4, lot.description);
ps.setString(5, lot.manufacturer);
ps.setString(6, lot.type);
ps.setInt(7, lot.year);
ps.setString(8, lot.category);
ps.setDouble(9, lot.currentBid);
ps.setString(10, lot.currency);
ps.setString(11, lot.url);
ps.setString(12, lot.closingTime != null ? lot.closingTime.toString() : null);
ps.setInt(13, lot.closingNotified ? 1 : 0);
ps.executeUpdate();
}
}
/**
* Inserts a new image record. Each image is associated with a lot and
* stores both the original URL and the local file path. Detected
* labels are stored as a comma separated string.
*/
synchronized void insertImage(int lotId, String url, String filePath, List<String> labels) throws SQLException {
var sql = "INSERT INTO images (lot_id, url, file_path, labels) VALUES (?, ?, ?, ?)";
try (var conn = DriverManager.getConnection(this.url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, lotId);
ps.setString(2, url);
ps.setString(3, filePath);
ps.setString(4, String.join(",", labels));
ps.executeUpdate();
}
}
/**
* Retrieves all lots that are still active (i.e., have a closing time
* in the future or unknown). Only these lots need to be monitored.
*/
synchronized List<Lot> getActiveLots() throws SQLException {
List<Lot> list = new ArrayList<>();
var sql = "SELECT lot_id, sale_id, current_bid, currency, closing_time, closing_notified FROM lots";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
var lot = new Lot();
lot.lotId = rs.getInt("lot_id");
lot.saleId = rs.getInt("sale_id");
lot.currentBid = rs.getDouble("current_bid");
lot.currency = rs.getString("currency");
var closing = rs.getString("closing_time");
lot.closingNotified = rs.getInt("closing_notified") != 0;
if (closing != null) {
lot.closingTime = LocalDateTime.parse(closing);
}
return auctions;
}
/**
* Inserts or updates a lot record (typically called by external scraper)
*/
synchronized void upsertLot(Lot lot) throws SQLException {
var sql = """
INSERT INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(lot_id) DO UPDATE SET
sale_id = excluded.sale_id,
title = excluded.title,
description = excluded.description,
manufacturer = excluded.manufacturer,
type = excluded.type,
year = excluded.year,
category = excluded.category,
current_bid = excluded.current_bid,
currency = excluded.currency,
url = excluded.url,
closing_time = excluded.closing_time
""";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, lot.lotId());
ps.setInt(2, lot.saleId());
ps.setString(3, lot.title());
ps.setString(4, lot.description());
ps.setString(5, lot.manufacturer());
ps.setString(6, lot.type());
ps.setInt(7, lot.year());
ps.setString(8, lot.category());
ps.setDouble(9, lot.currentBid());
ps.setString(10, lot.currency());
ps.setString(11, lot.url());
ps.setString(12, lot.closingTime() != null ? lot.closingTime().toString() : null);
ps.setInt(13, lot.closingNotified() ? 1 : 0);
ps.executeUpdate();
}
}
/**
* Inserts a new image record with object detection labels
*/
synchronized void insertImage(int lotId, String url, String filePath, List<String> labels) throws SQLException {
var sql = "INSERT INTO images (lot_id, url, file_path, labels, processed_at) VALUES (?, ?, ?, ?, ?)";
try (var conn = DriverManager.getConnection(this.url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, lotId);
ps.setString(2, url);
ps.setString(3, filePath);
ps.setString(4, String.join(",", labels));
ps.setLong(5, Instant.now().getEpochSecond());
ps.executeUpdate();
}
}
/**
* Retrieves images for a specific lot
*/
synchronized List<ImageRecord> getImagesForLot(int lotId) throws SQLException {
List<ImageRecord> images = new ArrayList<>();
var sql = "SELECT id, lot_id, url, file_path, labels FROM images WHERE lot_id = ?";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, lotId);
var rs = ps.executeQuery();
while (rs.next()) {
images.add(new ImageRecord(
rs.getInt("id"),
rs.getInt("lot_id"),
rs.getString("url"),
rs.getString("file_path"),
rs.getString("labels")
));
}
list.add(lot);
}
}
return list;
}
/**
* Retrieves all lots from the database.
*/
synchronized List<Lot> getAllLots() throws SQLException {
List<Lot> list = new ArrayList<>();
var sql = "SELECT lot_id, sale_id, title, current_bid, currency FROM lots";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
var lot = new Lot();
lot.lotId = rs.getInt("lot_id");
lot.saleId = rs.getInt("sale_id");
lot.title = rs.getString("title");
lot.currentBid = rs.getDouble("current_bid");
lot.currency = rs.getString("currency");
list.add(lot);
}
}
return list;
}
/**
* Gets the total number of images in the database.
*/
synchronized int getImageCount() throws SQLException {
var sql = "SELECT COUNT(*) as count FROM images";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
if (rs.next()) {
return rs.getInt("count");
}
}
return 0;
}
/**
* Updates the current bid of a lot after a bid refresh.
*/
synchronized void updateLotCurrentBid(Lot lot) throws SQLException {
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(
"UPDATE lots SET current_bid = ? WHERE lot_id = ?")) {
ps.setDouble(1, lot.currentBid);
ps.setInt(2, lot.lotId);
ps.executeUpdate();
}
}
/**
* Updates the closingNotified flag of a lot (set to 1 when we have
* warned the user about its imminent closure).
*/
synchronized void updateLotNotificationFlags(Lot lot) throws SQLException {
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(
"UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) {
ps.setInt(1, lot.closingNotified ? 1 : 0);
ps.setInt(2, lot.lotId);
ps.executeUpdate();
}
}
}
return images;
}
/**
* Retrieves all lots that are active and need monitoring
*/
synchronized List<Lot> getActiveLots() throws SQLException {
List<Lot> list = new ArrayList<>();
var sql = "SELECT lot_id, sale_id, title, description, manufacturer, type, year, category, " +
"current_bid, currency, url, closing_time, closing_notified FROM lots";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
var closingStr = rs.getString("closing_time");
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
list.add(new Lot(
rs.getInt("sale_id"),
rs.getInt("lot_id"),
rs.getString("title"),
rs.getString("description"),
rs.getString("manufacturer"),
rs.getString("type"),
rs.getInt("year"),
rs.getString("category"),
rs.getDouble("current_bid"),
rs.getString("currency"),
rs.getString("url"),
closing,
rs.getInt("closing_notified") != 0
));
}
}
return list;
}
/**
* Retrieves all lots from the database
*/
synchronized List<Lot> getAllLots() throws SQLException {
return getActiveLots();
}
/**
* Gets the total number of images in the database
*/
synchronized int getImageCount() throws SQLException {
var sql = "SELECT COUNT(*) as count FROM images";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
if (rs.next()) {
return rs.getInt("count");
}
}
return 0;
}
/**
* Updates the current bid of a lot (used by monitoring service)
*/
synchronized void updateLotCurrentBid(Lot lot) throws SQLException {
try (var conn = DriverManager.getConnection(url);
var ps = conn.prepareStatement("UPDATE lots SET current_bid = ? WHERE lot_id = ?")) {
ps.setDouble(1, lot.currentBid());
ps.setInt(2, lot.lotId());
ps.executeUpdate();
}
}
/**
* Updates the closingNotified flag of a lot
*/
synchronized void updateLotNotificationFlags(Lot lot) throws SQLException {
try (var conn = DriverManager.getConnection(url);
var ps = conn.prepareStatement("UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) {
ps.setInt(1, lot.closingNotified() ? 1 : 0);
ps.setInt(2, lot.lotId());
ps.executeUpdate();
}
}
/**
* Simple record for image data
*/
record ImageRecord(int id, int lotId, String url, String filePath, String labels) {}
}