tour/auctiora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Features

Browse Source
This commit is contained in:
Tour
2025-12-07 16:25:29 +01:00
parent 2da6049206
commit 394469923b
1 changed files with 80 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

408
src/main/java/auctiora/DatabaseService.java
View File

@@ -39,16 +39,27 @@ public class DatabaseService {
stmt.execute("PRAGMA busy_timeout=10000"); stmt.execute("PRAGMA busy_timeout=10000");
stmt.execute("PRAGMA synchronous=NORMAL"); stmt.execute("PRAGMA synchronous=NORMAL");
// Cache table (for HTTP caching)
stmt.execute("""
CREATE TABLE IF NOT EXISTS cache (
url TEXT PRIMARY KEY,
content BLOB,
timestamp REAL,
status_code INTEGER
)""");
// Auctions table (populated by external scraper) // Auctions table (populated by external scraper)
// auction_id is TEXT to match scraper format (e.g., "A7-40063-2")
stmt.execute(""" stmt.execute("""
CREATE TABLE IF NOT EXISTS auctions ( CREATE TABLE IF NOT EXISTS auctions (
auction_id TEXT PRIMARY KEY, auction_id TEXT PRIMARY KEY,
title TEXT NOT NULL, url TEXT UNIQUE,
title TEXT,
location TEXT, location TEXT,
lots_count INTEGER,
first_lot_closing_time TEXT,
scraped_at TEXT,
city TEXT, city TEXT,
country TEXT, country TEXT,
url TEXT NOT NULL UNIQUE,
type TEXT, type TEXT,
lot_count INTEGER DEFAULT 0, lot_count INTEGER DEFAULT 0,
closing_time TEXT, closing_time TEXT,
@@ -56,365 +67,106 @@ public class DatabaseService {
)"""); )""");
// Lots table (populated by external scraper) // Lots table (populated by external scraper)
// lot_id and sale_id are TEXT to match scraper format (e.g., "A1-34732-49")
stmt.execute(""" stmt.execute("""
CREATE TABLE IF NOT EXISTS lots ( CREATE TABLE IF NOT EXISTS lots (
lot_id TEXT PRIMARY KEY, lot_id TEXT PRIMARY KEY,
sale_id TEXT,
auction_id TEXT, auction_id TEXT,
url TEXT UNIQUE,
title TEXT, title TEXT,
current_bid TEXT,
bid_count INTEGER,
closing_time TEXT,
viewing_time TEXT,
pickup_date TEXT,
location TEXT,
description TEXT, description TEXT,
category TEXT,
scraped_at TEXT,
sale_id INTEGER,
manufacturer TEXT, manufacturer TEXT,
type TEXT, type TEXT,
year INTEGER, year INTEGER,
category TEXT, currency TEXT DEFAULT 'EUR',
current_bid REAL,
currency TEXT,
url TEXT UNIQUE,
closing_time TEXT,
closing_notified INTEGER DEFAULT 0, closing_notified INTEGER DEFAULT 0,
FOREIGN KEY (sale_id) REFERENCES auctions(auction_id), starting_bid TEXT,
minimum_bid TEXT,
status TEXT,
brand TEXT,
model TEXT,
attributes_json TEXT,
first_bid_time TEXT,
last_bid_time TEXT,
bid_velocity REAL,
bid_increment REAL,
year_manufactured INTEGER,
condition_score REAL,
condition_description TEXT,
serial_number TEXT,
damage_description TEXT,
followers_count INTEGER DEFAULT 0,
estimated_min_price REAL,
estimated_max_price REAL,
lot_condition TEXT,
appearance TEXT,
estimated_min REAL,
estimated_max REAL,
next_bid_step_cents INTEGER,
condition TEXT,
category_path TEXT,
city_location TEXT,
country_code TEXT,
bidding_status TEXT,
packaging TEXT,
quantity INTEGER,
vat REAL,
buyer_premium_percentage REAL,
remarks TEXT,
reserve_price REAL,
reserve_met INTEGER,
view_count INTEGER,
FOREIGN KEY (auction_id) REFERENCES auctions(auction_id) FOREIGN KEY (auction_id) REFERENCES auctions(auction_id)
)"""); )""");
// Images table (populated by external scraper with URLs and local_path) // Images table (populated by external scraper with URLs and local_path)
// This process only adds labels via object detection
// lot_id is TEXT to match scraper format
stmt.execute(""" stmt.execute("""
CREATE TABLE IF NOT EXISTS images ( CREATE TABLE IF NOT EXISTS images (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT, lot_id TEXT,
url TEXT, url TEXT,
local_path TEXT, local_path TEXT,
downloaded INTEGER DEFAULT 0,
labels TEXT, labels TEXT,
processed_at INTEGER, processed_at INTEGER,
downloaded INTEGER DEFAULT 0,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id) FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)"""); )""");
// Migrate existing tables to add missing columns // Bid history table
migrateSchema(stmt); stmt.execute("""
CREATE TABLE IF NOT EXISTS bid_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT NOT NULL,
bid_amount REAL NOT NULL,
bid_time TEXT NOT NULL,
is_autobid INTEGER DEFAULT 0,
bidder_id TEXT,
bidder_number INTEGER,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)""");
// Indexes for performance // Indexes for performance
stmt.execute("CREATE INDEX IF NOT EXISTS idx_timestamp ON cache(timestamp)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)"); stmt.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)"); stmt.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id)"); stmt.execute("CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id)");
stmt.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_lot_url ON images(lot_id, url)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_bid_history_lot_time ON bid_history(lot_id, bid_time)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_bid_history_bidder ON bid_history(bidder_id)");
} }
} }
/**
* Migrates existing database schema to add new columns.
* SQLite doesn't support DROP COLUMN, so we add columns with ALTER TABLE ADD COLUMN.
*/
private void migrateSchema(java.sql.Statement stmt) throws SQLException {
// Check auctions table for missing columns
try (var rs = stmt.executeQuery("PRAGMA table_info(auctions)")) {
var hasLocation = false;
var hasCity = false;
var hasCountry = false;
var hasType = false;
var hasLotCount = false;
var hasClosingTime = false;
var hasDiscoveredAt = false;
while (rs.next()) { /**
var colName = rs.getString("name");
switch (colName) {
case "location" -> hasLocation = true;
case "city" -> hasCity = true;
case "country" -> hasCountry = true;
case "type" -> hasType = true;
case "lot_count" -> hasLotCount = true;
case "closing_time" -> hasClosingTime = true;
case "discovered_at" -> hasDiscoveredAt = true;
}
}
if (!hasLocation) {
log.info("Migrating schema: Adding 'location' column to auctions table");
stmt.execute("ALTER TABLE auctions ADD COLUMN location TEXT");
}
if (!hasCity) {
log.info("Migrating schema: Adding 'city' column to auctions table");
stmt.execute("ALTER TABLE auctions ADD COLUMN city TEXT");
}
if (!hasCountry) {
log.info("Migrating schema: Adding 'country' column to auctions table");
stmt.execute("ALTER TABLE auctions ADD COLUMN country TEXT");
}
if (!hasType) {
log.info("Migrating schema: Adding 'type' column to auctions table");
stmt.execute("ALTER TABLE auctions ADD COLUMN type TEXT");
}
if (!hasLotCount) {
log.info("Migrating schema: Adding 'lot_count' column to auctions table");
stmt.execute("ALTER TABLE auctions ADD COLUMN lot_count INTEGER DEFAULT 0");
}
if (!hasClosingTime) {
log.info("Migrating schema: Adding 'closing_time' column to auctions table");
stmt.execute("ALTER TABLE auctions ADD COLUMN closing_time TEXT");
}
if (!hasDiscoveredAt) {
log.info("Migrating schema: Adding 'discovered_at' column to auctions table");
stmt.execute("ALTER TABLE auctions ADD COLUMN discovered_at INTEGER");
}
} catch (SQLException e) {
// Table might not exist yet, which is fine
log.debug("Could not check auctions table schema: " + e.getMessage());
}
// Check lots table for missing columns and migrate
try (var rs = stmt.executeQuery("PRAGMA table_info(lots)")) {
var hasSaleId = false;
var hasAuctionId = false;
var hasTitle = false;
var hasDescription = false;
var hasManufacturer = false;
var hasType = false;
var hasYear = false;
var hasCategory = false;
var hasCurrentBid = false;
var hasCurrency = false;
var hasUrl = false;
var hasClosingTime = false;
var hasClosingNotified = false;
while (rs.next()) {
var colName = rs.getString("name");
switch (colName) {
case "sale_id" -> hasSaleId = true;
case "auction_id" -> hasAuctionId = true;
case "title" -> hasTitle = true;
case "description" -> hasDescription = true;
case "manufacturer" -> hasManufacturer = true;
case "type" -> hasType = true;
case "year" -> hasYear = true;
case "category" -> hasCategory = true;
case "current_bid" -> hasCurrentBid = true;
case "currency" -> hasCurrency = true;
case "url" -> hasUrl = true;
case "closing_time" -> hasClosingTime = true;
case "closing_notified" -> hasClosingNotified = true;
}
}
// If we have auction_id but not sale_id, we need to rename the column
// SQLite doesn't support RENAME COLUMN before 3.25.0, so we add sale_id and copy data
if (hasAuctionId && !hasSaleId) {
log.info("Migrating schema: Adding 'sale_id' column to lots table and copying data from auction_id");
stmt.execute("ALTER TABLE lots ADD COLUMN sale_id INTEGER");
stmt.execute("UPDATE lots SET sale_id = auction_id");
} else if (!hasSaleId && !hasAuctionId) {
// New table, add sale_id
log.info("Migrating schema: Adding 'sale_id' column to new lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN sale_id INTEGER");
}
// Add missing columns for lot details
if (!hasTitle) {
log.info("Migrating schema: Adding 'title' column to lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN title TEXT");
}
if (!hasDescription) {
log.info("Migrating schema: Adding 'description' column to lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN description TEXT");
}
if (!hasManufacturer) {
log.info("Migrating schema: Adding 'manufacturer' column to lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN manufacturer TEXT");
}
if (!hasType) {
log.info("Migrating schema: Adding 'type' column to lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN type TEXT");
}
if (!hasYear) {
log.info("Migrating schema: Adding 'year' column to lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN year INTEGER");
}
if (!hasCategory) {
log.info("Migrating schema: Adding 'category' column to lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN category TEXT");
}
if (!hasCurrentBid) {
log.info("Migrating schema: Adding 'current_bid' column to lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN current_bid REAL");
}
if (!hasCurrency) {
log.info("Migrating schema: Adding 'currency' column to lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN currency TEXT DEFAULT 'EUR'");
}
if (!hasUrl) {
log.info("Migrating schema: Adding 'url' column to lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN url TEXT");
}
if (!hasClosingTime) {
log.info("Migrating schema: Adding 'closing_time' column to lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN closing_time TEXT");
}
if (!hasClosingNotified) {
log.info("Migrating schema: Adding 'closing_notified' column to lots table");
stmt.execute("ALTER TABLE lots ADD COLUMN closing_notified INTEGER DEFAULT 0");
}
// Migrate intelligence fields from GraphQL API
migrateIntelligenceFields(stmt);
} catch (SQLException e) {
// Table might not exist yet, which is fine
log.debug("Could not check lots table schema: " + e.getMessage());
}
// Check images table for missing columns and migrate
try (var rs = stmt.executeQuery("PRAGMA table_info(images)")) {
var hasLabels = false;
var hasLocalPath = false;
var hasProcessedAt = false;
var hasDownloaded = false;
while (rs.next()) {
var colName = rs.getString("name");
switch (colName) {
case "labels" -> hasLabels = true;
case "local_path" -> hasLocalPath = true;
case "processed_at" -> hasProcessedAt = true;
case "downloaded" -> hasDownloaded = true;
}
}
if (!hasLabels) {
log.info("Migrating schema: Adding 'labels' column to images table");
stmt.execute("ALTER TABLE images ADD COLUMN labels TEXT");
}
if (!hasLocalPath) {
log.info("Migrating schema: Adding 'local_path' column to images table");
stmt.execute("ALTER TABLE images ADD COLUMN local_path TEXT");
}
if (!hasProcessedAt) {
log.info("Migrating schema: Adding 'processed_at' column to images table");
stmt.execute("ALTER TABLE images ADD COLUMN processed_at INTEGER");
}
if (!hasDownloaded) {
log.info("Migrating schema: Adding 'downloaded' column to images table");
stmt.execute("ALTER TABLE images ADD COLUMN downloaded INTEGER DEFAULT 0");
}
} catch (SQLException e) {
// Table might not exist yet, which is fine
log.debug("Could not check images table schema: " + e.getMessage());
}
}
/**
* Migrates intelligence fields to lots table (GraphQL enrichment data)
*/
private void migrateIntelligenceFields(java.sql.Statement stmt) throws SQLException {
try (var rs = stmt.executeQuery("PRAGMA table_info(lots)")) {
var columns = new java.util.HashSet<String>();
while (rs.next()) {
columns.add(rs.getString("name"));
}
// HIGH PRIORITY FIELDS
if (!columns.contains("followers_count")) {
log.info("Adding followers_count column");
stmt.execute("ALTER TABLE lots ADD COLUMN followers_count INTEGER");
}
if (!columns.contains("estimated_min")) {
log.info("Adding estimated_min column");
stmt.execute("ALTER TABLE lots ADD COLUMN estimated_min REAL");
}
if (!columns.contains("estimated_max")) {
log.info("Adding estimated_max column");
stmt.execute("ALTER TABLE lots ADD COLUMN estimated_max REAL");
}
if (!columns.contains("next_bid_step_cents")) {
log.info("Adding next_bid_step_cents column");
stmt.execute("ALTER TABLE lots ADD COLUMN next_bid_step_cents INTEGER");
}
if (!columns.contains("condition")) {
log.info("Adding condition column");
stmt.execute("ALTER TABLE lots ADD COLUMN condition TEXT");
}
if (!columns.contains("category_path")) {
log.info("Adding category_path column");
stmt.execute("ALTER TABLE lots ADD COLUMN category_path TEXT");
}
if (!columns.contains("city_location")) {
log.info("Adding city_location column");
stmt.execute("ALTER TABLE lots ADD COLUMN city_location TEXT");
}
if (!columns.contains("country_code")) {
log.info("Adding country_code column");
stmt.execute("ALTER TABLE lots ADD COLUMN country_code TEXT");
}
// MEDIUM PRIORITY FIELDS
if (!columns.contains("bidding_status")) {
log.info("Adding bidding_status column");
stmt.execute("ALTER TABLE lots ADD COLUMN bidding_status TEXT");
}
if (!columns.contains("appearance")) {
log.info("Adding appearance column");
stmt.execute("ALTER TABLE lots ADD COLUMN appearance TEXT");
}
if (!columns.contains("packaging")) {
log.info("Adding packaging column");
stmt.execute("ALTER TABLE lots ADD COLUMN packaging TEXT");
}
if (!columns.contains("quantity")) {
log.info("Adding quantity column");
stmt.execute("ALTER TABLE lots ADD COLUMN quantity INTEGER");
}
if (!columns.contains("vat")) {
log.info("Adding vat column");
stmt.execute("ALTER TABLE lots ADD COLUMN vat REAL");
}
if (!columns.contains("buyer_premium_percentage")) {
log.info("Adding buyer_premium_percentage column");
stmt.execute("ALTER TABLE lots ADD COLUMN buyer_premium_percentage REAL");
}
if (!columns.contains("remarks")) {
log.info("Adding remarks column");
stmt.execute("ALTER TABLE lots ADD COLUMN remarks TEXT");
}
// BID INTELLIGENCE FIELDS
if (!columns.contains("starting_bid")) {
log.info("Adding starting_bid column");
stmt.execute("ALTER TABLE lots ADD COLUMN starting_bid REAL");
}
if (!columns.contains("reserve_price")) {
log.info("Adding reserve_price column");
stmt.execute("ALTER TABLE lots ADD COLUMN reserve_price REAL");
}
if (!columns.contains("reserve_met")) {
log.info("Adding reserve_met column");
stmt.execute("ALTER TABLE lots ADD COLUMN reserve_met INTEGER");
}
if (!columns.contains("bid_increment")) {
log.info("Adding bid_increment column");
stmt.execute("ALTER TABLE lots ADD COLUMN bid_increment REAL");
}
if (!columns.contains("view_count")) {
log.info("Adding view_count column");
stmt.execute("ALTER TABLE lots ADD COLUMN view_count INTEGER");
}
if (!columns.contains("first_bid_time")) {
log.info("Adding first_bid_time column");
stmt.execute("ALTER TABLE lots ADD COLUMN first_bid_time TEXT");
}
if (!columns.contains("last_bid_time")) {
log.info("Adding last_bid_time column");
stmt.execute("ALTER TABLE lots ADD COLUMN last_bid_time TEXT");
}
if (!columns.contains("bid_velocity")) {
log.info("Adding bid_velocity column");
stmt.execute("ALTER TABLE lots ADD COLUMN bid_velocity REAL");
}
} catch (SQLException e) {
log.warn("Could not migrate intelligence fields: {}", e.getMessage());
}
}
/**
* Inserts or updates an auction record (typically called by external scraper) * Inserts or updates an auction record (typically called by external scraper)
* Handles both auction_id conflicts and url uniqueness constraints * Handles both auction_id conflicts and url uniqueness constraints
*/ */