From 9cb19bbd8bea6ae8e6c0820b384cd84c42b17c36 Mon Sep 17 00:00:00 2001 From: Tour Date: Sun, 7 Dec 2025 16:25:29 +0100 Subject: [PATCH] Features Former-commit-id: 394469923bd947d75aa64738acdb78ece70a0325 --- src/main/java/auctiora/DatabaseService.java | 408 ++++---------------- 1 file changed, 80 insertions(+), 328 deletions(-) diff --git a/src/main/java/auctiora/DatabaseService.java b/src/main/java/auctiora/DatabaseService.java index 858579c..2fa3672 100644 --- a/src/main/java/auctiora/DatabaseService.java +++ b/src/main/java/auctiora/DatabaseService.java @@ -39,16 +39,27 @@ public class DatabaseService { stmt.execute("PRAGMA busy_timeout=10000"); stmt.execute("PRAGMA synchronous=NORMAL"); + // Cache table (for HTTP caching) + stmt.execute(""" + CREATE TABLE IF NOT EXISTS cache ( + url TEXT PRIMARY KEY, + content BLOB, + timestamp REAL, + status_code INTEGER + )"""); + // Auctions table (populated by external scraper) - // auction_id is TEXT to match scraper format (e.g., "A7-40063-2") stmt.execute(""" CREATE TABLE IF NOT EXISTS auctions ( auction_id TEXT PRIMARY KEY, - title TEXT NOT NULL, + url TEXT UNIQUE, + title TEXT, location TEXT, + lots_count INTEGER, + first_lot_closing_time TEXT, + scraped_at TEXT, city TEXT, country TEXT, - url TEXT NOT NULL UNIQUE, type TEXT, lot_count INTEGER DEFAULT 0, closing_time TEXT, @@ -56,365 +67,106 @@ public class DatabaseService { )"""); // Lots table (populated by external scraper) - // lot_id and sale_id are TEXT to match scraper format (e.g., "A1-34732-49") stmt.execute(""" CREATE TABLE IF NOT EXISTS lots ( lot_id TEXT PRIMARY KEY, - sale_id TEXT, auction_id TEXT, + url TEXT UNIQUE, title TEXT, + current_bid TEXT, + bid_count INTEGER, + closing_time TEXT, + viewing_time TEXT, + pickup_date TEXT, + location TEXT, description TEXT, + category TEXT, + scraped_at TEXT, + sale_id INTEGER, manufacturer TEXT, type TEXT, year INTEGER, - category TEXT, - current_bid REAL, - currency TEXT, - url TEXT UNIQUE, - closing_time TEXT, + currency TEXT DEFAULT 'EUR', closing_notified INTEGER DEFAULT 0, - FOREIGN KEY (sale_id) REFERENCES auctions(auction_id), + starting_bid TEXT, + minimum_bid TEXT, + status TEXT, + brand TEXT, + model TEXT, + attributes_json TEXT, + first_bid_time TEXT, + last_bid_time TEXT, + bid_velocity REAL, + bid_increment REAL, + year_manufactured INTEGER, + condition_score REAL, + condition_description TEXT, + serial_number TEXT, + damage_description TEXT, + followers_count INTEGER DEFAULT 0, + estimated_min_price REAL, + estimated_max_price REAL, + lot_condition TEXT, + appearance TEXT, + estimated_min REAL, + estimated_max REAL, + next_bid_step_cents INTEGER, + condition TEXT, + category_path TEXT, + city_location TEXT, + country_code TEXT, + bidding_status TEXT, + packaging TEXT, + quantity INTEGER, + vat REAL, + buyer_premium_percentage REAL, + remarks TEXT, + reserve_price REAL, + reserve_met INTEGER, + view_count INTEGER, FOREIGN KEY (auction_id) REFERENCES auctions(auction_id) )"""); // Images table (populated by external scraper with URLs and local_path) - // This process only adds labels via object detection - // lot_id is TEXT to match scraper format stmt.execute(""" CREATE TABLE IF NOT EXISTS images ( id INTEGER PRIMARY KEY AUTOINCREMENT, lot_id TEXT, url TEXT, local_path TEXT, + downloaded INTEGER DEFAULT 0, labels TEXT, processed_at INTEGER, - downloaded INTEGER DEFAULT 0, FOREIGN KEY (lot_id) REFERENCES lots(lot_id) )"""); - // Migrate existing tables to add missing columns - migrateSchema(stmt); + // Bid history table + stmt.execute(""" + CREATE TABLE IF NOT EXISTS bid_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + lot_id TEXT NOT NULL, + bid_amount REAL NOT NULL, + bid_time TEXT NOT NULL, + is_autobid INTEGER DEFAULT 0, + bidder_id TEXT, + bidder_number INTEGER, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (lot_id) REFERENCES lots(lot_id) + )"""); // Indexes for performance + stmt.execute("CREATE INDEX IF NOT EXISTS idx_timestamp ON cache(timestamp)"); stmt.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)"); stmt.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)"); stmt.execute("CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id)"); + stmt.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_lot_url ON images(lot_id, url)"); + stmt.execute("CREATE INDEX IF NOT EXISTS idx_bid_history_lot_time ON bid_history(lot_id, bid_time)"); + stmt.execute("CREATE INDEX IF NOT EXISTS idx_bid_history_bidder ON bid_history(bidder_id)"); } } - /** - * Migrates existing database schema to add new columns. - * SQLite doesn't support DROP COLUMN, so we add columns with ALTER TABLE ADD COLUMN. - */ - private void migrateSchema(java.sql.Statement stmt) throws SQLException { - // Check auctions table for missing columns - try (var rs = stmt.executeQuery("PRAGMA table_info(auctions)")) { - var hasLocation = false; - var hasCity = false; - var hasCountry = false; - var hasType = false; - var hasLotCount = false; - var hasClosingTime = false; - var hasDiscoveredAt = false; - while (rs.next()) { - var colName = rs.getString("name"); - switch (colName) { - case "location" -> hasLocation = true; - case "city" -> hasCity = true; - case "country" -> hasCountry = true; - case "type" -> hasType = true; - case "lot_count" -> hasLotCount = true; - case "closing_time" -> hasClosingTime = true; - case "discovered_at" -> hasDiscoveredAt = true; - } - } - - if (!hasLocation) { - log.info("Migrating schema: Adding 'location' column to auctions table"); - stmt.execute("ALTER TABLE auctions ADD COLUMN location TEXT"); - } - if (!hasCity) { - log.info("Migrating schema: Adding 'city' column to auctions table"); - stmt.execute("ALTER TABLE auctions ADD COLUMN city TEXT"); - } - if (!hasCountry) { - log.info("Migrating schema: Adding 'country' column to auctions table"); - stmt.execute("ALTER TABLE auctions ADD COLUMN country TEXT"); - } - if (!hasType) { - log.info("Migrating schema: Adding 'type' column to auctions table"); - stmt.execute("ALTER TABLE auctions ADD COLUMN type TEXT"); - } - if (!hasLotCount) { - log.info("Migrating schema: Adding 'lot_count' column to auctions table"); - stmt.execute("ALTER TABLE auctions ADD COLUMN lot_count INTEGER DEFAULT 0"); - } - if (!hasClosingTime) { - log.info("Migrating schema: Adding 'closing_time' column to auctions table"); - stmt.execute("ALTER TABLE auctions ADD COLUMN closing_time TEXT"); - } - if (!hasDiscoveredAt) { - log.info("Migrating schema: Adding 'discovered_at' column to auctions table"); - stmt.execute("ALTER TABLE auctions ADD COLUMN discovered_at INTEGER"); - } - } catch (SQLException e) { - // Table might not exist yet, which is fine - log.debug("Could not check auctions table schema: " + e.getMessage()); - } - - // Check lots table for missing columns and migrate - try (var rs = stmt.executeQuery("PRAGMA table_info(lots)")) { - var hasSaleId = false; - var hasAuctionId = false; - var hasTitle = false; - var hasDescription = false; - var hasManufacturer = false; - var hasType = false; - var hasYear = false; - var hasCategory = false; - var hasCurrentBid = false; - var hasCurrency = false; - var hasUrl = false; - var hasClosingTime = false; - var hasClosingNotified = false; - - while (rs.next()) { - var colName = rs.getString("name"); - switch (colName) { - case "sale_id" -> hasSaleId = true; - case "auction_id" -> hasAuctionId = true; - case "title" -> hasTitle = true; - case "description" -> hasDescription = true; - case "manufacturer" -> hasManufacturer = true; - case "type" -> hasType = true; - case "year" -> hasYear = true; - case "category" -> hasCategory = true; - case "current_bid" -> hasCurrentBid = true; - case "currency" -> hasCurrency = true; - case "url" -> hasUrl = true; - case "closing_time" -> hasClosingTime = true; - case "closing_notified" -> hasClosingNotified = true; - } - } - - // If we have auction_id but not sale_id, we need to rename the column - // SQLite doesn't support RENAME COLUMN before 3.25.0, so we add sale_id and copy data - if (hasAuctionId && !hasSaleId) { - log.info("Migrating schema: Adding 'sale_id' column to lots table and copying data from auction_id"); - stmt.execute("ALTER TABLE lots ADD COLUMN sale_id INTEGER"); - stmt.execute("UPDATE lots SET sale_id = auction_id"); - } else if (!hasSaleId && !hasAuctionId) { - // New table, add sale_id - log.info("Migrating schema: Adding 'sale_id' column to new lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN sale_id INTEGER"); - } - - // Add missing columns for lot details - if (!hasTitle) { - log.info("Migrating schema: Adding 'title' column to lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN title TEXT"); - } - if (!hasDescription) { - log.info("Migrating schema: Adding 'description' column to lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN description TEXT"); - } - if (!hasManufacturer) { - log.info("Migrating schema: Adding 'manufacturer' column to lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN manufacturer TEXT"); - } - if (!hasType) { - log.info("Migrating schema: Adding 'type' column to lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN type TEXT"); - } - if (!hasYear) { - log.info("Migrating schema: Adding 'year' column to lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN year INTEGER"); - } - if (!hasCategory) { - log.info("Migrating schema: Adding 'category' column to lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN category TEXT"); - } - if (!hasCurrentBid) { - log.info("Migrating schema: Adding 'current_bid' column to lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN current_bid REAL"); - } - if (!hasCurrency) { - log.info("Migrating schema: Adding 'currency' column to lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN currency TEXT DEFAULT 'EUR'"); - } - if (!hasUrl) { - log.info("Migrating schema: Adding 'url' column to lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN url TEXT"); - } - if (!hasClosingTime) { - log.info("Migrating schema: Adding 'closing_time' column to lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN closing_time TEXT"); - } - if (!hasClosingNotified) { - log.info("Migrating schema: Adding 'closing_notified' column to lots table"); - stmt.execute("ALTER TABLE lots ADD COLUMN closing_notified INTEGER DEFAULT 0"); - } - - // Migrate intelligence fields from GraphQL API - migrateIntelligenceFields(stmt); - } catch (SQLException e) { - // Table might not exist yet, which is fine - log.debug("Could not check lots table schema: " + e.getMessage()); - } - - // Check images table for missing columns and migrate - try (var rs = stmt.executeQuery("PRAGMA table_info(images)")) { - var hasLabels = false; - var hasLocalPath = false; - var hasProcessedAt = false; - var hasDownloaded = false; - - while (rs.next()) { - var colName = rs.getString("name"); - switch (colName) { - case "labels" -> hasLabels = true; - case "local_path" -> hasLocalPath = true; - case "processed_at" -> hasProcessedAt = true; - case "downloaded" -> hasDownloaded = true; - } - } - - if (!hasLabels) { - log.info("Migrating schema: Adding 'labels' column to images table"); - stmt.execute("ALTER TABLE images ADD COLUMN labels TEXT"); - } - if (!hasLocalPath) { - log.info("Migrating schema: Adding 'local_path' column to images table"); - stmt.execute("ALTER TABLE images ADD COLUMN local_path TEXT"); - } - if (!hasProcessedAt) { - log.info("Migrating schema: Adding 'processed_at' column to images table"); - stmt.execute("ALTER TABLE images ADD COLUMN processed_at INTEGER"); - } - if (!hasDownloaded) { - log.info("Migrating schema: Adding 'downloaded' column to images table"); - stmt.execute("ALTER TABLE images ADD COLUMN downloaded INTEGER DEFAULT 0"); - } - } catch (SQLException e) { - // Table might not exist yet, which is fine - log.debug("Could not check images table schema: " + e.getMessage()); - } - } - - /** - * Migrates intelligence fields to lots table (GraphQL enrichment data) - */ - private void migrateIntelligenceFields(java.sql.Statement stmt) throws SQLException { - try (var rs = stmt.executeQuery("PRAGMA table_info(lots)")) { - var columns = new java.util.HashSet(); - while (rs.next()) { - columns.add(rs.getString("name")); - } - - // HIGH PRIORITY FIELDS - if (!columns.contains("followers_count")) { - log.info("Adding followers_count column"); - stmt.execute("ALTER TABLE lots ADD COLUMN followers_count INTEGER"); - } - if (!columns.contains("estimated_min")) { - log.info("Adding estimated_min column"); - stmt.execute("ALTER TABLE lots ADD COLUMN estimated_min REAL"); - } - if (!columns.contains("estimated_max")) { - log.info("Adding estimated_max column"); - stmt.execute("ALTER TABLE lots ADD COLUMN estimated_max REAL"); - } - if (!columns.contains("next_bid_step_cents")) { - log.info("Adding next_bid_step_cents column"); - stmt.execute("ALTER TABLE lots ADD COLUMN next_bid_step_cents INTEGER"); - } - if (!columns.contains("condition")) { - log.info("Adding condition column"); - stmt.execute("ALTER TABLE lots ADD COLUMN condition TEXT"); - } - if (!columns.contains("category_path")) { - log.info("Adding category_path column"); - stmt.execute("ALTER TABLE lots ADD COLUMN category_path TEXT"); - } - if (!columns.contains("city_location")) { - log.info("Adding city_location column"); - stmt.execute("ALTER TABLE lots ADD COLUMN city_location TEXT"); - } - if (!columns.contains("country_code")) { - log.info("Adding country_code column"); - stmt.execute("ALTER TABLE lots ADD COLUMN country_code TEXT"); - } - - // MEDIUM PRIORITY FIELDS - if (!columns.contains("bidding_status")) { - log.info("Adding bidding_status column"); - stmt.execute("ALTER TABLE lots ADD COLUMN bidding_status TEXT"); - } - if (!columns.contains("appearance")) { - log.info("Adding appearance column"); - stmt.execute("ALTER TABLE lots ADD COLUMN appearance TEXT"); - } - if (!columns.contains("packaging")) { - log.info("Adding packaging column"); - stmt.execute("ALTER TABLE lots ADD COLUMN packaging TEXT"); - } - if (!columns.contains("quantity")) { - log.info("Adding quantity column"); - stmt.execute("ALTER TABLE lots ADD COLUMN quantity INTEGER"); - } - if (!columns.contains("vat")) { - log.info("Adding vat column"); - stmt.execute("ALTER TABLE lots ADD COLUMN vat REAL"); - } - if (!columns.contains("buyer_premium_percentage")) { - log.info("Adding buyer_premium_percentage column"); - stmt.execute("ALTER TABLE lots ADD COLUMN buyer_premium_percentage REAL"); - } - if (!columns.contains("remarks")) { - log.info("Adding remarks column"); - stmt.execute("ALTER TABLE lots ADD COLUMN remarks TEXT"); - } - - // BID INTELLIGENCE FIELDS - if (!columns.contains("starting_bid")) { - log.info("Adding starting_bid column"); - stmt.execute("ALTER TABLE lots ADD COLUMN starting_bid REAL"); - } - if (!columns.contains("reserve_price")) { - log.info("Adding reserve_price column"); - stmt.execute("ALTER TABLE lots ADD COLUMN reserve_price REAL"); - } - if (!columns.contains("reserve_met")) { - log.info("Adding reserve_met column"); - stmt.execute("ALTER TABLE lots ADD COLUMN reserve_met INTEGER"); - } - if (!columns.contains("bid_increment")) { - log.info("Adding bid_increment column"); - stmt.execute("ALTER TABLE lots ADD COLUMN bid_increment REAL"); - } - if (!columns.contains("view_count")) { - log.info("Adding view_count column"); - stmt.execute("ALTER TABLE lots ADD COLUMN view_count INTEGER"); - } - if (!columns.contains("first_bid_time")) { - log.info("Adding first_bid_time column"); - stmt.execute("ALTER TABLE lots ADD COLUMN first_bid_time TEXT"); - } - if (!columns.contains("last_bid_time")) { - log.info("Adding last_bid_time column"); - stmt.execute("ALTER TABLE lots ADD COLUMN last_bid_time TEXT"); - } - if (!columns.contains("bid_velocity")) { - log.info("Adding bid_velocity column"); - stmt.execute("ALTER TABLE lots ADD COLUMN bid_velocity REAL"); - } - } catch (SQLException e) { - log.warn("Could not migrate intelligence fields: {}", e.getMessage()); - } - } - - /** + /** * Inserts or updates an auction record (typically called by external scraper) * Handles both auction_id conflicts and url uniqueness constraints */