Tour/auctiora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitor-page

Browse Source
This commit is contained in:
Tour
2025-12-05 07:06:34 +01:00
parent 04df491d64
commit 6325d07909
17 changed files with 1117 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
src/main/java/auctiora/DatabaseService.java
View File

@@ -24,7 +24,8 @@ public class DatabaseService {
private final String url;
DatabaseService(String dbPath) {
this.url = "jdbc:sqlite:" + dbPath;
// Enable WAL mode and busy timeout for concurrent access
this.url = "jdbc:sqlite:" + dbPath + "?journal_mode=WAL&busy_timeout=10000";
}
/**
@@ -33,6 +34,11 @@ public class DatabaseService {
*/
void ensureSchema() throws SQLException {
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
// Enable WAL mode for better concurrent access
stmt.execute("PRAGMA journal_mode=WAL");
stmt.execute("PRAGMA busy_timeout=10000");
stmt.execute("PRAGMA synchronous=NORMAL");
// Auctions table (populated by external scraper)
stmt.execute("""
CREATE TABLE IF NOT EXISTS auctions (
@@ -359,38 +365,67 @@ public class DatabaseService {
* Inserts or updates a lot record (typically called by external scraper)
*/
synchronized void upsertLot(Lot lot) throws SQLException {
var sql = """
INSERT INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(lot_id) DO UPDATE SET
sale_id = excluded.sale_id,
title = excluded.title,
description = excluded.description,
manufacturer = excluded.manufacturer,
type = excluded.type,
year = excluded.year,
category = excluded.category,
current_bid = excluded.current_bid,
currency = excluded.currency,
url = excluded.url,
closing_time = excluded.closing_time
// First try to update existing lot by lot_id
var updateSql = """
UPDATE lots SET
sale_id = ?,
title = ?,
description = ?,
manufacturer = ?,
type = ?,
year = ?,
category = ?,
current_bid = ?,
currency = ?,
url = ?,
closing_time = ?
WHERE lot_id = ?
""";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setLong(1, lot.lotId());
ps.setLong(2, lot.saleId());
ps.setString(3, lot.title());
ps.setString(4, lot.description());
ps.setString(5, lot.manufacturer());
ps.setString(6, lot.type());
ps.setInt(7, lot.year());
ps.setString(8, lot.category());
ps.setDouble(9, lot.currentBid());
ps.setString(10, lot.currency());
ps.setString(11, lot.url());
ps.setString(12, lot.closingTime() != null ? lot.closingTime().toString() : null);
ps.setInt(13, lot.closingNotified() ? 1 : 0);
ps.executeUpdate();
var insertSql = """
INSERT OR IGNORE INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""";
try (var conn = DriverManager.getConnection(url)) {
// Try UPDATE first
try (var ps = conn.prepareStatement(updateSql)) {
ps.setLong(1, lot.saleId());
ps.setString(2, lot.title());
ps.setString(3, lot.description());
ps.setString(4, lot.manufacturer());
ps.setString(5, lot.type());
ps.setInt(6, lot.year());
ps.setString(7, lot.category());
ps.setDouble(8, lot.currentBid());
ps.setString(9, lot.currency());
ps.setString(10, lot.url());
ps.setString(11, lot.closingTime() != null ? lot.closingTime().toString() : null);
ps.setLong(12, lot.lotId());
int updated = ps.executeUpdate();
if (updated > 0) {
return; // Successfully updated existing record
}
}
// If no rows updated, try INSERT (ignore if conflicts with UNIQUE constraints)
try (var ps = conn.prepareStatement(insertSql)) {
ps.setLong(1, lot.lotId());
ps.setLong(2, lot.saleId());
ps.setString(3, lot.title());
ps.setString(4, lot.description());
ps.setString(5, lot.manufacturer());
ps.setString(6, lot.type());
ps.setInt(7, lot.year());
ps.setString(8, lot.category());
ps.setDouble(9, lot.currentBid());
ps.setString(10, lot.currency());
ps.setString(11, lot.url());
ps.setString(12, lot.closingTime() != null ? lot.closingTime().toString() : null);
ps.setInt(13, lot.closingNotified() ? 1 : 0);
ps.executeUpdate();
}
}
}
@@ -520,20 +555,39 @@ public class DatabaseService {
/**
* Imports auctions from scraper's schema format.
* Reads from scraper's tables and converts to monitor format using adapter.
* Since the scraper doesn't populate a separate auctions table,
* we derive auction metadata by aggregating lots data.
*
* @return List of imported auctions
*/
synchronized List<AuctionInfo> importAuctionsFromScraper() throws SQLException {
List<AuctionInfo> imported = new ArrayList<>();
var sql = "SELECT auction_id, title, location, url, lots_count, first_lot_closing_time, scraped_at " +
"FROM auctions WHERE location LIKE '%NL%'";
// Derive auctions from lots table (scraper doesn't populate auctions table)
var sql = """
SELECT
l.auction_id,
MIN(l.title) as title,
MIN(l.location) as location,
MIN(l.url) as url,
COUNT(*) as lots_count,
MIN(l.closing_time) as first_lot_closing_time,
MIN(l.scraped_at) as scraped_at
FROM lots l
WHERE l.auction_id IS NOT NULL
GROUP BY l.auction_id
""";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
try {
var auction = ScraperDataAdapter.fromScraperAuction(rs);
// Skip auctions with invalid IDs (0 indicates parsing failed)
if (auction.auctionId() == 0L) {
log.debug("Skipping auction with invalid ID: auction_id={}", auction.auctionId());
continue;
}
upsertAuction(auction);
imported.add(auction);
} catch (Exception e) {
@@ -542,9 +596,9 @@ public class DatabaseService {
}
} catch (SQLException e) {
// Table might not exist in scraper format - that's ok
log.info("ℹ️ Scraper auction table not found or incompatible schema");
log.info("ℹ️ Scraper lots table not found or incompatible schema: {}", e.getMessage());
}
return imported;
}
@@ -559,12 +613,17 @@ public class DatabaseService {
var sql = "SELECT lot_id, auction_id, title, description, category, " +
"current_bid, closing_time, url " +
"FROM lots";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
try {
var lot = ScraperDataAdapter.fromScraperLot(rs);
// Skip lots with invalid IDs (0 indicates parsing failed)
if (lot.lotId() == 0L || lot.saleId() == 0L) {
log.debug("Skipping lot with invalid ID: lot_id={}, sale_id={}", lot.lotId(), lot.saleId());
continue;
}
upsertLot(lot);
imported.add(lot);
} catch (Exception e) {
@@ -575,7 +634,7 @@ public class DatabaseService {
// Table might not exist in scraper format - that's ok
log.info("ℹ️ Scraper lots table not found or incompatible schema");
}
return imported;
}
@@ -600,10 +659,16 @@ public class DatabaseService {
while (rs.next()) {
var lotIdStr = rs.getString("lot_id");
var auctionIdStr = rs.getString("auction_id");
var lotId = ScraperDataAdapter.extractNumericId(lotIdStr);
var saleId = ScraperDataAdapter.extractNumericId(auctionIdStr);
// Skip images with invalid IDs (0 indicates parsing failed)
if (lotId == 0L || saleId == 0L) {
log.debug("Skipping image with invalid ID: lot_id={}, sale_id={}", lotId, saleId);
continue;
}
images.add(new ImageImportRecord(
lotId,
saleId,

9
src/test/java/auctiora/ScraperDataAdapterTest.java
View File

@@ -34,6 +34,15 @@ class ScraperDataAdapterTest {
assertEquals(123456, ScraperDataAdapter.extractNumericId("A7-1234-56"));
}
@Test
@DisplayName("Should return 0 for IDs that exceed Long.MAX_VALUE")
void testExtractNumericIdTooLarge() {
// These IDs are too large for a long (> 19 digits or > Long.MAX_VALUE)
assertEquals(0, ScraperDataAdapter.extractNumericId("856462986966260305674"));
assertEquals(0, ScraperDataAdapter.extractNumericId("28492384530402679688"));
assertEquals(0, ScraperDataAdapter.extractNumericId("A7-856462986966260305674"));
}
@Test
@DisplayName("Should convert scraper auction format to AuctionInfo")
void testFromScraperAuction() throws SQLException {