This commit is contained in:
Tour
2025-12-03 15:40:19 +01:00
parent d3dc37576d
commit febd08821a
6 changed files with 861 additions and 47 deletions

View File

@@ -329,7 +329,111 @@ public class DatabaseService {
}
/**
* Simple record for image data
* Imports auctions from scraper's schema format.
* Reads from scraper's tables and converts to monitor format using adapter.
*
* @return List of imported auctions
*/
synchronized List<AuctionInfo> importAuctionsFromScraper() throws SQLException {
List<AuctionInfo> imported = new ArrayList<>();
var sql = "SELECT auction_id, title, location, url, lots_count, first_lot_closing_time, scraped_at " +
"FROM auctions WHERE location LIKE '%NL%'";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
try {
var auction = ScraperDataAdapter.fromScraperAuction(rs);
upsertAuction(auction);
imported.add(auction);
} catch (Exception e) {
System.err.println("Failed to import auction: " + e.getMessage());
}
}
} catch (SQLException e) {
// Table might not exist in scraper format - that's ok
Console.println(" Scraper auction table not found or incompatible schema");
}
return imported;
}
/**
* Imports lots from scraper's schema format.
* Reads from scraper's tables and converts to monitor format using adapter.
*
* @return List of imported lots
*/
synchronized List<Lot> importLotsFromScraper() throws SQLException {
List<Lot> imported = new ArrayList<>();
var sql = "SELECT lot_id, auction_id, title, description, category, " +
"current_bid, closing_time, url " +
"FROM lots";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
try {
var lot = ScraperDataAdapter.fromScraperLot(rs);
upsertLot(lot);
imported.add(lot);
} catch (Exception e) {
System.err.println("Failed to import lot: " + e.getMessage());
}
}
} catch (SQLException e) {
// Table might not exist in scraper format - that's ok
Console.println(" Scraper lots table not found or incompatible schema");
}
return imported;
}
/**
* Imports image URLs from scraper's schema.
* The scraper populates the images table with URLs but doesn't download them.
* This method retrieves undownloaded images for processing.
*
* @return List of image URLs that need to be downloaded
*/
synchronized List<ImageImportRecord> getUnprocessedImagesFromScraper() throws SQLException {
List<ImageImportRecord> images = new ArrayList<>();
var sql = """
SELECT i.lot_id, i.url, l.auction_id
FROM images i
LEFT JOIN lots l ON i.lot_id = l.lot_id
WHERE i.downloaded = 0 OR i.local_path IS NULL
""";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
String lotIdStr = rs.getString("lot_id");
String auctionIdStr = rs.getString("auction_id");
int lotId = ScraperDataAdapter.extractNumericId(lotIdStr);
int saleId = ScraperDataAdapter.extractNumericId(auctionIdStr);
images.add(new ImageImportRecord(
lotId,
saleId,
rs.getString("url")
));
}
} catch (SQLException e) {
Console.println(" No unprocessed images found in scraper format");
}
return images;
}
/**
* Simple record for image data from database
*/
record ImageRecord(int id, int lotId, String url, String filePath, String labels) {}
/**
* Record for importing images from scraper format
*/
record ImageImportRecord(int lotId, int saleId, String url) {}
}