diff --git a/README.md b/README.md index 43a18c2..1f51f27 100644 --- a/README.md +++ b/README.md @@ -384,7 +384,7 @@ The scraper creates three tables: **images** - `id` (PRIMARY KEY) -- `lot_id`, `url`, `file_path`, `labels` (detected objects) +- `lot_id`, `url`, `local_path`, `labels` (detected objects) ## Notification Examples diff --git a/src/main/java/auctiora/DatabaseService.java b/src/main/java/auctiora/DatabaseService.java index 3fb4fb2..d36d9f9 100644 --- a/src/main/java/auctiora/DatabaseService.java +++ b/src/main/java/auctiora/DatabaseService.java @@ -79,7 +79,7 @@ public class DatabaseService { id INTEGER PRIMARY KEY AUTOINCREMENT, lot_id INTEGER, url TEXT, - file_path TEXT, + local_path TEXT, labels TEXT, processed_at INTEGER, FOREIGN KEY (lot_id) REFERENCES lots(lot_id) @@ -433,7 +433,7 @@ public class DatabaseService { * Inserts a new image record with object detection labels */ synchronized void insertImage(long lotId, String url, String filePath, List labels) throws SQLException { - var sql = "INSERT INTO images (lot_id, url, file_path, labels, processed_at) VALUES (?, ?, ?, ?, ?)"; + var sql = "INSERT INTO images (lot_id, url, local_path, labels, processed_at) VALUES (?, ?, ?, ?, ?)"; try (var conn = DriverManager.getConnection(this.url); var ps = conn.prepareStatement(sql)) { ps.setLong(1, lotId); ps.setString(2, url); @@ -449,7 +449,7 @@ public class DatabaseService { */ synchronized List getImagesForLot(long lotId) throws SQLException { List images = new ArrayList<>(); - var sql = "SELECT id, lot_id, url, file_path, labels FROM images WHERE lot_id = ?"; + var sql = "SELECT id, lot_id, url, local_path, labels FROM images WHERE lot_id = ?"; try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) { ps.setLong(1, lotId); @@ -459,7 +459,7 @@ public class DatabaseService { rs.getInt("id"), rs.getLong("lot_id"), rs.getString("url"), - rs.getString("file_path"), + rs.getString("local_path"), rs.getString("labels") )); } diff --git a/wiki/DATABASE_ARCHITECTURE.md b/wiki/DATABASE_ARCHITECTURE.md index 072299b..c49307b 100644 --- a/wiki/DATABASE_ARCHITECTURE.md +++ b/wiki/DATABASE_ARCHITECTURE.md @@ -90,7 +90,7 @@ CREATE TABLE images ( id INTEGER PRIMARY KEY AUTOINCREMENT, lot_id INTEGER, url TEXT, - file_path TEXT, + local_path TEXT, labels TEXT, -- Object detection results processed_at INTEGER, FOREIGN KEY (lot_id) REFERENCES lots(lot_id) diff --git a/wiki/INTEGRATION_GUIDE.md b/wiki/INTEGRATION_GUIDE.md index 890f29b..1190b08 100644 --- a/wiki/INTEGRATION_GUIDE.md +++ b/wiki/INTEGRATION_GUIDE.md @@ -49,58 +49,58 @@ This document describes how **Troostwijk Monitor** (this Java project) integrate The scraper and monitor use **slightly different schemas** that need to be reconciled: -| Scraper Table | Monitor Table | Integration Notes | -|---------------|---------------|-------------------| -| `auctions` | `auctions` | ✅ **Compatible** - same structure | -| `lots` | `lots` | ⚠️ **Needs mapping** - field name differences | -| `images` | `images` | ⚠️ **Partial overlap** - different purposes | -| `cache` | N/A | ❌ Monitor doesn't use cache | +| Scraper Table | Monitor Table | Integration Notes | +|---------------|---------------|-----------------------------------------------| +| `auctions` | `auctions` | ✅ **Compatible** - same structure | +| `lots` | `lots` | ⚠️ **Needs mapping** - field name differences | +| `images` | `images` | ⚠️ **Partial overlap** - different purposes | +| `cache` | N/A | ❌ Monitor doesn't use cache | ### Field Mapping: `auctions` Table -| Scraper Field | Monitor Field | Notes | -|---------------|---------------|-------| -| `auction_id` (TEXT) | `auction_id` (INTEGER) | ⚠️ **TYPE MISMATCH** - Scraper uses "A7-39813", Monitor expects INT | -| `url` | `url` | ✅ Compatible | -| `title` | `title` | ✅ Compatible | -| `location` | `location`, `city`, `country` | ⚠️ Monitor splits into 3 fields | -| `lots_count` | `lot_count` | ⚠️ Name difference | -| `first_lot_closing_time` | `closing_time` | ⚠️ Name difference | -| `scraped_at` | `discovered_at` | ⚠️ Name + type difference (TEXT vs INTEGER timestamp) | +| Scraper Field | Monitor Field | Notes | +|--------------------------|-------------------------------|---------------------------------------------------------------------| +| `auction_id` (TEXT) | `auction_id` (INTEGER) | ⚠️ **TYPE MISMATCH** - Scraper uses "A7-39813", Monitor expects INT | +| `url` | `url` | ✅ Compatible | +| `title` | `title` | ✅ Compatible | +| `location` | `location`, `city`, `country` | ⚠️ Monitor splits into 3 fields | +| `lots_count` | `lot_count` | ⚠️ Name difference | +| `first_lot_closing_time` | `closing_time` | ⚠️ Name difference | +| `scraped_at` | `discovered_at` | ⚠️ Name + type difference (TEXT vs INTEGER timestamp) | ### Field Mapping: `lots` Table -| Scraper Field | Monitor Field | Notes | -|---------------|---------------|-------| -| `lot_id` (TEXT) | `lot_id` (INTEGER) | ⚠️ **TYPE MISMATCH** - "A1-28505-5" vs INT | -| `auction_id` | `sale_id` | ⚠️ Different name | -| `url` | `url` | ✅ Compatible | -| `title` | `title` | ✅ Compatible | -| `current_bid` (TEXT) | `current_bid` (REAL) | ⚠️ **TYPE MISMATCH** - "€123.45" vs 123.45 | -| `bid_count` | N/A | ℹ️ Monitor doesn't track | -| `closing_time` | `closing_time` | ⚠️ Format difference (TEXT vs LocalDateTime) | -| `viewing_time` | N/A | ℹ️ Monitor doesn't track | -| `pickup_date` | N/A | ℹ️ Monitor doesn't track | -| `location` | N/A | ℹ️ Monitor doesn't track lot location separately | -| `description` | `description` | ✅ Compatible | -| `category` | `category` | ✅ Compatible | -| N/A | `manufacturer` | ℹ️ Monitor has additional field | -| N/A | `type` | ℹ️ Monitor has additional field | -| N/A | `year` | ℹ️ Monitor has additional field | -| N/A | `currency` | ℹ️ Monitor has additional field | -| N/A | `closing_notified` | ℹ️ Monitor tracking field | +| Scraper Field | Monitor Field | Notes | +|----------------------|----------------------|--------------------------------------------------| +| `lot_id` (TEXT) | `lot_id` (INTEGER) | ⚠️ **TYPE MISMATCH** - "A1-28505-5" vs INT | +| `auction_id` | `sale_id` | ⚠️ Different name | +| `url` | `url` | ✅ Compatible | +| `title` | `title` | ✅ Compatible | +| `current_bid` (TEXT) | `current_bid` (REAL) | ⚠️ **TYPE MISMATCH** - "€123.45" vs 123.45 | +| `bid_count` | N/A | ℹ️ Monitor doesn't track | +| `closing_time` | `closing_time` | ⚠️ Format difference (TEXT vs LocalDateTime) | +| `viewing_time` | N/A | ℹ️ Monitor doesn't track | +| `pickup_date` | N/A | ℹ️ Monitor doesn't track | +| `location` | N/A | ℹ️ Monitor doesn't track lot location separately | +| `description` | `description` | ✅ Compatible | +| `category` | `category` | ✅ Compatible | +| N/A | `manufacturer` | ℹ️ Monitor has additional field | +| N/A | `type` | ℹ️ Monitor has additional field | +| N/A | `year` | ℹ️ Monitor has additional field | +| N/A | `currency` | ℹ️ Monitor has additional field | +| N/A | `closing_notified` | ℹ️ Monitor tracking field | ### Field Mapping: `images` Table -| Scraper Field | Monitor Field | Notes | -|---------------|---------------|-------| -| `id` | `id` | ✅ Compatible | -| `lot_id` | `lot_id` | ⚠️ Type difference (TEXT vs INTEGER) | -| `url` | `url` | ✅ Compatible | -| `local_path` | `file_path` | ⚠️ Different name | -| `downloaded` (INTEGER) | N/A | ℹ️ Monitor uses `processed_at` instead | -| N/A | `labels` (TEXT) | ℹ️ Monitor adds detected objects | -| N/A | `processed_at` (INTEGER) | ℹ️ Monitor tracking field | +| Scraper Field | Monitor Field | Notes | +|------------------------|--------------------------|----------------------------------------| +| `id` | `id` | ✅ Compatible | +| `lot_id` | `lot_id` | ⚠️ Type difference (TEXT vs INTEGER) | +| `url` | `url` | ✅ Compatible | +| `local_path` | `Local_path` | ⚠️ Different name | +| `downloaded` (INTEGER) | N/A | ℹ️ Monitor uses `processed_at` instead | +| N/A | `labels` (TEXT) | ℹ️ Monitor adds detected objects | +| N/A | `processed_at` (INTEGER) | ℹ️ Monitor tracking field | ## Integration Options @@ -252,8 +252,7 @@ CREATE TABLE IF NOT EXISTS images ( id INTEGER PRIMARY KEY AUTOINCREMENT, lot_id TEXT, -- FK: "A1-28505-5" url TEXT, -- Image URL from website - file_path TEXT, -- Local path after download - local_path TEXT, -- Alias for compatibility + local_path TEXT, -- Local path after download labels TEXT, -- Detected objects (comma-separated) downloaded INTEGER DEFAULT 0, -- 0=pending, 1=downloaded processed_at INTEGER, -- Unix timestamp when processed @@ -469,7 +468,7 @@ sqlite3 /mnt/okcomputer/output/cache.db "SELECT COUNT(*) FROM lots" **Check**: 1. Scraper writes image URLs to `images` table 2. Monitor reads from `images` table with `downloaded=0` -3. Field name mapping: `local_path` vs `file_path` +3. Field name mapping: `local_path` vs `local_path` ## Next Steps