From 4f0d5113f550dddafe9f851148cd34668ecd9f5a Mon Sep 17 00:00:00 2001 From: Tour Date: Sun, 7 Dec 2025 13:31:40 +0100 Subject: [PATCH] Fix GraphQL enrichment to use displayId instead of numeric lotId MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added displayId (String) field to Lot record for full lot ID (e.g., "A1-34732-49") - Updated ScraperDataAdapter to extract both numeric ID and displayId from database - Fixed TroostwijkGraphQLClient to query by displayId using lotDetails() instead of lot() - Matched Python scraper's query structure with LOT_BIDDING_QUERY pattern - Updated GraphQL response parsing to handle lotDetails.location and biddingStatistics - Added upsertLotWithIntelligence() method to DatabaseService for full intelligence updates - Updated LotEnrichmentService to pass displayId to GraphQL client This fixes the "No intelligence data returned" error on production server. GraphQL API requires string displayId parameter, not numeric lot ID. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Former-commit-id: 4d7da943159b8071828ee929271db8f83093ce63 --- src/main/java/auctiora/DatabaseService.java | 93 +++++++++++++++++- src/main/java/auctiora/Lot.java | 3 +- .../java/auctiora/LotEnrichmentService.java | 66 ++++++------- .../java/auctiora/ScraperDataAdapter.java | 12 ++- .../auctiora/TroostwijkGraphQLClient.java | 98 ++++++++++++------- 5 files changed, 195 insertions(+), 77 deletions(-) diff --git a/src/main/java/auctiora/DatabaseService.java b/src/main/java/auctiora/DatabaseService.java index 646da3e..35369a6 100644 --- a/src/main/java/auctiora/DatabaseService.java +++ b/src/main/java/auctiora/DatabaseService.java @@ -633,7 +633,98 @@ public class DatabaseService { } } } - + + /** + * Updates a lot with full intelligence data from GraphQL enrichment. + * This is a comprehensive update that includes all 24 intelligence fields. + */ + synchronized void upsertLotWithIntelligence(Lot lot) throws SQLException { + var sql = """ + UPDATE lots SET + sale_id = ?, + title = ?, + description = ?, + manufacturer = ?, + type = ?, + year = ?, + category = ?, + current_bid = ?, + currency = ?, + url = ?, + closing_time = ?, + followers_count = ?, + estimated_min = ?, + estimated_max = ?, + next_bid_step_in_cents = ?, + condition = ?, + category_path = ?, + city_location = ?, + country_code = ?, + bidding_status = ?, + appearance = ?, + packaging = ?, + quantity = ?, + vat = ?, + buyer_premium_percentage = ?, + remarks = ?, + starting_bid = ?, + reserve_price = ?, + reserve_met = ?, + bid_increment = ?, + view_count = ?, + first_bid_time = ?, + last_bid_time = ?, + bid_velocity = ? + WHERE lot_id = ? + """; + + try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) { + ps.setLong(1, lot.saleId()); + ps.setString(2, lot.title()); + ps.setString(3, lot.description()); + ps.setString(4, lot.manufacturer()); + ps.setString(5, lot.type()); + ps.setInt(6, lot.year()); + ps.setString(7, lot.category()); + ps.setDouble(8, lot.currentBid()); + ps.setString(9, lot.currency()); + ps.setString(10, lot.url()); + ps.setString(11, lot.closingTime() != null ? lot.closingTime().toString() : null); + + // Intelligence fields + if (lot.followersCount() != null) ps.setInt(12, lot.followersCount()); else ps.setNull(12, java.sql.Types.INTEGER); + if (lot.estimatedMin() != null) ps.setDouble(13, lot.estimatedMin()); else ps.setNull(13, java.sql.Types.REAL); + if (lot.estimatedMax() != null) ps.setDouble(14, lot.estimatedMax()); else ps.setNull(14, java.sql.Types.REAL); + if (lot.nextBidStepInCents() != null) ps.setLong(15, lot.nextBidStepInCents()); else ps.setNull(15, java.sql.Types.BIGINT); + ps.setString(16, lot.condition()); + ps.setString(17, lot.categoryPath()); + ps.setString(18, lot.cityLocation()); + ps.setString(19, lot.countryCode()); + ps.setString(20, lot.biddingStatus()); + ps.setString(21, lot.appearance()); + ps.setString(22, lot.packaging()); + if (lot.quantity() != null) ps.setLong(23, lot.quantity()); else ps.setNull(23, java.sql.Types.BIGINT); + if (lot.vat() != null) ps.setDouble(24, lot.vat()); else ps.setNull(24, java.sql.Types.REAL); + if (lot.buyerPremiumPercentage() != null) ps.setDouble(25, lot.buyerPremiumPercentage()); else ps.setNull(25, java.sql.Types.REAL); + ps.setString(26, lot.remarks()); + if (lot.startingBid() != null) ps.setDouble(27, lot.startingBid()); else ps.setNull(27, java.sql.Types.REAL); + if (lot.reservePrice() != null) ps.setDouble(28, lot.reservePrice()); else ps.setNull(28, java.sql.Types.REAL); + if (lot.reserveMet() != null) ps.setInt(29, lot.reserveMet() ? 1 : 0); else ps.setNull(29, java.sql.Types.INTEGER); + if (lot.bidIncrement() != null) ps.setDouble(30, lot.bidIncrement()); else ps.setNull(30, java.sql.Types.REAL); + if (lot.viewCount() != null) ps.setInt(31, lot.viewCount()); else ps.setNull(31, java.sql.Types.INTEGER); + ps.setString(32, lot.firstBidTime() != null ? lot.firstBidTime().toString() : null); + ps.setString(33, lot.lastBidTime() != null ? lot.lastBidTime().toString() : null); + if (lot.bidVelocity() != null) ps.setDouble(34, lot.bidVelocity()); else ps.setNull(34, java.sql.Types.REAL); + + ps.setLong(35, lot.lotId()); + + int updated = ps.executeUpdate(); + if (updated == 0) { + log.warn("Failed to update lot {} - lot not found in database", lot.lotId()); + } + } + } + /** * Inserts a complete image record (for testing/legacy compatibility). * In production, scraper inserts with local_path, monitor updates labels via updateImageLabels. diff --git a/src/main/java/auctiora/Lot.java b/src/main/java/auctiora/Lot.java index 1fc07c6..fe9c423 100644 --- a/src/main/java/auctiora/Lot.java +++ b/src/main/java/auctiora/Lot.java @@ -11,6 +11,7 @@ import java.time.LocalDateTime; record Lot( long saleId, long lotId, + String displayId, // Full lot ID string (e.g., "A1-34732-49") for GraphQL queries String title, String description, String manufacturer, @@ -140,7 +141,7 @@ record Lot( double currentBid, String currency, String url, LocalDateTime closingTime, boolean closingNotified) { return new Lot( - saleId, lotId, title, description, manufacturer, type, year, category, + saleId, lotId, null, title, description, manufacturer, type, year, category, currentBid, currency, url, closingTime, closingNotified, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, diff --git a/src/main/java/auctiora/LotEnrichmentService.java b/src/main/java/auctiora/LotEnrichmentService.java index 51e8139..a813702 100644 --- a/src/main/java/auctiora/LotEnrichmentService.java +++ b/src/main/java/auctiora/LotEnrichmentService.java @@ -27,16 +27,21 @@ public class LotEnrichmentService { * Enriches a single lot with GraphQL intelligence data */ public boolean enrichLot(Lot lot) { + if (lot.displayId() == null || lot.displayId().isBlank()) { + log.debug("Cannot enrich lot {} - missing displayId", lot.lotId()); + return false; + } + try { - var intelligence = graphQLClient.fetchLotIntelligence(lot.lotId()); + var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId()); if (intelligence == null) { - log.debug("No intelligence data for lot {}", lot.lotId()); + log.debug("No intelligence data for lot {}", lot.displayId()); return false; } // Merge intelligence with existing lot data var enrichedLot = mergeLotWithIntelligence(lot, intelligence); - db.upsertLot(enrichedLot); + db.upsertLotWithIntelligence(enrichedLot); log.debug("Enriched lot {} with GraphQL data", lot.lotId()); return true; @@ -48,7 +53,7 @@ public class LotEnrichmentService { } /** - * Enriches multiple lots in batch (more efficient) + * Enriches multiple lots sequentially * @param lots List of lots to enrich * @return Number of successfully enriched lots */ @@ -57,47 +62,33 @@ public class LotEnrichmentService { return 0; } - try { - List lotIds = lots.stream() - .map(Lot::lotId) - .collect(Collectors.toList()); + log.info("Enriching {} lots via GraphQL", lots.size()); + int enrichedCount = 0; - log.info("Fetching intelligence for {} lots via GraphQL batch query", lotIds.size()); - var intelligenceList = graphQLClient.fetchBatchLotIntelligence(lotIds); - - if (intelligenceList.isEmpty()) { - log.warn("No intelligence data returned for batch of {} lots", lotIds.size()); - return 0; + for (var lot : lots) { + if (lot.displayId() == null || lot.displayId().isBlank()) { + log.debug("Skipping lot {} - missing displayId", lot.lotId()); + continue; } - // Create map for fast lookup - var intelligenceMap = intelligenceList.stream() - .collect(Collectors.toMap( - LotIntelligence::lotId, - intel -> intel - )); - - int enrichedCount = 0; - for (var lot : lots) { - var intelligence = intelligenceMap.get(lot.lotId()); + try { + var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId()); if (intelligence != null) { - try { - var enrichedLot = mergeLotWithIntelligence(lot, intelligence); - db.upsertLot(enrichedLot); - enrichedCount++; - } catch (SQLException e) { - log.warn("Failed to update lot {}: {}", lot.lotId(), e.getMessage()); - } + var enrichedLot = mergeLotWithIntelligence(lot, intelligence); + db.upsertLotWithIntelligence(enrichedLot); + enrichedCount++; + } else { + log.debug("No intelligence data for lot {}", lot.displayId()); } + } catch (Exception e) { + log.warn("Failed to enrich lot {}: {}", lot.displayId(), e.getMessage()); } - log.info("Successfully enriched {}/{} lots", enrichedCount, lots.size()); - return enrichedCount; - - } catch (Exception e) { - log.error("Failed to enrich lots batch: {}", e.getMessage()); - return 0; + // Small delay to respect rate limits (handled by RateLimitedHttpClient) } + + log.info("Successfully enriched {}/{} lots", enrichedCount, lots.size()); + return enrichedCount; } /** @@ -169,6 +160,7 @@ public class LotEnrichmentService { return new Lot( lot.saleId(), lot.lotId(), + lot.displayId(), // Preserve displayId lot.title(), lot.description(), lot.manufacturer(), diff --git a/src/main/java/auctiora/ScraperDataAdapter.java b/src/main/java/auctiora/ScraperDataAdapter.java index 7dd17bc..930a436 100644 --- a/src/main/java/auctiora/ScraperDataAdapter.java +++ b/src/main/java/auctiora/ScraperDataAdapter.java @@ -47,18 +47,20 @@ public class ScraperDataAdapter { } public static Lot fromScraperLot(ResultSet rs) throws SQLException { - var lotId = extractNumericId(rs.getString("lot_id")); - var saleId = extractNumericId(rs.getString("auction_id")); - + var lotIdStr = rs.getString("lot_id"); // Full display ID (e.g., "A1-34732-49") + var lotId = extractNumericId(lotIdStr); + var saleId = extractNumericId(rs.getString("auction_id")); + var bidStr = getStringOrNull(rs, "current_bid"); var bid = parseBidAmount(bidStr); var currency = parseBidCurrency(bidStr); - + var closing = parseTimestamp(getStringOrNull(rs, "closing_time")); - + return new Lot( saleId, lotId, + lotIdStr, // Store full displayId for GraphQL queries rs.getString("title"), getStringOrDefault(rs, "description", ""), "", "", 0, diff --git a/src/main/java/auctiora/TroostwijkGraphQLClient.java b/src/main/java/auctiora/TroostwijkGraphQLClient.java index 59e8c0f..fe53bcd 100644 --- a/src/main/java/auctiora/TroostwijkGraphQLClient.java +++ b/src/main/java/auctiora/TroostwijkGraphQLClient.java @@ -33,13 +33,19 @@ public class TroostwijkGraphQLClient { /** * Fetches enriched lot data from GraphQL API - * @param lotId The lot ID to fetch + * @param displayId The lot display ID (e.g., "A1-34732-49") + * @param lotId The numeric lot ID for mapping back to database * @return LotIntelligence with enriched fields, or null if failed */ - public LotIntelligence fetchLotIntelligence(long lotId) { + public LotIntelligence fetchLotIntelligence(String displayId, long lotId) { + if (displayId == null || displayId.isBlank()) { + log.debug("Cannot fetch intelligence for null/blank displayId"); + return null; + } + try { - String query = buildLotQuery(lotId); - String variables = buildVariables(lotId); + String query = buildLotQuery(); + String variables = buildVariables(displayId); // Proper GraphQL request format with query and variables String requestBody = String.format( @@ -61,11 +67,11 @@ public class TroostwijkGraphQLClient { ); if (response == null || response.body() == null) { - log.debug("No response from GraphQL for lot {}", lotId); + log.debug("No response from GraphQL for lot {}", displayId); return null; } - log.debug("GraphQL response for lot {}: {}", lotId, response.body().substring(0, Math.min(200, response.body().length()))); + log.debug("GraphQL response for lot {}: {}", displayId, response.body().substring(0, Math.min(200, response.body().length()))); return parseLotIntelligence(response.body(), lotId); } catch (Exception e) { @@ -114,12 +120,12 @@ public class TroostwijkGraphQLClient { return results; } - private String buildLotQuery(long lotId) { - // TBAuctions API uses lot objects with specific fields - // Note: This query structure needs to be adjusted based on actual TBAuctions schema + private String buildLotQuery() { + // Match Python scraper's LOT_BIDDING_QUERY structure + // Uses lotDetails(displayId:...) instead of lot(id:...) return """ - query GetLot($lotId: String!, $locale: String!, $platform: Platform!) { - lot(id: $lotId, locale: $locale, platform: $platform) { + query LotBiddingData($lotDisplayId: String!, $locale: String!, $platform: Platform!) { + lotDetails(displayId: $lotDisplayId, locale: $locale, platform: $platform) { id displayId followersCount @@ -128,21 +134,31 @@ public class TroostwijkGraphQLClient { condition description biddingStatus - buyerPremium + buyersPremium viewCount + estimatedValueInCentsMin + estimatedValueInCentsMax + categoryPath + location { + city + country + } + biddingStatistics { + numberOfBids + } } } """.replaceAll("\\s+", " "); } - private String buildVariables(long lotId) { + private String buildVariables(String displayId) { return String.format(""" { - "lotId": "%d", + "lotDisplayId": "%s", "locale": "%s", "platform": "%s" } - """, lotId, LOCALE, PLATFORM).replaceAll("\\s+", " "); + """, displayId, LOCALE, PLATFORM).replaceAll("\\s+", " "); } private String buildBatchLotQuery(List lotIds) { @@ -182,37 +198,53 @@ public class TroostwijkGraphQLClient { } JsonNode root = objectMapper.readTree(json); - JsonNode lotNode = root.path("data").path("lot"); + JsonNode lotNode = root.path("data").path("lotDetails"); if (lotNode.isMissingNode()) { + log.debug("No lotDetails in GraphQL response"); return null; } + // Extract location from nested object + JsonNode locationNode = lotNode.path("location"); + String city = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "city"); + String countryCode = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "country"); + + // Extract bids count from nested biddingStatistics + JsonNode statsNode = lotNode.path("biddingStatistics"); + Integer bidsCount = statsNode.isMissingNode() ? null : getIntOrNull(statsNode, "numberOfBids"); + + // Convert cents to euros for estimates + Long estimatedMinCents = getLongOrNull(lotNode, "estimatedValueInCentsMin"); + Long estimatedMaxCents = getLongOrNull(lotNode, "estimatedValueInCentsMax"); + Double estimatedMin = estimatedMinCents != null ? estimatedMinCents.doubleValue() : null; + Double estimatedMax = estimatedMaxCents != null ? estimatedMaxCents.doubleValue() : null; + return new LotIntelligence( lotId, getIntOrNull(lotNode, "followersCount"), - getDoubleOrNull(lotNode, "estimatedMin"), - getDoubleOrNull(lotNode, "estimatedMax"), + estimatedMin, + estimatedMax, getLongOrNull(lotNode, "nextBidStepInCents"), getStringOrNull(lotNode, "condition"), getStringOrNull(lotNode, "categoryPath"), - getStringOrNull(lotNode, "city"), - getStringOrNull(lotNode, "countryCode"), + city, + countryCode, getStringOrNull(lotNode, "biddingStatus"), - getStringOrNull(lotNode, "appearance"), - getStringOrNull(lotNode, "packaging"), - getLongOrNull(lotNode, "quantity"), - getDoubleOrNull(lotNode, "vat"), - getDoubleOrNull(lotNode, "buyerPremiumPercentage"), - getStringOrNull(lotNode, "remarks"), - getDoubleOrNull(lotNode, "startingBid"), - getDoubleOrNull(lotNode, "reservePrice"), - getBooleanOrNull(lotNode, "reserveMet"), - getDoubleOrNull(lotNode, "bidIncrement"), + null, // appearance - not in API response + null, // packaging - not in API response + null, // quantity - not in API response + null, // vat - not in API response + null, // buyerPremiumPercentage - could extract from buyersPremium + null, // remarks - not in API response + null, // startingBid - not in API response + null, // reservePrice - not in API response + null, // reserveMet - not in API response + null, // bidIncrement - not in API response getIntOrNull(lotNode, "viewCount"), - parseDateTime(getStringOrNull(lotNode, "firstBidTime")), - parseDateTime(getStringOrNull(lotNode, "lastBidTime")), - calculateBidVelocity(lotNode) + null, // firstBidTime - not in API response + null, // lastBidTime - not in API response + null // bidVelocity - could calculate from bidsCount if we had timing data ); } catch (Exception e) {