Tour/auctiora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix mock tests

Browse Source
This commit is contained in:
Tour
2025-12-04 19:38:38 +01:00
parent 9857f053a1
commit ed74bb5e93
34 changed files with 2312 additions and 2006 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
.idea/compiler.xml generated
View File

@@ -2,17 +2,37 @@
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile default="true" name="Default" enabled="true" />
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="troostwijk-scraper" />
</profile>
<profile name="Annotation profile for Troostwijk Auction Scraper" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<processorPath useClasspath="false">
<entry name="$MAVEN_REPOSITORY$/org/projectlombok/lombok/1.18.40/lombok-1.18.40.jar" />
<entry name="$MAVEN_REPOSITORY$/io/quarkus/quarkus-extension-processor/3.17.7/quarkus-extension-processor-3.17.7.jar" />
<entry name="$MAVEN_REPOSITORY$/org/jboss/jdeparser/jdeparser/2.0.3.Final/jdeparser-2.0.3.Final.jar" />
<entry name="$MAVEN_REPOSITORY$/org/jsoup/jsoup/1.15.3/jsoup-1.15.3.jar" />
<entry name="$MAVEN_REPOSITORY$/com/github/javaparser/javaparser-core/3.26.2/javaparser-core-3.26.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-databind/2.18.2/jackson-databind-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-annotations/2.18.2/jackson-annotations-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-core/2.18.2/jackson-core-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/dataformat/jackson-dataformat-yaml/2.18.2/jackson-dataformat-yaml-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/org/yaml/snakeyaml/2.3/snakeyaml-2.3.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/module/jackson-module-parameter-names/2.18.2/jackson-module-parameter-names-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/io/quarkus/quarkus-bootstrap-app-model/3.17.7/quarkus-bootstrap-app-model-3.17.7.jar" />
</processorPath>
<module name="auctiora" />
</profile>
</annotationProcessing>
</component>
<component name="JavacSettings">
<option name="ADDITIONAL_OPTIONS_OVERRIDE">
<module name="troostwijk-scraper" options="" />
<module name="auctiora" options="-Xdiags:verbose -Xlint:all -parameters" />
</option>
</component>
</project>

5
.idea/jarRepositories.xml generated
View File

@@ -1,6 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="RemoteRepositoriesConfiguration">
<remote-repository>
<option name="id" value="jitpack.io" />
<option name="name" value="jitpack.io" />
<option name="url" value="https://jitpack.io" />
</remote-repository>
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Apache Central Repository" />

67
Dockerfile
View File

@@ -1,52 +1,27 @@
# Multi-stage Dockerfile for Quarkus Auction Monitor
# Build stage
FROM maven:3.9-eclipse-temurin-25-alpine AS build
# ==================== BUILD STAGE ====================
FROM maven:3.9-eclipse-temurin-25-alpine AS builder
WORKDIR /app
# Copy POM first (allows for cached dependency layer)
COPY pom.xml .
# This will now work if the opencv dependency has no classifier
# -----LOCAL----
RUN mvn dependency:resolve -B
# -----LOCAL----
# RUN mvn dependency:go-offline -B
# Copy Maven files for dependency caching
COPY pom.xml ./
RUN mvn dependency:go-offline -B
# Copy source code
COPY src/ ./src/
# Build Quarkus application (fast-jar for production)
RUN mvn package -DskipTests -Dquarkus.package.jar.type=fast-jar
# Runtime stage
FROM eclipse-temurin:25-jre-alpine
COPY src ./src
# Updated with both properties to avoid the warning
RUN mvn package -DskipTests -Dquarkus.package.jar.type=uber-jar -Dquarkus.package.jar.enabled=true
# ==================== RUNTIME STAGE ====================
FROM eclipse-temurin:25-jre
WORKDIR /app
# Create non-root user
RUN addgroup -g 1001 quarkus && \
adduser -u 1001 -G quarkus -s /bin/sh -D quarkus
# Create directories for data
RUN mkdir -p /mnt/okcomputer/output/images && \
chown -R quarkus:quarkus /mnt/okcomputer
# Copy Quarkus fast-jar structure
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/lib/ /app/lib/
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/*.jar /app/
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/app/ /app/app/
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/quarkus/ /app/quarkus/
# Switch to non-root user
RUN groupadd -r quarkus && useradd -r -g quarkus quarkusa
COPY --from=builder --chown=quarkus:quarkus /app/target/scrape-ui-*.jar app.jar
USER quarkus
# Expose ports
EXPOSE 8081
# Set environment variables
ENV JAVA_OPTS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager"
ENV JAVA_APP_JAR="/app/quarkus-run.jar"
# Health check
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD wget --no-verbose --tries=1 --spider http://localhost:8081/health/live || exit 1
# Run the Quarkus application
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar $JAVA_APP_JAR"]
ENTRYPOINT ["java", \
"-Dio.netty.tryReflectionSetAccessible=true", \
"--enable-native-access=ALL-UNNAMED", \
"--sun-misc-unsafe-memory-access=allow", \
"-jar", "app.jar"]

11
auction.iml
View File

@@ -1,11 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

11
docker-compose.yml
View File

@@ -1,11 +1,8 @@
version: '3.8'
services:
auction-monitor:
sophena:
build:
context: .
dockerfile: Dockerfile
container_name: troostwijk-auction-monitor
dockerfile: .
container_name: sophena
ports:
- "8081:8081"
volumes:
@@ -39,6 +36,7 @@ services:
- AUCTION_WORKFLOW_IMAGE_PROCESSING_CRON=0 0 * * * ?
- AUCTION_WORKFLOW_BID_MONITORING_CRON=0 */15 * * * ?
- AUCTION_WORKFLOW_CLOSING_ALERTS_CRON=0 */5 * * * ?
- JAVA_TOOL_OPTIONS=-Dio.netty.tryReflectionSetAccessible=true --enable-native-access=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8081/health/live"]
@@ -52,6 +50,7 @@ services:
networks:
- auction-network
networks:
auction-network:
driver: bridge

204
pom.xml
View File

@@ -11,16 +11,39 @@
<name>Troostwijk Auction Scraper</name>
<description>Web scraper for Troostwijk Auctions with object detection and notifications</description>
<repositories>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
</repositories>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>25</maven.compiler.source>
<maven.compiler.target>25</maven.compiler.target>
<maven.compiler.release>25</maven.compiler.release>
<jackson.version>2.17.0</jackson.version>
<opencv.version>4.9.0-0</opencv.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<quarkus.platform.version>3.17.7</quarkus.platform.version>
<asm.version>9.8</asm.version>
<lombok.version>1.18.40</lombok.version>
<!--this is not a bug, its feature-->
<lombok-version>${lombok.version}</lombok-version>
<lombok-maven-version>1.18.20.0</lombok-maven-version>
<maven-compiler-plugin-version>3.14.0</maven-compiler-plugin-version>
<versions-maven-plugin.version>2.19.0</versions-maven-plugin.version>
<jandex-maven-plugin-version>3.5.0</jandex-maven-plugin-version>
<maven.compiler.args>
--enable-native-access=ALL-UNNAMED
--add-opens java.base/sun.misc=ALL-UNNAMED
-Xdiags:verbose
-Xlint:all
</maven.compiler.args>
<uberJar>true</uberJar> <!-- Your existing properties... -->
<quarkus.package.jar.type>uber-jar</quarkus.package.jar.type>
<quarkus.package.jar.enabled>true</quarkus.package.jar.enabled>
<maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
</properties>
<dependencyManagement>
@@ -53,10 +76,28 @@
<artifactId>asm-util</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>4.1.124.Final</version> <!-- This version has the fix -->
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.github.vladimir-bukhtoyarov.bucket4j</groupId>
<artifactId>bucket4j-core</artifactId>
<version>7.6.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.github.bucket4j/bucket4j -->
<!--<dependency>
<groupId>io.github.bucket4j</groupId>
<artifactId>bucket4j</artifactId>
<version>8.9.0</version>
</dependency>-->
<!-- JSoup for HTML parsing and HTTP client -->
<dependency>
<groupId>org.jsoup</groupId>
@@ -77,7 +118,19 @@
<artifactId>sqlite-jdbc</artifactId>
<version>3.45.1.0</version>
</dependency>
<!-- Lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven</artifactId>
<version>${lombok-maven-version}</version>
<type>pom</type>
</dependency>
<!-- JavaMail API for email notifications -->
<dependency>
<groupId>com.sun.mail</groupId>
@@ -85,12 +138,6 @@
<version>1.6.2</version>
</dependency>
<!-- OpenCV for image processing and object detection -->
<dependency>
<groupId>org.openpnp</groupId>
<artifactId>opencv</artifactId>
<version>${opencv.version}</version>
</dependency>
<dependency>
<groupId>com.microsoft.playwright</groupId>
<artifactId>playwright</artifactId>
@@ -167,6 +214,14 @@
<artifactId>quarkus-config-yaml</artifactId>
</dependency>
<!-- OSGi annotations -->
<dependency>
<groupId>org.osgi</groupId>
<artifactId>org.osgi.annotation.bundle</artifactId>
<version>2.0.0</version>
<scope>provided</scope>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>io.quarkus</groupId>
@@ -178,10 +233,23 @@
<artifactId>rest-assured</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.openpnp</groupId>
<artifactId>opencv</artifactId>
<version>4.9.0-0</version>
<!--<classifier>windows-x86_64</classifier>-->
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>io.quarkus.platform</groupId>
<artifactId>quarkus-maven-plugin</artifactId>
@@ -196,46 +264,74 @@
</goals>
</execution>
</executions>
</plugin>
<!-- Maven Compiler Plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.11.0</version>
<configuration>
<source>25</source>
<target>25</target>
<properties>
<build.timestamp>${maven.build.timestamp}</build.timestamp>
</properties>
</configuration>
</plugin>
<!-- Maven Exec Plugin for running with native access -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<mainClass>com.auction.TroostwijkAuctionExtractor</mainClass>
<cleanupDaemonThreads>false</cleanupDaemonThreads>
<arguments>
<argument>--max-visits</argument>
<argument>3</argument>
</arguments>
<additionalClasspathElements>
<!--suppress MavenModelInspection -->
<additionalClasspathElement>${project.build.outputDirectory}</additionalClasspathElement>
</additionalClasspathElements>
<commandlineArgs>--enable-native-access=ALL-UNNAMED</commandlineArgs>
</configuration>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven-plugin</artifactId>
<version>${lombok-maven-version}</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>java</goal>
<goal>delombok</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Maven Exec Plugin for running with native access -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin-version}</version>
<configuration>
<release>${maven.compiler.release}</release>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok-version}</version>
</path>
<path>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-extension-processor</artifactId>
<version>${quarkus.platform.version}</version>
</path>
</annotationProcessorPaths>
<compilerArgs>
<arg>-Xdiags:verbose</arg>
<arg>-Xlint:all</arg>
<arg>-parameters</arg>
</compilerArgs>
<fork>true</fork>
<excludes>
<exclude>module-info.java</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
<version>${versions-maven-plugin.version}</version>
</plugin>
<!-- Maven Surefire Plugin for tests with native access -->
<plugin>
<groupId>io.smallrye</groupId>
<artifactId>jandex-maven-plugin</artifactId>
<version>${jandex-maven-plugin-version}</version>
<executions>
<execution>
<id>make-index</id>
<goals>
<goal>jandex</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
@@ -262,31 +358,17 @@
</archive>
</configuration>
</plugin>-->
<!-- Maven Assembly Plugin for creating executable JAR with dependencies -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<archive>
<manifest>
<mainClass>com.auction.Main</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<!-- In your pom.xml, alongside <build> and <dependencies> -->
<distributionManagement>
<repository>
<id>gitea</id>
<url>https://git.appmodel.nl/api/packages/Tour/maven</url>
</repository>
<snapshotRepository>
<id>gitea</id>
<url>https://git.appmodel.nl/api/packages/Tour/maven</url>
</snapshotRepository>
</distributionManagement>
</project>

20
src/main/java/auctiora/AuctionInfo.java
View File

@@ -7,13 +7,13 @@ import java.time.LocalDateTime;
* Data typically populated by the external scraper process
*/
public record AuctionInfo(
int auctionId, // Unique auction ID (from URL)
String title, // Auction title
String location, // Location (e.g., "Amsterdam, NL")
String city, // City name
String country, // Country code (e.g., "NL")
String url, // Full auction URL
String type, // Auction type (A1 or A7)
int lotCount, // Number of lots/kavels
LocalDateTime closingTime // Closing time if available
) {}
int auctionId, // Unique auction ID (from URL)
String title, // Auction title
String location, // Location (e.g., "Amsterdam, NL")
String city, // City name
String country, // Country code (e.g., "NL")
String url, // Full auction URL
String typePrefix, // Auction type (A1 or A7)
int lotCount, // Number of lots/kavels
LocalDateTime firstLotClosingTime // Closing time if available
) { }

4
src/main/java/auctiora/AuctionMonitorProducer.java
View File

@@ -35,7 +35,7 @@ public class AuctionMonitorProducer {
@ConfigProperty(name = "auction.notification.config") String config) {
LOG.infof("Initializing NotificationService with config: %s", config);
return new NotificationService(config, "");
return new NotificationService(config);
}
@Produces
@@ -54,7 +54,7 @@ public class AuctionMonitorProducer {
public ImageProcessingService produceImageProcessingService(
DatabaseService db,
ObjectDetectionService detector,
RateLimitedHttpClient httpClient) {
RateLimitedHttpClient2 httpClient) {
LOG.infof("Initializing ImageProcessingService");
return new ImageProcessingService(db, detector, httpClient);

2
src/main/java/auctiora/AuctionMonitorResource.java
View File

@@ -33,7 +33,7 @@ public class AuctionMonitorResource {
NotificationService notifier;
@Inject
RateLimitedHttpClient httpClient;
RateLimitedHttpClient2 httpClient;
/**
* GET /api/monitor/status

10
src/main/java/auctiora/Console.java
View File

@@ -1,10 +0,0 @@
package auctiora;
/**
* Simple console output utility (renamed from IO to avoid Java 25 conflict)
*/
class Console {
static void println(String message) {
System.out.println(message);
}
}

701
src/main/java/auctiora/DatabaseService.java
View File

@@ -1,5 +1,7 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.io.Console;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.time.Instant;
@@ -12,120 +14,121 @@ import java.util.List;
* Data is typically populated by an external scraper process;
* this service enriches it with image processing and monitoring.
*/
@Slf4j
public class DatabaseService {
private final String url;
private final String url;
DatabaseService(String dbPath) {
this.url = "jdbc:sqlite:" + dbPath;
}
DatabaseService(String dbPath) {
this.url = "jdbc:sqlite:" + dbPath;
}
/**
* Creates tables if they do not already exist.
* Schema supports data from external scraper and adds image processing results.
*/
void ensureSchema() throws SQLException {
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
// Auctions table (populated by external scraper)
stmt.execute("""
CREATE TABLE IF NOT EXISTS auctions (
auction_id INTEGER PRIMARY KEY,
title TEXT NOT NULL,
location TEXT,
city TEXT,
country TEXT,
url TEXT NOT NULL,
type TEXT,
lot_count INTEGER DEFAULT 0,
closing_time TEXT,
discovered_at INTEGER
)""");
/**
* Creates tables if they do not already exist.
* Schema supports data from external scraper and adds image processing results.
*/
void ensureSchema() throws SQLException {
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
// Auctions table (populated by external scraper)
stmt.execute("""
CREATE TABLE IF NOT EXISTS auctions (
auction_id INTEGER PRIMARY KEY,
title TEXT NOT NULL,
location TEXT,
city TEXT,
country TEXT,
url TEXT NOT NULL,
type TEXT,
lot_count INTEGER DEFAULT 0,
closing_time TEXT,
discovered_at INTEGER
)""");
// Lots table (populated by external scraper)
stmt.execute("""
CREATE TABLE IF NOT EXISTS lots (
lot_id INTEGER PRIMARY KEY,
sale_id INTEGER,
title TEXT,
description TEXT,
manufacturer TEXT,
type TEXT,
year INTEGER,
category TEXT,
current_bid REAL,
currency TEXT,
url TEXT,
closing_time TEXT,
closing_notified INTEGER DEFAULT 0,
FOREIGN KEY (sale_id) REFERENCES auctions(auction_id)
)""");
// Lots table (populated by external scraper)
stmt.execute("""
CREATE TABLE IF NOT EXISTS lots (
lot_id INTEGER PRIMARY KEY,
sale_id INTEGER,
title TEXT,
description TEXT,
manufacturer TEXT,
type TEXT,
year INTEGER,
category TEXT,
current_bid REAL,
currency TEXT,
url TEXT,
closing_time TEXT,
closing_notified INTEGER DEFAULT 0,
FOREIGN KEY (sale_id) REFERENCES auctions(auction_id)
)""");
// Images table (populated by this process)
stmt.execute("""
CREATE TABLE IF NOT EXISTS images (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id INTEGER,
url TEXT,
file_path TEXT,
labels TEXT,
processed_at INTEGER,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)""");
// Images table (populated by this process)
stmt.execute("""
CREATE TABLE IF NOT EXISTS images (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id INTEGER,
url TEXT,
file_path TEXT,
labels TEXT,
processed_at INTEGER,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)""");
// Indexes for performance
stmt.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id)");
}
}
// Indexes for performance
stmt.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)");
stmt.execute("CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id)");
}
}
/**
* Inserts or updates an auction record (typically called by external scraper)
*/
synchronized void upsertAuction(AuctionInfo auction) throws SQLException {
var sql = """
INSERT INTO auctions (auction_id, title, location, city, country, url, type, lot_count, closing_time, discovered_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(auction_id) DO UPDATE SET
title = excluded.title,
location = excluded.location,
city = excluded.city,
country = excluded.country,
url = excluded.url,
type = excluded.type,
lot_count = excluded.lot_count,
closing_time = excluded.closing_time
""";
/**
* Inserts or updates an auction record (typically called by external scraper)
*/
synchronized void upsertAuction(AuctionInfo auction) throws SQLException {
var sql = """
INSERT INTO auctions (auction_id, title, location, city, country, url, type, lot_count, closing_time, discovered_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(auction_id) DO UPDATE SET
title = excluded.title,
location = excluded.location,
city = excluded.city,
country = excluded.country,
url = excluded.url,
type = excluded.type,
lot_count = excluded.lot_count,
closing_time = excluded.closing_time
""";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, auction.auctionId());
ps.setString(2, auction.title());
ps.setString(3, auction.location());
ps.setString(4, auction.city());
ps.setString(5, auction.country());
ps.setString(6, auction.url());
ps.setString(7, auction.type());
ps.setInt(8, auction.lotCount());
ps.setString(9, auction.closingTime() != null ? auction.closingTime().toString() : null);
ps.setLong(10, Instant.now().getEpochSecond());
ps.executeUpdate();
}
}
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, auction.auctionId());
ps.setString(2, auction.title());
ps.setString(3, auction.location());
ps.setString(4, auction.city());
ps.setString(5, auction.country());
ps.setString(6, auction.url());
ps.setString(7, auction.typePrefix());
ps.setInt(8, auction.lotCount());
ps.setString(9, auction.firstLotClosingTime() != null ? auction.firstLotClosingTime().toString() : null);
ps.setLong(10, Instant.now().getEpochSecond());
ps.executeUpdate();
}
}
/**
* Retrieves all auctions from the database
*/
synchronized List<AuctionInfo> getAllAuctions() throws SQLException {
List<AuctionInfo> auctions = new ArrayList<>();
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time FROM auctions";
/**
* Retrieves all auctions from the database
*/
synchronized List<AuctionInfo> getAllAuctions() throws SQLException {
List<AuctionInfo> auctions = new ArrayList<>();
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time FROM auctions";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
var closingStr = rs.getString("closing_time");
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
var closingStr = rs.getString("closing_time");
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
auctions.add(new AuctionInfo(
auctions.add(new AuctionInfo(
rs.getInt("auction_id"),
rs.getString("title"),
rs.getString("location"),
@@ -135,28 +138,28 @@ public class DatabaseService {
rs.getString("type"),
rs.getInt("lot_count"),
closing
));
}
}
return auctions;
}
));
}
}
return auctions;
}
/**
* Retrieves auctions by country code
*/
synchronized List<AuctionInfo> getAuctionsByCountry(String countryCode) throws SQLException {
List<AuctionInfo> auctions = new ArrayList<>();
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time "
/**
* Retrieves auctions by country code
*/
synchronized List<AuctionInfo> getAuctionsByCountry(String countryCode) throws SQLException {
List<AuctionInfo> auctions = new ArrayList<>();
var sql = "SELECT auction_id, title, location, city, country, url, type, lot_count, closing_time "
+ "FROM auctions WHERE country = ?";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setString(1, countryCode);
var rs = ps.executeQuery();
while (rs.next()) {
var closingStr = rs.getString("closing_time");
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setString(1, countryCode);
var rs = ps.executeQuery();
while (rs.next()) {
var closingStr = rs.getString("closing_time");
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
auctions.add(new AuctionInfo(
auctions.add(new AuctionInfo(
rs.getInt("auction_id"),
rs.getString("title"),
rs.getString("location"),
@@ -166,104 +169,104 @@ public class DatabaseService {
rs.getString("type"),
rs.getInt("lot_count"),
closing
));
}
}
return auctions;
}
));
}
}
return auctions;
}
/**
* Inserts or updates a lot record (typically called by external scraper)
*/
synchronized void upsertLot(Lot lot) throws SQLException {
var sql = """
INSERT INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(lot_id) DO UPDATE SET
sale_id = excluded.sale_id,
title = excluded.title,
description = excluded.description,
manufacturer = excluded.manufacturer,
type = excluded.type,
year = excluded.year,
category = excluded.category,
current_bid = excluded.current_bid,
currency = excluded.currency,
url = excluded.url,
closing_time = excluded.closing_time
""";
/**
* Inserts or updates a lot record (typically called by external scraper)
*/
synchronized void upsertLot(Lot lot) throws SQLException {
var sql = """
INSERT INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(lot_id) DO UPDATE SET
sale_id = excluded.sale_id,
title = excluded.title,
description = excluded.description,
manufacturer = excluded.manufacturer,
type = excluded.type,
year = excluded.year,
category = excluded.category,
current_bid = excluded.current_bid,
currency = excluded.currency,
url = excluded.url,
closing_time = excluded.closing_time
""";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, lot.lotId());
ps.setInt(2, lot.saleId());
ps.setString(3, lot.title());
ps.setString(4, lot.description());
ps.setString(5, lot.manufacturer());
ps.setString(6, lot.type());
ps.setInt(7, lot.year());
ps.setString(8, lot.category());
ps.setDouble(9, lot.currentBid());
ps.setString(10, lot.currency());
ps.setString(11, lot.url());
ps.setString(12, lot.closingTime() != null ? lot.closingTime().toString() : null);
ps.setInt(13, lot.closingNotified() ? 1 : 0);
ps.executeUpdate();
}
}
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, lot.lotId());
ps.setInt(2, lot.saleId());
ps.setString(3, lot.title());
ps.setString(4, lot.description());
ps.setString(5, lot.manufacturer());
ps.setString(6, lot.type());
ps.setInt(7, lot.year());
ps.setString(8, lot.category());
ps.setDouble(9, lot.currentBid());
ps.setString(10, lot.currency());
ps.setString(11, lot.url());
ps.setString(12, lot.closingTime() != null ? lot.closingTime().toString() : null);
ps.setInt(13, lot.closingNotified() ? 1 : 0);
ps.executeUpdate();
}
}
/**
* Inserts a new image record with object detection labels
*/
synchronized void insertImage(int lotId, String url, String filePath, List<String> labels) throws SQLException {
var sql = "INSERT INTO images (lot_id, url, file_path, labels, processed_at) VALUES (?, ?, ?, ?, ?)";
try (var conn = DriverManager.getConnection(this.url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, lotId);
ps.setString(2, url);
ps.setString(3, filePath);
ps.setString(4, String.join(",", labels));
ps.setLong(5, Instant.now().getEpochSecond());
ps.executeUpdate();
}
}
/**
* Inserts a new image record with object detection labels
*/
synchronized void insertImage(int lotId, String url, String filePath, List<String> labels) throws SQLException {
var sql = "INSERT INTO images (lot_id, url, file_path, labels, processed_at) VALUES (?, ?, ?, ?, ?)";
try (var conn = DriverManager.getConnection(this.url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, lotId);
ps.setString(2, url);
ps.setString(3, filePath);
ps.setString(4, String.join(",", labels));
ps.setLong(5, Instant.now().getEpochSecond());
ps.executeUpdate();
}
}
/**
* Retrieves images for a specific lot
*/
synchronized List<ImageRecord> getImagesForLot(int lotId) throws SQLException {
List<ImageRecord> images = new ArrayList<>();
var sql = "SELECT id, lot_id, url, file_path, labels FROM images WHERE lot_id = ?";
/**
* Retrieves images for a specific lot
*/
synchronized List<ImageRecord> getImagesForLot(int lotId) throws SQLException {
List<ImageRecord> images = new ArrayList<>();
var sql = "SELECT id, lot_id, url, file_path, labels FROM images WHERE lot_id = ?";
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, lotId);
var rs = ps.executeQuery();
while (rs.next()) {
images.add(new ImageRecord(
try (var conn = DriverManager.getConnection(url); var ps = conn.prepareStatement(sql)) {
ps.setInt(1, lotId);
var rs = ps.executeQuery();
while (rs.next()) {
images.add(new ImageRecord(
rs.getInt("id"),
rs.getInt("lot_id"),
rs.getString("url"),
rs.getString("file_path"),
rs.getString("labels")
));
}
}
return images;
}
));
}
}
return images;
}
/**
* Retrieves all lots that are active and need monitoring
*/
synchronized List<Lot> getActiveLots() throws SQLException {
List<Lot> list = new ArrayList<>();
var sql = "SELECT lot_id, sale_id, title, description, manufacturer, type, year, category, " +
"current_bid, currency, url, closing_time, closing_notified FROM lots";
/**
* Retrieves all lots that are active and need monitoring
*/
synchronized List<Lot> getActiveLots() throws SQLException {
List<Lot> list = new ArrayList<>();
var sql = "SELECT lot_id, sale_id, title, description, manufacturer, type, year, category, " +
"current_bid, currency, url, closing_time, closing_notified FROM lots";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
var closingStr = rs.getString("closing_time");
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
var closingStr = rs.getString("closing_time");
var closing = closingStr != null ? LocalDateTime.parse(closingStr) : null;
list.add(new Lot(
list.add(new Lot(
rs.getInt("sale_id"),
rs.getInt("lot_id"),
rs.getString("title"),
@@ -277,163 +280,163 @@ public class DatabaseService {
rs.getString("url"),
closing,
rs.getInt("closing_notified") != 0
));
));
}
}
return list;
}
/**
* Retrieves all lots from the database
*/
synchronized List<Lot> getAllLots() throws SQLException {
return getActiveLots();
}
/**
* Gets the total number of images in the database
*/
synchronized int getImageCount() throws SQLException {
var sql = "SELECT COUNT(*) as count FROM images";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
if (rs.next()) {
return rs.getInt("count");
}
}
return 0;
}
/**
* Updates the current bid of a lot (used by monitoring service)
*/
synchronized void updateLotCurrentBid(Lot lot) throws SQLException {
try (var conn = DriverManager.getConnection(url);
var ps = conn.prepareStatement("UPDATE lots SET current_bid = ? WHERE lot_id = ?")) {
ps.setDouble(1, lot.currentBid());
ps.setInt(2, lot.lotId());
ps.executeUpdate();
}
}
/**
* Updates the closingNotified flag of a lot
*/
synchronized void updateLotNotificationFlags(Lot lot) throws SQLException {
try (var conn = DriverManager.getConnection(url);
var ps = conn.prepareStatement("UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) {
ps.setInt(1, lot.closingNotified() ? 1 : 0);
ps.setInt(2, lot.lotId());
ps.executeUpdate();
}
}
/**
* Imports auctions from scraper's schema format.
* Reads from scraper's tables and converts to monitor format using adapter.
*
* @return List of imported auctions
*/
synchronized List<AuctionInfo> importAuctionsFromScraper() throws SQLException {
List<AuctionInfo> imported = new ArrayList<>();
var sql = "SELECT auction_id, title, location, url, lots_count, first_lot_closing_time, scraped_at " +
"FROM auctions WHERE location LIKE '%NL%'";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
try {
var auction = ScraperDataAdapter.fromScraperAuction(rs);
upsertAuction(auction);
imported.add(auction);
} catch (Exception e) {
System.err.println("Failed to import auction: " + e.getMessage());
}
}
return list;
}
}
} catch (SQLException e) {
// Table might not exist in scraper format - that's ok
IO.println("ℹ️ Scraper auction table not found or incompatible schema");
}
/**
* Retrieves all lots from the database
*/
synchronized List<Lot> getAllLots() throws SQLException {
return getActiveLots();
}
return imported;
}
/**
* Gets the total number of images in the database
*/
synchronized int getImageCount() throws SQLException {
var sql = "SELECT COUNT(*) as count FROM images";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
if (rs.next()) {
return rs.getInt("count");
/**
* Imports lots from scraper's schema format.
* Reads from scraper's tables and converts to monitor format using adapter.
*
* @return List of imported lots
*/
synchronized List<Lot> importLotsFromScraper() throws SQLException {
List<Lot> imported = new ArrayList<>();
var sql = "SELECT lot_id, auction_id, title, description, category, " +
"current_bid, closing_time, url " +
"FROM lots";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
try {
var lot = ScraperDataAdapter.fromScraperLot(rs);
upsertLot(lot);
imported.add(lot);
} catch (Exception e) {
System.err.println("Failed to import lot: " + e.getMessage());
}
}
return 0;
}
}
} catch (SQLException e) {
// Table might not exist in scraper format - that's ok
log.info("ℹ️ Scraper lots table not found or incompatible schema");
}
/**
* Updates the current bid of a lot (used by monitoring service)
*/
synchronized void updateLotCurrentBid(Lot lot) throws SQLException {
try (var conn = DriverManager.getConnection(url);
var ps = conn.prepareStatement("UPDATE lots SET current_bid = ? WHERE lot_id = ?")) {
ps.setDouble(1, lot.currentBid());
ps.setInt(2, lot.lotId());
ps.executeUpdate();
}
}
return imported;
}
/**
* Updates the closingNotified flag of a lot
*/
synchronized void updateLotNotificationFlags(Lot lot) throws SQLException {
try (var conn = DriverManager.getConnection(url);
var ps = conn.prepareStatement("UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) {
ps.setInt(1, lot.closingNotified() ? 1 : 0);
ps.setInt(2, lot.lotId());
ps.executeUpdate();
}
}
/**
* Imports image URLs from scraper's schema.
* The scraper populates the images table with URLs but doesn't download them.
* This method retrieves undownloaded images for processing.
*
* @return List of image URLs that need to be downloaded
*/
synchronized List<ImageImportRecord> getUnprocessedImagesFromScraper() throws SQLException {
List<ImageImportRecord> images = new ArrayList<>();
var sql = """
SELECT i.lot_id, i.url, l.auction_id
FROM images i
LEFT JOIN lots l ON i.lot_id = l.lot_id
WHERE i.downloaded = 0 OR i.local_path IS NULL
""";
/**
* Imports auctions from scraper's schema format.
* Reads from scraper's tables and converts to monitor format using adapter.
*
* @return List of imported auctions
*/
synchronized List<AuctionInfo> importAuctionsFromScraper() throws SQLException {
List<AuctionInfo> imported = new ArrayList<>();
var sql = "SELECT auction_id, title, location, url, lots_count, first_lot_closing_time, scraped_at " +
"FROM auctions WHERE location LIKE '%NL%'";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
String lotIdStr = rs.getString("lot_id");
String auctionIdStr = rs.getString("auction_id");
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
try {
var auction = ScraperDataAdapter.fromScraperAuction(rs);
upsertAuction(auction);
imported.add(auction);
} catch (Exception e) {
System.err.println("Failed to import auction: " + e.getMessage());
}
}
} catch (SQLException e) {
// Table might not exist in scraper format - that's ok
IO.println("ℹ️ Scraper auction table not found or incompatible schema");
}
int lotId = ScraperDataAdapter.extractNumericId(lotIdStr);
int saleId = ScraperDataAdapter.extractNumericId(auctionIdStr);
return imported;
}
/**
* Imports lots from scraper's schema format.
* Reads from scraper's tables and converts to monitor format using adapter.
*
* @return List of imported lots
*/
synchronized List<Lot> importLotsFromScraper() throws SQLException {
List<Lot> imported = new ArrayList<>();
var sql = "SELECT lot_id, auction_id, title, description, category, " +
"current_bid, closing_time, url " +
"FROM lots";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
try {
var lot = ScraperDataAdapter.fromScraperLot(rs);
upsertLot(lot);
imported.add(lot);
} catch (Exception e) {
System.err.println("Failed to import lot: " + e.getMessage());
}
}
} catch (SQLException e) {
// Table might not exist in scraper format - that's ok
Console.println("ℹ️ Scraper lots table not found or incompatible schema");
}
return imported;
}
/**
* Imports image URLs from scraper's schema.
* The scraper populates the images table with URLs but doesn't download them.
* This method retrieves undownloaded images for processing.
*
* @return List of image URLs that need to be downloaded
*/
synchronized List<ImageImportRecord> getUnprocessedImagesFromScraper() throws SQLException {
List<ImageImportRecord> images = new ArrayList<>();
var sql = """
SELECT i.lot_id, i.url, l.auction_id
FROM images i
LEFT JOIN lots l ON i.lot_id = l.lot_id
WHERE i.downloaded = 0 OR i.local_path IS NULL
""";
try (var conn = DriverManager.getConnection(url); var stmt = conn.createStatement()) {
var rs = stmt.executeQuery(sql);
while (rs.next()) {
String lotIdStr = rs.getString("lot_id");
String auctionIdStr = rs.getString("auction_id");
int lotId = ScraperDataAdapter.extractNumericId(lotIdStr);
int saleId = ScraperDataAdapter.extractNumericId(auctionIdStr);
images.add(new ImageImportRecord(
images.add(new ImageImportRecord(
lotId,
saleId,
rs.getString("url")
));
}
} catch (SQLException e) {
Console.println("ℹ️ No unprocessed images found in scraper format");
}
));
}
} catch (SQLException e) {
log.info("ℹ️ No unprocessed images found in scraper format");
}
return images;
}
return images;
}
/**
* Simple record for image data from database
*/
record ImageRecord(int id, int lotId, String url, String filePath, String labels) {}
/**
* Simple record for image data from database
*/
record ImageRecord(int id, int lotId, String url, String filePath, String labels) { }
/**
* Record for importing images from scraper format
*/
record ImageImportRecord(int lotId, int saleId, String url) {}
/**
* Record for importing images from scraper format
*/
record ImageImportRecord(int lotId, int saleId, String url) { }
}

189
src/main/java/auctiora/ImageProcessingService.java
View File

@@ -1,5 +1,7 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.io.Console;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -13,111 +15,112 @@ import java.util.List;
* This separates image processing concerns from scraping, allowing this project
* to focus on enriching data scraped by the external process.
*/
@Slf4j
class ImageProcessingService {
private final RateLimitedHttpClient httpClient;
private final DatabaseService db;
private final ObjectDetectionService detector;
private final RateLimitedHttpClient2 httpClient;
private final DatabaseService db;
private final ObjectDetectionService detector;
ImageProcessingService(DatabaseService db, ObjectDetectionService detector, RateLimitedHttpClient httpClient) {
this.httpClient = httpClient;
this.db = db;
this.detector = detector;
}
ImageProcessingService(DatabaseService db, ObjectDetectionService detector, RateLimitedHttpClient2 httpClient) {
this.httpClient = httpClient;
this.db = db;
this.detector = detector;
}
/**
* Downloads an image from the given URL to local storage.
* Images are organized by saleId/lotId for easy management.
*
* @param imageUrl remote image URL
* @param saleId sale identifier
* @param lotId lot identifier
* @return absolute path to saved file or null on failure
*/
String downloadImage(String imageUrl, int saleId, int lotId) {
try {
var response = httpClient.sendGetBytes(imageUrl);
/**
* Downloads an image from the given URL to local storage.
* Images are organized by saleId/lotId for easy management.
*
* @param imageUrl remote image URL
* @param saleId sale identifier
* @param lotId lot identifier
* @return absolute path to saved file or null on failure
*/
String downloadImage(String imageUrl, int saleId, int lotId) {
try {
var response = httpClient.sendGetBytes(imageUrl);
if (response.statusCode() == 200) {
// Use Windows path: C:\mnt\okcomputer\output\images
var baseDir = Paths.get("C:", "mnt", "okcomputer", "output", "images");
var dir = baseDir.resolve(String.valueOf(saleId)).resolve(String.valueOf(lotId));
Files.createDirectories(dir);
if (response.statusCode() == 200) {
// Use Windows path: C:\mnt\okcomputer\output\images
var baseDir = Paths.get("C:", "mnt", "okcomputer", "output", "images");
var dir = baseDir.resolve(String.valueOf(saleId)).resolve(String.valueOf(lotId));
Files.createDirectories(dir);
// Extract filename from URL
var fileName = imageUrl.substring(imageUrl.lastIndexOf('/') + 1);
// Remove query parameters if present
int queryIndex = fileName.indexOf('?');
if (queryIndex > 0) {
fileName = fileName.substring(0, queryIndex);
}
var dest = dir.resolve(fileName);
Files.write(dest, response.body());
return dest.toAbsolutePath().toString();
// Extract filename from URL
var fileName = imageUrl.substring(imageUrl.lastIndexOf('/') + 1);
// Remove query parameters if present
int queryIndex = fileName.indexOf('?');
if (queryIndex > 0) {
fileName = fileName.substring(0, queryIndex);
}
} catch (IOException | InterruptedException e) {
System.err.println("Failed to download image " + imageUrl + ": " + e.getMessage());
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
var dest = dir.resolve(fileName);
Files.write(dest, response.body());
return dest.toAbsolutePath().toString();
}
} catch (IOException | InterruptedException e) {
System.err.println("Failed to download image " + imageUrl + ": " + e.getMessage());
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
}
return null;
}
/**
* Processes images for a specific lot: downloads and runs object detection.
*
* @param lotId lot identifier
* @param saleId sale identifier
* @param imageUrls list of image URLs to process
*/
void processImagesForLot(int lotId, int saleId, List<String> imageUrls) {
log.info(" Processing {} images for lot {}", imageUrls.size(), lotId);
for (var imgUrl : imageUrls) {
var fileName = downloadImage(imgUrl, saleId, lotId);
if (fileName != null) {
// Run object detection
var labels = detector.detectObjects(fileName);
// Save to database
try {
db.insertImage(lotId, imgUrl, fileName, labels);
if (!labels.isEmpty()) {
log.info(" Detected: {}", String.join(", ", labels));
}
} catch (SQLException e) {
System.err.println(" Failed to save image to database: " + e.getMessage());
}
}
return null;
}
}
}
}
/**
* Processes images for a specific lot: downloads and runs object detection.
*
* @param lotId lot identifier
* @param saleId sale identifier
* @param imageUrls list of image URLs to process
*/
void processImagesForLot(int lotId, int saleId, List<String> imageUrls) {
Console.println(" Processing " + imageUrls.size() + " images for lot " + lotId);
/**
* Batch processes all pending images in the database.
* Useful for processing images after the external scraper has populated lot data.
*/
void processPendingImages() {
log.info("Processing pending images...");
for (var imgUrl : imageUrls) {
var fileName = downloadImage(imgUrl, saleId, lotId);
try {
var lots = db.getAllLots();
log.info("Found {} lots to check for images", lots.size());
if (fileName != null) {
// Run object detection
var labels = detector.detectObjects(fileName);
for (var lot : lots) {
// Check if images already processed for this lot
var existingImages = db.getImagesForLot(lot.lotId());
// Save to database
try {
db.insertImage(lotId, imgUrl, fileName, labels);
if (!labels.isEmpty()) {
Console.println(" Detected: " + String.join(", ", labels));
}
} catch (SQLException e) {
System.err.println(" Failed to save image to database: " + e.getMessage());
}
if (existingImages.isEmpty()) {
log.info(" Lot {} has no images yet - needs external scraper data", lot.lotId());
}
}
}
}
/**
* Batch processes all pending images in the database.
* Useful for processing images after the external scraper has populated lot data.
*/
void processPendingImages() {
Console.println("Processing pending images...");
try {
var lots = db.getAllLots();
Console.println("Found " + lots.size() + " lots to check for images");
for (var lot : lots) {
// Check if images already processed for this lot
var existingImages = db.getImagesForLot(lot.lotId());
if (existingImages.isEmpty()) {
Console.println(" Lot " + lot.lotId() + " has no images yet - needs external scraper data");
}
}
} catch (SQLException e) {
System.err.println("Error processing pending images: " + e.getMessage());
}
}
} catch (SQLException e) {
System.err.println("Error processing pending images: " + e.getMessage());
}
}
}

49
src/main/java/auctiora/Lot.java
View File

@@ -1,5 +1,6 @@
package auctiora;
import lombok.With;
import java.time.Duration;
import java.time.LocalDateTime;
@@ -8,23 +9,37 @@ import java.time.LocalDateTime;
* Data typically populated by the external scraper process.
* This project enriches the data with image analysis and monitoring.
*/
@With
record Lot(
int saleId,
int lotId,
String title,
String description,
String manufacturer,
String type,
int year,
String category,
double currentBid,
String currency,
String url,
LocalDateTime closingTime,
boolean closingNotified
int saleId,
int lotId,
String title,
String description,
String manufacturer,
String type,
int year,
String category,
double currentBid,
String currency,
String url,
LocalDateTime closingTime,
boolean closingNotified
) {
long minutesUntilClose() {
if (closingTime == null) return Long.MAX_VALUE;
return Duration.between(LocalDateTime.now(), closingTime).toMinutes();
}
public long minutesUntilClose() {
if (closingTime == null) return Long.MAX_VALUE;
return Duration.between(LocalDateTime.now(), closingTime).toMinutes();
}
public Lot withCurrentBid(double newBid) {
return new Lot(saleId, lotId, title, description,
manufacturer, type, year, category,
newBid, currency, url, closingTime, closingNotified);
}
public Lot withClosingNotified(boolean flag) {
return new Lot(saleId, lotId, title, description,
manufacturer, type, year, category,
currentBid, currency, url, closingTime, flag);
}
}

84
src/main/java/auctiora/Main.java
View File

@@ -1,5 +1,6 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import org.opencv.core.Core;
/**
@@ -19,10 +20,11 @@ import org.opencv.core.Core;
* - Bid monitoring
* - Notifications
*/
@Slf4j
public class Main {
public static void main(String[] args) throws Exception {
Console.println("=== Troostwijk Auction Monitor ===\n");
log.info("=== Troostwijk Auction Monitor ===\n");
// Parse command line arguments
String mode = args.length > 0 ? args[0] : "workflow";
@@ -39,9 +41,9 @@ public class Main {
// Load native OpenCV library (only if models exist)
try {
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
Console.println("✓ OpenCV loaded");
log.info("✓ OpenCV loaded");
} catch (UnsatisfiedLinkError e) {
Console.println("⚠️ OpenCV not available - image detection disabled");
log.info("⚠️ OpenCV not available - image detection disabled");
}
switch (mode.toLowerCase()) {
@@ -75,7 +77,7 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
Console.println("🚀 Starting in WORKFLOW MODE (Orchestrated Scheduling)\n");
log.info("🚀 Starting in WORKFLOW MODE (Orchestrated Scheduling)\n");
WorkflowOrchestrator orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
@@ -87,16 +89,16 @@ public class Main {
// Start all scheduled workflows
orchestrator.startScheduledWorkflows();
Console.println("✓ All workflows are running");
Console.println(" - Scraper import: every 30 min");
Console.println(" - Image processing: every 1 hour");
Console.println(" - Bid monitoring: every 15 min");
Console.println(" - Closing alerts: every 5 min");
Console.println("\nPress Ctrl+C to stop.\n");
log.info("✓ All workflows are running");
log.info(" - Scraper import: every 30 min");
log.info(" - Image processing: every 1 hour");
log.info(" - Bid monitoring: every 15 min");
log.info(" - Closing alerts: every 5 min");
log.info("\nPress Ctrl+C to stop.\n");
// Add shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
Console.println("\n🛑 Shutdown signal received...");
log.info("\n🛑 Shutdown signal received...");
orchestrator.shutdown();
}));
@@ -117,7 +119,7 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
Console.println("🔄 Starting in ONCE MODE (Single Execution)\n");
log.info("🔄 Starting in ONCE MODE (Single Execution)\n");
WorkflowOrchestrator orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
@@ -125,7 +127,7 @@ public class Main {
orchestrator.runCompleteWorkflowOnce();
Console.println("✓ Workflow execution completed. Exiting.\n");
log.info("✓ Workflow execution completed. Exiting.\n");
}
/**
@@ -136,29 +138,29 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
Console.println("⚙️ Starting in LEGACY MODE\n");
log.info("⚙️ Starting in LEGACY MODE\n");
var monitor = new TroostwijkMonitor(dbPath, notifConfig,
yoloCfg, yoloWeights, yoloClasses);
Console.println("\n📊 Current Database State:");
log.info("\n📊 Current Database State:");
monitor.printDatabaseStats();
Console.println("\n[1/2] Processing images...");
log.info("\n[1/2] Processing images...");
monitor.processPendingImages();
Console.println("\n[2/2] Starting bid monitoring...");
log.info("\n[2/2] Starting bid monitoring...");
monitor.scheduleMonitoring();
Console.println("\n✓ Monitor is running. Press Ctrl+C to stop.\n");
Console.println("NOTE: This process expects auction/lot data from the external scraper.");
Console.println(" Make sure ARCHITECTURE-TROOSTWIJK-SCRAPER is running and populating the database.\n");
log.info("\n✓ Monitor is running. Press Ctrl+C to stop.\n");
log.info("NOTE: This process expects auction/lot data from the external scraper.");
log.info(" Make sure ARCHITECTURE-TROOSTWIJK-SCRAPER is running and populating the database.\n");
try {
Thread.sleep(Long.MAX_VALUE);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
Console.println("Monitor interrupted, exiting.");
log.info("Monitor interrupted, exiting.");
}
}
@@ -169,7 +171,7 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
Console.println("📊 Checking Status...\n");
log.info("📊 Checking Status...\n");
WorkflowOrchestrator orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
@@ -182,21 +184,21 @@ public class Main {
* Show usage information
*/
private static void showUsage() {
Console.println("Usage: java -jar troostwijk-monitor.jar [mode]\n");
Console.println("Modes:");
Console.println(" workflow - Run orchestrated scheduled workflows (default)");
Console.println(" once - Run complete workflow once and exit (for cron)");
Console.println(" legacy - Run original monitoring approach");
Console.println(" status - Show current status and exit");
Console.println("\nEnvironment Variables:");
Console.println(" DATABASE_FILE - Path to SQLite database");
Console.println(" (default: C:\\mnt\\okcomputer\\output\\cache.db)");
Console.println(" NOTIFICATION_CONFIG - 'desktop' or 'smtp:user:pass:email'");
Console.println(" (default: desktop)");
Console.println("\nExamples:");
Console.println(" java -jar troostwijk-monitor.jar workflow");
Console.println(" java -jar troostwijk-monitor.jar once");
Console.println(" java -jar troostwijk-monitor.jar status");
log.info("Usage: java -jar troostwijk-monitor.jar [mode]\n");
log.info("Modes:");
log.info(" workflow - Run orchestrated scheduled workflows (default)");
log.info(" once - Run complete workflow once and exit (for cron)");
log.info(" legacy - Run original monitoring approach");
log.info(" status - Show current status and exit");
log.info("\nEnvironment Variables:");
log.info(" DATABASE_FILE - Path to SQLite database");
log.info(" (default: C:\\mnt\\okcomputer\\output\\cache.db)");
log.info(" NOTIFICATION_CONFIG - 'desktop' or 'smtp:user:pass:email'");
log.info(" (default: desktop)");
log.info("\nExamples:");
log.info(" java -jar troostwijk-monitor.jar workflow");
log.info(" java -jar troostwijk-monitor.jar once");
log.info(" java -jar troostwijk-monitor.jar status");
IO.println();
}
@@ -206,18 +208,18 @@ public class Main {
*/
public static void main2(String[] args) {
if (args.length > 0) {
Console.println("Command mode - exiting to allow shell commands");
log.info("Command mode - exiting to allow shell commands");
return;
}
Console.println("Troostwijk Monitor container is running and healthy.");
Console.println("Use 'docker exec' or 'dokku run' to execute commands.");
log.info("Troostwijk Monitor container is running and healthy.");
log.info("Use 'docker exec' or 'dokku run' to execute commands.");
try {
Thread.sleep(Long.MAX_VALUE);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
Console.println("Container interrupted, exiting.");
log.info("Container interrupted, exiting.");
}
}
}

190
src/main/java/auctiora/NotificationService.java
View File

@@ -1,122 +1,48 @@
package auctiora;
import javax.mail.Authenticator;
import javax.mail.Message.RecipientType;
import javax.mail.PasswordAuthentication;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import java.awt.SystemTray;
import java.awt.Toolkit;
import java.awt.TrayIcon;
import java.awt.TrayIcon.MessageType;
import javax.mail.*;
import javax.mail.internet.*;
import lombok.extern.slf4j.Slf4j;
import java.awt.*;
import java.util.Date;
import java.util.Properties;
/**
* Service for sending notifications via desktop notifications and/or email.
* Supports free notification methods:
* 1. Desktop notifications (Windows/Linux/macOS system tray)
* 2. Email via Gmail SMTP (free, requires app password)
*
* Configuration:
* - For email: Set notificationEmail to your Gmail address
* - Enable 2FA in Gmail and create an App Password
* - Use format "smtp:username:appPassword:toEmail" for credentials
* - Or use "desktop" for desktop-only notifications
*/
class NotificationService {
private final boolean useDesktop;
private final boolean useEmail;
private final String smtpUsername;
private final String smtpPassword;
private final String toEmail;
@Slf4j
public class NotificationService {
/**
* Creates a notification service.
*
* @param config "desktop" for desktop only, or "smtp:username:password:toEmail" for email
* @param unusedParam Kept for compatibility (can pass empty string)
*/
NotificationService(String config, String unusedParam) {
private final Config config;
if ("desktop".equalsIgnoreCase(config)) {
this.useDesktop = true;
this.useEmail = false;
this.smtpUsername = null;
this.smtpPassword = null;
this.toEmail = null;
} else if (config.startsWith("smtp:")) {
var parts = config.split(":", 4);
if (parts.length != 4) {
throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'");
}
this.useDesktop = true; // Always include desktop
this.useEmail = true;
this.smtpUsername = parts[1];
this.smtpPassword = parts[2];
this.toEmail = parts[3];
} else {
throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'");
}
public NotificationService(String cfg) {
this.config = Config.parse(cfg);
}
/**
* Sends notification via configured channels.
*
* @param message The message body
* @param title Message title
* @param priority Priority level (0=normal, 1=high)
*/
void sendNotification(String message, String title, int priority) {
if (useDesktop) {
sendDesktopNotification(title, message, priority);
}
if (useEmail) {
sendEmailNotification(title, message, priority);
}
public void sendNotification(String message, String title, int priority) {
if (config.useDesktop()) sendDesktop(title, message, priority);
if (config.useEmail()) sendEmail(title, message, priority);
}
/**
* Sends a desktop notification using system tray.
* Works on Windows, macOS, and Linux with desktop environments.
*/
private void sendDesktopNotification(String title, String message, int priority) {
private void sendDesktop(String title, String msg, int prio) {
try {
if (SystemTray.isSupported()) {
var tray = SystemTray.getSystemTray();
var image = Toolkit.getDefaultToolkit()
.createImage(new byte[0]); // Empty image
var trayIcon = new TrayIcon(image, "Troostwijk Scraper");
trayIcon.setImageAutoSize(true);
var messageType = priority > 0
? MessageType.WARNING
: MessageType.INFO;
tray.add(trayIcon);
trayIcon.displayMessage(title, message, messageType);
// Remove icon after 2 seconds to avoid clutter
Thread.sleep(2000);
tray.remove(trayIcon);
Console.println("Desktop notification sent: " + title);
} else {
Console.println("Desktop notifications not supported, logging: " + title + " - " + message);
if (!SystemTray.isSupported()) {
log.info("Desktop notifications not supported — " + title + " / " + msg);
return;
}
var tray = SystemTray.getSystemTray();
var image = Toolkit.getDefaultToolkit().createImage(new byte[0]);
var trayIcon = new TrayIcon(image, "NotificationService");
trayIcon.setImageAutoSize(true);
var type = prio > 0 ? TrayIcon.MessageType.WARNING : TrayIcon.MessageType.INFO;
tray.add(trayIcon);
trayIcon.displayMessage(title, msg, type);
Thread.sleep(2000);
tray.remove(trayIcon);
log.info("Desktop notification sent: " + title);
} catch (Exception e) {
System.err.println("Desktop notification failed: " + e.getMessage());
System.err.println("Desktop notification failed: " + e);
}
}
/**
* Sends email notification via Gmail SMTP (free).
* Uses Gmail's SMTP server with app password authentication.
*/
private void sendEmailNotification(String title, String message, int priority) {
private void sendEmail(String title, String msg, int prio) {
try {
var props = new Properties();
props.put("mail.smtp.auth", "true");
@@ -125,32 +51,48 @@ class NotificationService {
props.put("mail.smtp.port", "587");
props.put("mail.smtp.ssl.trust", "smtp.gmail.com");
var session = Session.getInstance(props,
new Authenticator() {
var session = Session.getInstance(props, new Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(smtpUsername, smtpPassword);
}
});
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(config.smtpUsername(), config.smtpPassword());
}
});
var msg = new MimeMessage(session);
msg.setFrom(new InternetAddress(smtpUsername));
msg.setRecipients(RecipientType.TO,
InternetAddress.parse(toEmail));
msg.setSubject("[Troostwijk] " + title);
msg.setText(message);
msg.setSentDate(new Date());
if (priority > 0) {
msg.setHeader("X-Priority", "1");
msg.setHeader("Importance", "High");
var m = new MimeMessage(session);
m.setFrom(new InternetAddress(config.smtpUsername()));
m.setRecipients(Message.RecipientType.TO, InternetAddress.parse(config.toEmail()));
m.setSubject("[Troostwijk] " + title);
m.setText(msg);
m.setSentDate(new Date());
if (prio > 0) {
m.setHeader("X-Priority", "1");
m.setHeader("Importance", "High");
}
Transport.send(msg);
Console.println("Email notification sent: " + title);
Transport.send(m);
log.info("Email notification sent: " + title);
} catch (Exception e) {
System.err.println("Email notification failed: " + e.getMessage());
log.info("Email notification failed: " + e);
}
}
private record Config(
boolean useDesktop,
boolean useEmail,
String smtpUsername,
String smtpPassword,
String toEmail
) {
static Config parse(String cfg) {
if ("desktop".equalsIgnoreCase(cfg)) {
return new Config(true, false, null, null, null);
} else if (cfg.startsWith("smtp:")) {
var parts = cfg.split(":", 4);
if (parts.length != 4)
throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'");
return new Config(true, true, parts[1], parts[2], parts[3]);
}
throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'");
}
}
}

22
src/main/java/auctiora/ObjectDetectionService.java
View File

@@ -1,13 +1,16 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import org.opencv.core.Mat;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgcodecs.Imgcodecs;
import java.io.Console;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
@@ -24,7 +27,8 @@ import static org.opencv.dnn.Dnn.DNN_TARGET_CPU;
* If model files are not found, the service operates in disabled mode
* and returns empty lists.
*/
class ObjectDetectionService {
@Slf4j
public class ObjectDetectionService {
private final Net net;
private final List<String> classNames;
@@ -34,15 +38,15 @@ class ObjectDetectionService {
// Check if model files exist
var cfgFile = Paths.get(cfgPath);
var weightsFile = Paths.get(weightsPath);
var classNamesFile = Paths.get(classNamesPath);
var classNamesFile = Paths.get(classNamesPath);
if (!Files.exists(cfgFile) || !Files.exists(weightsFile) || !Files.exists(classNamesFile)) {
Console.println("⚠️ Object detection disabled: YOLO model files not found");
Console.println(" Expected files:");
Console.println(" - " + cfgPath);
Console.println(" - " + weightsPath);
Console.println(" - " + classNamesPath);
Console.println(" Scraper will continue without image analysis.");
log.info("⚠️ Object detection disabled: YOLO model files not found");
log.info(" Expected files:");
log.info(" - " + cfgPath);
log.info(" - " + weightsPath);
log.info(" - " + classNamesPath);
log.info(" Scraper will continue without image analysis.");
this.enabled = false;
this.net = null;
this.classNames = new ArrayList<>();
@@ -57,7 +61,7 @@ class ObjectDetectionService {
// Load class names (one per line)
this.classNames = Files.readAllLines(classNamesFile);
this.enabled = true;
Console.println("✓ Object detection enabled with YOLO");
log.info("✓ Object detection enabled with YOLO");
} catch (Exception e) {
System.err.println("⚠️ Object detection disabled: " + e.getMessage());
throw new IOException("Failed to initialize object detection", e);

271
src/main/java/auctiora/RateLimitedHttpClient.java
View File

@@ -2,9 +2,7 @@ package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
@@ -12,259 +10,66 @@ import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import io.github.bucket4j.*;
/**
* Rate-limited HTTP client that enforces per-host request limits.
*
* Features:
* - Per-host rate limiting (configurable max requests per second)
* - Request counting and monitoring
* - Thread-safe using semaphores
* - Automatic host extraction from URLs
*
* This prevents overloading external services like Troostwijk and getting blocked.
*/
@ApplicationScoped
public class RateLimitedHttpClient {
private static final Logger LOG = Logger.getLogger(RateLimitedHttpClient.class);
private final HttpClient httpClient;
private final Map<String, RateLimiter> rateLimiters;
private final Map<String, RequestStats> requestStats;
private final HttpClient client = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(30))
.build();
@ConfigProperty(name = "auction.http.rate-limit.default-max-rps", defaultValue = "2")
int defaultMaxRequestsPerSecond;
int defaultRps;
@ConfigProperty(name = "auction.http.rate-limit.troostwijk-max-rps", defaultValue = "1")
int troostwijkMaxRequestsPerSecond;
int troostwijkRps;
@ConfigProperty(name = "auction.http.timeout-seconds", defaultValue = "30")
int timeoutSeconds;
public RateLimitedHttpClient() {
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(30))
.build();
this.rateLimiters = new ConcurrentHashMap<>();
this.requestStats = new ConcurrentHashMap<>();
}
private final Map<String, Bucket> buckets = new ConcurrentHashMap<>();
/**
* Sends a GET request with automatic rate limiting based on host.
*/
public HttpResponse<String> sendGet(String url) throws IOException, InterruptedException {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofString());
}
/**
* Sends a request for binary data (like images) with rate limiting.
*/
public HttpResponse<byte[]> sendGetBytes(String url) throws IOException, InterruptedException {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofByteArray());
}
/**
* Sends any HTTP request with automatic rate limiting.
*/
public <T> HttpResponse<T> send(HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler)
throws IOException, InterruptedException {
String host = extractHost(request.uri());
RateLimiter limiter = getRateLimiter(host);
RequestStats stats = getRequestStats(host);
// Enforce rate limit (blocks if necessary)
limiter.acquire();
// Track request
stats.incrementTotal();
long startTime = System.currentTimeMillis();
try {
HttpResponse<T> response = httpClient.send(request, bodyHandler);
long duration = System.currentTimeMillis() - startTime;
stats.recordSuccess(duration);
LOG.debugf("HTTP %d %s %s (%dms)",
response.statusCode(), request.method(), host, duration);
// Track rate limit violations (429 = Too Many Requests)
if (response.statusCode() == 429) {
stats.incrementRateLimited();
LOG.warnf("⚠️ Rate limited by %s (HTTP 429)", host);
}
return response;
} catch (IOException | InterruptedException e) {
stats.incrementFailed();
LOG.warnf("❌ HTTP request failed for %s: %s", host, e.getMessage());
throw e;
}
}
/**
* Gets or creates a rate limiter for a specific host.
*/
private RateLimiter getRateLimiter(String host) {
return rateLimiters.computeIfAbsent(host, h -> {
int maxRps = getMaxRequestsPerSecond(h);
LOG.infof("Initializing rate limiter for %s: %d req/s", h, maxRps);
return new RateLimiter(maxRps);
private Bucket bucketForHost(String host) {
return buckets.computeIfAbsent(host, h -> {
int rps = host.contains("troostwijk") ? troostwijkRps : defaultRps;
var limit = Bandwidth.simple(rps, Duration.ofSeconds(1));
return Bucket4j.builder().addLimit(limit).build();
});
}
/**
* Gets or creates request stats for a specific host.
*/
private RequestStats getRequestStats(String host) {
return requestStats.computeIfAbsent(host, h -> new RequestStats(h));
public HttpResponse<String> sendGet(String url) throws Exception {
var req = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(req, HttpResponse.BodyHandlers.ofString());
}
/**
* Determines max requests per second for a given host.
*/
private int getMaxRequestsPerSecond(String host) {
if (host.contains("troostwijk")) {
return troostwijkMaxRequestsPerSecond;
}
return defaultMaxRequestsPerSecond;
public HttpResponse<byte[]> sendGetBytes(String url) throws Exception {
var req = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(req, HttpResponse.BodyHandlers.ofByteArray());
}
/**
* Extracts host from URI (e.g., "api.troostwijkauctions.com").
*/
private String extractHost(URI uri) {
return uri.getHost() != null ? uri.getHost() : uri.toString();
}
public <T> HttpResponse<T> send(HttpRequest req,
HttpResponse.BodyHandler<T> handler) throws Exception {
String host = req.uri().getHost();
var bucket = bucketForHost(host);
bucket.asBlocking().consume(1);
/**
* Gets statistics for all hosts.
*/
public Map<String, RequestStats> getAllStats() {
return Map.copyOf(requestStats);
}
var start = System.currentTimeMillis();
var resp = client.send(req, handler);
var duration = System.currentTimeMillis() - start;
/**
* Gets statistics for a specific host.
*/
public RequestStats getStats(String host) {
return requestStats.get(host);
}
// (Optional) Logging
System.out.printf("HTTP %d %s %s in %d ms%n",
resp.statusCode(), req.method(), host, duration);
/**
* Rate limiter implementation using token bucket algorithm.
* Allows burst traffic up to maxRequestsPerSecond, then enforces steady rate.
*/
private static class RateLimiter {
private final Semaphore semaphore;
private final int maxRequestsPerSecond;
private final long intervalNanos;
RateLimiter(int maxRequestsPerSecond) {
this.maxRequestsPerSecond = maxRequestsPerSecond;
this.intervalNanos = TimeUnit.SECONDS.toNanos(1) / maxRequestsPerSecond;
this.semaphore = new Semaphore(maxRequestsPerSecond);
// Refill tokens periodically
startRefillThread();
}
void acquire() throws InterruptedException {
semaphore.acquire();
// Enforce minimum delay between requests
long delayMillis = intervalNanos / 1_000_000;
if (delayMillis > 0) {
Thread.sleep(delayMillis);
}
}
private void startRefillThread() {
Thread refillThread = new Thread(() -> {
while (!Thread.currentThread().isInterrupted()) {
try {
Thread.sleep(1000); // Refill every second
int toRelease = maxRequestsPerSecond - semaphore.availablePermits();
if (toRelease > 0) {
semaphore.release(toRelease);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}, "RateLimiter-Refill");
refillThread.setDaemon(true);
refillThread.start();
}
}
/**
* Statistics tracker for HTTP requests per host.
*/
public static class RequestStats {
private final String host;
private final AtomicLong totalRequests = new AtomicLong(0);
private final AtomicLong successfulRequests = new AtomicLong(0);
private final AtomicLong failedRequests = new AtomicLong(0);
private final AtomicLong rateLimitedRequests = new AtomicLong(0);
private final AtomicLong totalDurationMs = new AtomicLong(0);
RequestStats(String host) {
this.host = host;
}
void incrementTotal() {
totalRequests.incrementAndGet();
}
void recordSuccess(long durationMs) {
successfulRequests.incrementAndGet();
totalDurationMs.addAndGet(durationMs);
}
void incrementFailed() {
failedRequests.incrementAndGet();
}
void incrementRateLimited() {
rateLimitedRequests.incrementAndGet();
}
// Getters
public String getHost() { return host; }
public long getTotalRequests() { return totalRequests.get(); }
public long getSuccessfulRequests() { return successfulRequests.get(); }
public long getFailedRequests() { return failedRequests.get(); }
public long getRateLimitedRequests() { return rateLimitedRequests.get(); }
public long getAverageDurationMs() {
long successful = successfulRequests.get();
return successful > 0 ? totalDurationMs.get() / successful : 0;
}
@Override
public String toString() {
return String.format("%s: %d total, %d success, %d failed, %d rate-limited, avg %dms",
host, getTotalRequests(), getSuccessfulRequests(),
getFailedRequests(), getRateLimitedRequests(), getAverageDurationMs());
}
return resp;
}
}

270
src/main/java/auctiora/RateLimitedHttpClient2.java Normal file
View File

@@ -0,0 +1,270 @@
package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/**
* Rate-limited HTTP client that enforces per-host request limits.
*
* Features:
* - Per-host rate limiting (configurable max requests per second)
* - Request counting and monitoring
* - Thread-safe using semaphores
* - Automatic host extraction from URLs
*
* This prevents overloading external services like Troostwijk and getting blocked.
*/
@ApplicationScoped
public class RateLimitedHttpClient2 {
private static final Logger LOG = Logger.getLogger(RateLimitedHttpClient2.class);
private final HttpClient httpClient;
private final Map<String, RateLimiter> rateLimiters;
private final Map<String, RequestStats> requestStats;
@ConfigProperty(name = "auction.http.rate-limit.default-max-rps", defaultValue = "2")
int defaultMaxRequestsPerSecond;
@ConfigProperty(name = "auction.http.rate-limit.troostwijk-max-rps", defaultValue = "1")
int troostwijkMaxRequestsPerSecond;
@ConfigProperty(name = "auction.http.timeout-seconds", defaultValue = "30")
int timeoutSeconds;
public RateLimitedHttpClient2() {
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(30))
.build();
this.rateLimiters = new ConcurrentHashMap<>();
this.requestStats = new ConcurrentHashMap<>();
}
/**
* Sends a GET request with automatic rate limiting based on host.
*/
public HttpResponse<String> sendGet(String url) throws IOException, InterruptedException {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofString());
}
/**
* Sends a request for binary data (like images) with rate limiting.
*/
public HttpResponse<byte[]> sendGetBytes(String url) throws IOException, InterruptedException {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofByteArray());
}
/**
* Sends any HTTP request with automatic rate limiting.
*/
public <T> HttpResponse<T> send(HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler)
throws IOException, InterruptedException {
String host = extractHost(request.uri());
RateLimiter limiter = getRateLimiter(host);
RequestStats stats = getRequestStats(host);
// Enforce rate limit (blocks if necessary)
limiter.acquire();
// Track request
stats.incrementTotal();
long startTime = System.currentTimeMillis();
try {
HttpResponse<T> response = httpClient.send(request, bodyHandler);
long duration = System.currentTimeMillis() - startTime;
stats.recordSuccess(duration);
LOG.debugf("HTTP %d %s %s (%dms)",
response.statusCode(), request.method(), host, duration);
// Track rate limit violations (429 = Too Many Requests)
if (response.statusCode() == 429) {
stats.incrementRateLimited();
LOG.warnf("⚠️ Rate limited by %s (HTTP 429)", host);
}
return response;
} catch (IOException | InterruptedException e) {
stats.incrementFailed();
LOG.warnf("❌ HTTP request failed for %s: %s", host, e.getMessage());
throw e;
}
}
/**
* Gets or creates a rate limiter for a specific host.
*/
private RateLimiter getRateLimiter(String host) {
return rateLimiters.computeIfAbsent(host, h -> {
int maxRps = getMaxRequestsPerSecond(h);
LOG.infof("Initializing rate limiter for %s: %d req/s", h, maxRps);
return new RateLimiter(maxRps);
});
}
/**
* Gets or creates request stats for a specific host.
*/
private RequestStats getRequestStats(String host) {
return requestStats.computeIfAbsent(host, h -> new RequestStats(h));
}
/**
* Determines max requests per second for a given host.
*/
private int getMaxRequestsPerSecond(String host) {
if (host.contains("troostwijk")) {
return troostwijkMaxRequestsPerSecond;
}
return defaultMaxRequestsPerSecond;
}
/**
* Extracts host from URI (e.g., "api.troostwijkauctions.com").
*/
private String extractHost(URI uri) {
return uri.getHost() != null ? uri.getHost() : uri.toString();
}
/**
* Gets statistics for all hosts.
*/
public Map<String, RequestStats> getAllStats() {
return Map.copyOf(requestStats);
}
/**
* Gets statistics for a specific host.
*/
public RequestStats getStats(String host) {
return requestStats.get(host);
}
/**
* Rate limiter implementation using token bucket algorithm.
* Allows burst traffic up to maxRequestsPerSecond, then enforces steady rate.
*/
private static class RateLimiter {
private final Semaphore semaphore;
private final int maxRequestsPerSecond;
private final long intervalNanos;
RateLimiter(int maxRequestsPerSecond) {
this.maxRequestsPerSecond = maxRequestsPerSecond;
this.intervalNanos = TimeUnit.SECONDS.toNanos(1) / maxRequestsPerSecond;
this.semaphore = new Semaphore(maxRequestsPerSecond);
// Refill tokens periodically
startRefillThread();
}
void acquire() throws InterruptedException {
semaphore.acquire();
// Enforce minimum delay between requests
long delayMillis = intervalNanos / 1_000_000;
if (delayMillis > 0) {
Thread.sleep(delayMillis);
}
}
private void startRefillThread() {
Thread refillThread = new Thread(() -> {
while (!Thread.currentThread().isInterrupted()) {
try {
Thread.sleep(1000); // Refill every second
int toRelease = maxRequestsPerSecond - semaphore.availablePermits();
if (toRelease > 0) {
semaphore.release(toRelease);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}, "RateLimiter-Refill");
refillThread.setDaemon(true);
refillThread.start();
}
}
/**
* Statistics tracker for HTTP requests per host.
*/
public static class RequestStats {
private final String host;
private final AtomicLong totalRequests = new AtomicLong(0);
private final AtomicLong successfulRequests = new AtomicLong(0);
private final AtomicLong failedRequests = new AtomicLong(0);
private final AtomicLong rateLimitedRequests = new AtomicLong(0);
private final AtomicLong totalDurationMs = new AtomicLong(0);
RequestStats(String host) {
this.host = host;
}
void incrementTotal() {
totalRequests.incrementAndGet();
}
void recordSuccess(long durationMs) {
successfulRequests.incrementAndGet();
totalDurationMs.addAndGet(durationMs);
}
void incrementFailed() {
failedRequests.incrementAndGet();
}
void incrementRateLimited() {
rateLimitedRequests.incrementAndGet();
}
// Getters
public String getHost() { return host; }
public long getTotalRequests() { return totalRequests.get(); }
public long getSuccessfulRequests() { return successfulRequests.get(); }
public long getFailedRequests() { return failedRequests.get(); }
public long getRateLimitedRequests() { return rateLimitedRequests.get(); }
public long getAverageDurationMs() {
long successful = successfulRequests.get();
return successful > 0 ? totalDurationMs.get() / successful : 0;
}
@Override
public String toString() {
return String.format("%s: %d total, %d success, %d failed, %d rate-limited, avg %dms",
host, getTotalRequests(), getSuccessfulRequests(),
getFailedRequests(), getRateLimitedRequests(), getAverageDurationMs());
}
}
}

328
src/main/java/auctiora/ScraperDataAdapter.java
View File

@@ -1,246 +1,136 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
/**
* Adapter to convert data from the Python scraper's schema to the Monitor's schema.
*
* SCRAPER SCHEMA DIFFERENCES:
* - auction_id: TEXT ("A7-39813") vs INTEGER (39813)
* - lot_id: TEXT ("A1-28505-5") vs INTEGER (285055)
* - current_bid: TEXT ("€123.45") vs REAL (123.45)
* - Field names: lots_count vs lot_count, auction_id vs sale_id, etc.
*
* This adapter handles the translation between the two schemas.
*/
class ScraperDataAdapter {
@Slf4j
public class ScraperDataAdapter {
private static final DateTimeFormatter[] TIMESTAMP_FORMATS = {
DateTimeFormatter.ISO_LOCAL_DATE_TIME,
DateTimeFormatter.ISO_DATE_TIME,
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
};
private static final DateTimeFormatter[] TIMESTAMP_FORMATS = {
DateTimeFormatter.ISO_LOCAL_DATE_TIME,
DateTimeFormatter.ISO_DATE_TIME,
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
};
/**
* Converts scraper's auction format to monitor's AuctionInfo record.
*
* Scraper format:
* - auction_id: "A7-39813" (TEXT)
* - location: "Cluj-Napoca, RO" (combined)
* - lots_count: INTEGER
* - first_lot_closing_time: TEXT
* - scraped_at: TEXT
*/
static AuctionInfo fromScraperAuction(ResultSet rs) throws SQLException {
// Parse "A7-39813" → auctionId=39813, type="A7"
String auctionIdStr = rs.getString("auction_id");
int auctionId = extractNumericId(auctionIdStr);
String type = extractTypePrefix(auctionIdStr);
static AuctionInfo fromScraperAuction(ResultSet rs) throws SQLException {
// Parse "A7-39813" → auctionId=39813, type="A7"
String auctionIdStr = rs.getString("auction_id");
int auctionId = extractNumericId(auctionIdStr);
String type = extractTypePrefix(auctionIdStr);
// Split "Cluj-Napoca, RO" → city="Cluj-Napoca", country="RO"
String location = rs.getString("location");
String[] locationParts = parseLocation(location);
String city = locationParts[0];
String country = locationParts[1];
// Split "Cluj-Napoca, RO" → city="Cluj-Napoca", country="RO"
String location = rs.getString("location");
String[] locationParts = parseLocation(location);
String city = locationParts[0];
String country = locationParts[1];
// Map field names
int lotCount = getIntOrDefault(rs, "lots_count", 0);
LocalDateTime closingTime = parseTimestamp(getStringOrNull(rs, "first_lot_closing_time"));
// Map field names
int lotCount = getIntOrDefault(rs, "lots_count", 0);
LocalDateTime closingTime = parseTimestamp(getStringOrNull(rs, "first_lot_closing_time"));
return new AuctionInfo(
auctionId,
rs.getString("title"),
location,
city,
country,
rs.getString("url"),
type,
lotCount,
closingTime
);
}
return new AuctionInfo(
auctionId,
rs.getString("title"),
location,
city,
country,
rs.getString("url"),
type,
lotCount,
closingTime
);
}
/**
* Converts scraper's lot format to monitor's Lot record.
*
* Scraper format:
* - lot_id: "A1-28505-5" (TEXT)
* - auction_id: "A7-39813" (TEXT)
* - current_bid: "€123.45" or "No bids" (TEXT)
* - bid_count: INTEGER
* - closing_time: TEXT
*/
static Lot fromScraperLot(ResultSet rs) throws SQLException {
// Parse "A1-28505-5" → lotId=285055
String lotIdStr = rs.getString("lot_id");
int lotId = extractNumericId(lotIdStr);
public static Lot fromScraperLot(ResultSet rs) throws SQLException {
var lotId = extractNumericId(rs.getString("lot_id"));
var saleId = extractNumericId(rs.getString("auction_id"));
// Parse "A7-39813" → saleId=39813
String auctionIdStr = rs.getString("auction_id");
int saleId = extractNumericId(auctionIdStr);
var bidStr = getStringOrNull(rs, "current_bid");
var bid = parseBidAmount(bidStr);
var currency = parseBidCurrency(bidStr);
// Parse "€123.45" → currentBid=123.45, currency="EUR"
String currentBidStr = getStringOrNull(rs, "current_bid");
double currentBid = parseBidAmount(currentBidStr);
String currency = parseBidCurrency(currentBidStr);
var closing = parseTimestamp(getStringOrNull(rs, "closing_time"));
// Parse timestamp
LocalDateTime closingTime = parseTimestamp(getStringOrNull(rs, "closing_time"));
return new Lot(
saleId,
lotId,
rs.getString("title"),
getStringOrDefault(rs, "description", ""),
"", "", 0,
getStringOrDefault(rs, "category", ""),
bid,
currency,
rs.getString("url"),
closing,
false
);
}
return new Lot(
saleId,
lotId,
rs.getString("title"),
getStringOrDefault(rs, "description", ""),
"", // manufacturer - not in scraper schema
"", // type - not in scraper schema
0, // year - not in scraper schema
getStringOrDefault(rs, "category", ""),
currentBid,
currency,
rs.getString("url"),
closingTime,
false // closing_notified - not yet notified
);
}
public static int extractNumericId(String id) {
if (id == null || id.isBlank()) return 0;
var digits = id.replaceAll("\\D+", "");
return digits.isEmpty() ? 0 : Integer.parseInt(digits);
}
/**
* Extracts numeric ID from scraper's text format.
* Examples:
* - "A7-39813" → 39813
* - "A1-28505-5" → 285055 (concatenates all digits)
*/
static int extractNumericId(String id) {
if (id == null || id.isEmpty()) {
return 0;
}
String digits = id.replaceAll("[^0-9]", "");
return digits.isEmpty() ? 0 : Integer.parseInt(digits);
}
private static String extractTypePrefix(String id) {
if (id == null) return "";
var idx = id.indexOf('-');
return idx > 0 ? id.substring(0, idx) : "";
}
/**
* Extracts type prefix from scraper's auction/lot ID.
* Examples:
* - "A7-39813" → "A7"
* - "A1-28505-5" → "A1"
*/
private static String extractTypePrefix(String id) {
if (id == null || id.isEmpty()) {
return "";
}
int dashIndex = id.indexOf('-');
return dashIndex > 0 ? id.substring(0, dashIndex) : "";
}
private static String[] parseLocation(String location) {
if (location == null || location.isBlank()) return new String[]{ "", "" };
var parts = location.split(",\\s*");
var city = parts[0].trim();
var country = parts.length > 1 ? parts[parts.length - 1].trim() : "";
return new String[]{ city, country };
}
/**
* Parses location string into [city, country] array.
* Examples:
* - "Cluj-Napoca, RO" → ["Cluj-Napoca", "RO"]
* - "Amsterdam" → ["Amsterdam", ""]
*/
private static String[] parseLocation(String location) {
if (location == null || location.isEmpty()) {
return new String[]{"", ""};
}
private static double parseBidAmount(String bid) {
if (bid == null || bid.isBlank() || bid.toLowerCase().contains("no")) return 0.0;
var cleaned = bid.replaceAll("[^0-9.]", "");
try {
return cleaned.isEmpty() ? 0.0 : Double.parseDouble(cleaned);
} catch (NumberFormatException e) {
return 0.0;
}
}
String[] parts = location.split(",\\s*");
String city = parts.length > 0 ? parts[0].trim() : "";
String country = parts.length > 1 ? parts[parts.length - 1].trim() : "";
private static String parseBidCurrency(String bid) {
if (bid == null) return "EUR";
return bid.contains("€") ? "EUR"
: bid.contains("$") ? "USD"
: bid.contains("£") ? "GBP"
: "EUR";
}
return new String[]{city, country};
}
private static LocalDateTime parseTimestamp(String ts) {
if (ts == null || ts.isBlank()) return null;
for (var fmt : TIMESTAMP_FORMATS) {
try {
return LocalDateTime.parse(ts, fmt);
} catch (DateTimeParseException ignored) { }
}
log.info("Unable to parse timestamp: {}", ts);
return null;
}
/**
* Parses bid amount from scraper's text format.
* Examples:
* - "€123.45" → 123.45
* - "$50.00" → 50.0
* - "No bids" → 0.0
* - "123.45" → 123.45
*/
private static double parseBidAmount(String bid) {
if (bid == null || bid.isEmpty() || bid.toLowerCase().contains("no")) {
return 0.0;
}
private static String getStringOrNull(ResultSet rs, String col) throws SQLException {
return rs.getString(col);
}
try {
// Remove all non-numeric characters except decimal point
String cleanBid = bid.replaceAll("[^0-9.]", "");
return cleanBid.isEmpty() ? 0.0 : Double.parseDouble(cleanBid);
} catch (NumberFormatException e) {
return 0.0;
}
}
private static String getStringOrDefault(ResultSet rs, String col, String def) throws SQLException {
var v = rs.getString(col);
return v != null ? v : def;
}
/**
* Extracts currency from bid string.
* Examples:
* - "€123.45" → "EUR"
* - "$50.00" → "USD"
* - "123.45" → "EUR" (default)
*/
private static String parseBidCurrency(String bid) {
if (bid == null || bid.isEmpty()) {
return "EUR";
}
if (bid.contains("€")) return "EUR";
if (bid.contains("$")) return "USD";
if (bid.contains("£")) return "GBP";
return "EUR"; // Default
}
/**
* Parses timestamp from various formats used by the scraper.
* Tries multiple formats in order.
*/
private static LocalDateTime parseTimestamp(String timestamp) {
if (timestamp == null || timestamp.isEmpty()) {
return null;
}
for (DateTimeFormatter formatter : TIMESTAMP_FORMATS) {
try {
return LocalDateTime.parse(timestamp, formatter);
} catch (DateTimeParseException e) {
// Try next format
}
}
// Couldn't parse - return null
Console.println("⚠️ Could not parse timestamp: " + timestamp);
return null;
}
// Helper methods for safe ResultSet access
private static String getStringOrNull(ResultSet rs, String column) throws SQLException {
try {
return rs.getString(column);
} catch (SQLException e) {
return null;
}
}
private static String getStringOrDefault(ResultSet rs, String column, String defaultValue) throws SQLException {
try {
String value = rs.getString(column);
return value != null ? value : defaultValue;
} catch (SQLException e) {
return defaultValue;
}
}
private static int getIntOrDefault(ResultSet rs, String column, int defaultValue) throws SQLException {
try {
return rs.getInt(column);
} catch (SQLException e) {
return defaultValue;
}
}
private static int getIntOrDefault(ResultSet rs, String col, int def) throws SQLException {
var v = rs.getInt(col);
return rs.wasNull() ? def : v;
}
}

84
src/main/java/auctiora/StatusResource.java Normal file
View File

@@ -0,0 +1,84 @@
package auctiora;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Map;
@Slf4j
@Path("/api")
public class StatusResource {
private static final DateTimeFormatter FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z")
.withZone(ZoneId.systemDefault());
@ConfigProperty(name = "application.version", defaultValue = "1.0-SNAPSHOT")
String appVersion;
@ConfigProperty(name = "application.groupId")
String groupId;
@ConfigProperty(name = "application.artifactId")
String artifactId;
@ConfigProperty(name = "application.version")
String version;
// Java 16+ Record for structured response
public record StatusResponse(
String groupId,
String artifactId,
String version,
String status,
String timestamp,
String mvnVersion,
String javaVersion,
String os,
String openCvVersion
) { }
@GET
@Path("/status")
@Produces(MediaType.APPLICATION_JSON)
public StatusResponse getStatus() {
log.info("Status endpoint called");
return new StatusResponse(groupId, artifactId, version,
"running",
FORMATTER.format(Instant.now()),
appVersion,
System.getProperty("java.version"),
System.getProperty("os.name") + " " + System.getProperty("os.arch"),
getOpenCvVersion()
);
}
@GET
@Path("/hello")
@Produces(MediaType.APPLICATION_JSON)
public Map<String, String> sayHello() {
log.info("hello endpoint called");
return Map.of(
"message", "Hello from Scrape-UI!",
"timestamp", FORMATTER.format(Instant.now()),
"openCvVersion", getOpenCvVersion()
);
}
private String getOpenCvVersion() {
try {
// Load OpenCV if not already loaded (safe to call multiple times)
nu.pattern.OpenCV.loadLocally();
return org.opencv.core.Core.VERSION;
} catch (Exception e) {
return "4.9.0 (default)";
}
}
}

262
src/main/java/auctiora/TroostwijkMonitor.java
View File

@@ -1,174 +1,136 @@
package auctiora;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.sql.SQLException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* Monitoring service for Troostwijk auction lots.
* This class focuses on:
* - Monitoring bid changes on lots (populated by external scraper)
* - Sending notifications for important events
* - Coordinating image processing
*
* Does NOT handle scraping - that's done by the external ARCHITECTURE-TROOSTWIJK-SCRAPER process.
*/
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Slf4j
public class TroostwijkMonitor {
private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list";
private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list";
private final RateLimitedHttpClient httpClient;
private final ObjectMapper objectMapper;
public final DatabaseService db;
private final NotificationService notifier;
private final ObjectDetectionService detector;
private final ImageProcessingService imageProcessor;
RateLimitedHttpClient2 httpClient;
ObjectMapper objectMapper;
@Getter DatabaseService db;
NotificationService notifier;
ObjectDetectionService detector;
ImageProcessingService imageProcessor;
/**
* Constructor for the monitoring service.
*
* @param databasePath Path to SQLite database file (shared with external scraper)
* @param notificationConfig "desktop" or "smtp:user:pass:email"
* @param yoloCfgPath YOLO config file path
* @param yoloWeightsPath YOLO weights file path
* @param classNamesPath Class names file path
*/
public TroostwijkMonitor(String databasePath, String notificationConfig,
String yoloCfgPath, String yoloWeightsPath, String classNamesPath)
throws SQLException, IOException {
this.httpClient = new RateLimitedHttpClient();
this.objectMapper = new ObjectMapper();
this.db = new DatabaseService(databasePath);
this.notifier = new NotificationService(notificationConfig, "");
this.detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
this.imageProcessor = new ImageProcessingService(db, detector, httpClient);
ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
var t = new Thread(r, "troostwijk-monitor-thread");
t.setDaemon(true);
return t;
});
// Initialize database schema
db.ensureSchema();
}
public TroostwijkMonitor(String databasePath,
String notificationConfig,
String yoloCfgPath,
String yoloWeightsPath,
String classNamesPath)
throws SQLException, IOException {
/**
* Schedules periodic monitoring of all lots.
* Runs every hour to refresh bids and detect changes.
* Increases frequency for lots closing soon.
*/
public void scheduleMonitoring() {
var scheduler = Executors.newScheduledThreadPool(1);
scheduler.scheduleAtFixedRate(() -> {
httpClient = new RateLimitedHttpClient2();
objectMapper = new ObjectMapper();
db = new DatabaseService(databasePath);
notifier = new NotificationService(notificationConfig);
detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
imageProcessor = new ImageProcessingService(db, detector, httpClient);
db.ensureSchema();
}
public void scheduleMonitoring() {
scheduler.scheduleAtFixedRate(this::monitorAllLots, 0, 1, TimeUnit.HOURS);
log.info("✓ Monitoring service started");
}
private void monitorAllLots() {
try {
var activeLots = db.getActiveLots();
log.info("Monitoring {} active lots …", activeLots.size());
for (var lot : activeLots) {
checkAndUpdateLot(lot);
}
} catch (SQLException e) {
log.error("Error during scheduled monitoring", e);
}
}
private void checkAndUpdateLot(Lot lot) {
refreshLotBid(lot);
var minutesLeft = lot.minutesUntilClose();
if (minutesLeft < 30) {
if (minutesLeft <= 5 && !lot.closingNotified()) {
notifier.sendNotification(
"Kavel " + lot.lotId() + " sluit binnen " + minutesLeft + " min.",
"Lot nearing closure", 1);
try {
var activeLots = db.getActiveLots();
Console.println("Monitoring " + activeLots.size() + " active lots...");
for (var lot : activeLots) {
// Refresh lot bidding information
refreshLotBid(lot);
// Check closing time
var minutesLeft = lot.minutesUntilClose();
if (minutesLeft < 30) {
// Send warning when within 5 minutes
if (minutesLeft <= 5 && !lot.closingNotified()) {
notifier.sendNotification(
"Kavel " + lot.lotId() + " sluit binnen " + minutesLeft + " min.",
"Lot nearing closure", 1);
// Update notification flag
var updated = new Lot(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
lot.currentBid(), lot.currency(), lot.url(),
lot.closingTime(), true
);
db.updateLotNotificationFlags(updated);
}
// Schedule additional quick check
scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
}
}
db.updateLotNotificationFlags(lot.withClosingNotified(true));
} catch (SQLException e) {
System.err.println("Error during scheduled monitoring: " + e.getMessage());
throw new RuntimeException(e);
}
}, 0, 1, TimeUnit.HOURS);
}
scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
}
}
Console.println("✓ Monitoring service started");
}
private void refreshLotBid(Lot lot) {
try {
var url = LOT_API +
"?batchSize=1&listType=7&offset=0&sortOption=0" +
"&saleID=" + lot.saleId() +
"&parentID=0&relationID=0&buildversion=201807311" +
"&lotID=" + lot.lotId();
/**
* Refreshes the bid for a single lot and sends notification if changed.
*
* @param lot the lot to refresh
*/
private void refreshLotBid(Lot lot) {
try {
var url = LOT_API + "?batchSize=1&listType=7&offset=0&sortOption=0&saleID=" + lot.saleId()
+ "&parentID=0&relationID=0&buildversion=201807311&lotID=" + lot.lotId();
var resp = httpClient.sendGet(url);
if (resp.statusCode() != 200) return;
var response = httpClient.sendGet(url);
if (response.statusCode() != 200) return;
var root = objectMapper.readTree(response.body());
var results = root.path("results");
if (results.isArray() && !results.isEmpty()) {
var node = results.get(0);
var newBid = node.path("cb").asDouble();
if (Double.compare(newBid, lot.currentBid()) > 0) {
var previous = lot.currentBid();
// Create updated lot with new bid
var updatedLot = new Lot(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
newBid, lot.currency(), lot.url(),
lot.closingTime(), lot.closingNotified()
);
db.updateLotCurrentBid(updatedLot);
var msg = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previous);
notifier.sendNotification(msg, "Kavel bieding update", 0);
}
var root = objectMapper.readTree(resp.body());
var results = root.path("results");
if (results.isArray() && results.size() > 0) {
var newBid = results.get(0).path("cb").asDouble();
if (Double.compare(newBid, lot.currentBid()) > 0) {
var previous = lot.currentBid();
var updatedLot = lot.withCurrentBid(newBid);
db.updateLotCurrentBid(updatedLot);
var msg = String.format(
"Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previous);
notifier.sendNotification(msg, "Kavel bieding update", 0);
}
} catch (IOException | InterruptedException | SQLException e) {
System.err.println("Failed to refresh bid for lot " + lot.lotId() + ": " + e.getMessage());
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
}
}
}
} catch (IOException | InterruptedException | SQLException e) {
log.warn("Failed to refresh bid for lot {}", lot.lotId(), e);
if (e instanceof InterruptedException) Thread.currentThread().interrupt();
}
}
/**
* Prints statistics about the data in the database.
*/
public void printDatabaseStats() {
try {
var allLots = db.getAllLots();
var imageCount = db.getImageCount();
public void printDatabaseStats() {
try {
var allLots = db.getAllLots();
var imageCount = db.getImageCount();
log.info("📊 Database Summary: total lots = {}, total images = {}",
allLots.size(), imageCount);
if (!allLots.isEmpty()) {
var sum = allLots.stream().mapToDouble(Lot::currentBid).sum();
log.info("Total current bids: €{:.2f}", sum);
}
} catch (SQLException e) {
log.warn("Could not retrieve database stats", e);
}
}
Console.println("📊 Database Summary:");
Console.println(" Total lots in database: " + allLots.size());
Console.println(" Total images processed: " + imageCount);
if (!allLots.isEmpty()) {
var totalBids = allLots.stream().mapToDouble(Lot::currentBid).sum();
Console.println(" Total current bids: €" + String.format("%.2f", totalBids));
}
} catch (SQLException e) {
System.err.println(" ⚠️ Could not retrieve database stats: " + e.getMessage());
}
}
/**
* Process pending images for lots in the database.
* This should be called after the external scraper has populated lot data.
*/
public void processPendingImages() {
imageProcessor.processPendingImages();
}
public void processPendingImages() {
imageProcessor.processPendingImages();
}
}

769
src/main/java/auctiora/WorkflowOrchestrator.java
View File

@@ -1,5 +1,7 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.io.Console;
import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
@@ -14,429 +16,430 @@ import java.util.concurrent.TimeUnit;
* This class coordinates all services and provides scheduled execution,
* event-driven triggers, and manual workflow execution.
*/
@Slf4j
public class WorkflowOrchestrator {
private final TroostwijkMonitor monitor;
private final DatabaseService db;
private final ImageProcessingService imageProcessor;
private final NotificationService notifier;
private final ObjectDetectionService detector;
private final TroostwijkMonitor monitor;
private final DatabaseService db;
private final ImageProcessingService imageProcessor;
private final NotificationService notifier;
private final ObjectDetectionService detector;
private final ScheduledExecutorService scheduler;
private boolean isRunning = false;
private final ScheduledExecutorService scheduler;
private boolean isRunning = false;
/**
* Creates a workflow orchestrator with all necessary services.
*/
public WorkflowOrchestrator(String databasePath, String notificationConfig,
/**
* Creates a workflow orchestrator with all necessary services.
*/
public WorkflowOrchestrator(String databasePath, String notificationConfig,
String yoloCfg, String yoloWeights, String yoloClasses)
throws SQLException, IOException {
throws SQLException, IOException {
Console.println("🔧 Initializing Workflow Orchestrator...");
log.info("🔧 Initializing Workflow Orchestrator...");
// Initialize core services
this.db = new DatabaseService(databasePath);
this.db.ensureSchema();
// Initialize core services
this.db = new DatabaseService(databasePath);
this.db.ensureSchema();
this.notifier = new NotificationService(notificationConfig, "");
this.detector = new ObjectDetectionService(yoloCfg, yoloWeights, yoloClasses);
RateLimitedHttpClient httpClient = new RateLimitedHttpClient();
this.imageProcessor = new ImageProcessingService(db, detector, httpClient);
this.notifier = new NotificationService(notificationConfig);
this.detector = new ObjectDetectionService(yoloCfg, yoloWeights, yoloClasses);
RateLimitedHttpClient2 httpClient = new RateLimitedHttpClient2();
this.imageProcessor = new ImageProcessingService(db, detector, httpClient);
this.monitor = new TroostwijkMonitor(databasePath, notificationConfig,
yoloCfg, yoloWeights, yoloClasses);
this.monitor = new TroostwijkMonitor(databasePath, notificationConfig,
yoloCfg, yoloWeights, yoloClasses);
this.scheduler = Executors.newScheduledThreadPool(3);
this.scheduler = Executors.newScheduledThreadPool(3);
Console.println("✓ Workflow Orchestrator initialized");
}
log.info("✓ Workflow Orchestrator initialized");
}
/**
* Starts all scheduled workflows.
* This is the main entry point for automated operation.
*/
public void startScheduledWorkflows() {
if (isRunning) {
Console.println("⚠️ Workflows already running");
return;
}
/**
* Starts all scheduled workflows.
* This is the main entry point for automated operation.
*/
public void startScheduledWorkflows() {
if (isRunning) {
log.info("⚠️ Workflows already running");
return;
}
Console.println("\n🚀 Starting Scheduled Workflows...\n");
log.info("\n🚀 Starting Scheduled Workflows...\n");
// Workflow 1: Import scraper data (every 30 minutes)
scheduleScraperDataImport();
// Workflow 1: Import scraper data (every 30 minutes)
scheduleScraperDataImport();
// Workflow 2: Process pending images (every 1 hour)
scheduleImageProcessing();
// Workflow 2: Process pending images (every 1 hour)
scheduleImageProcessing();
// Workflow 3: Monitor bids (every 15 minutes)
scheduleBidMonitoring();
// Workflow 3: Monitor bids (every 15 minutes)
scheduleBidMonitoring();
// Workflow 4: Check closing times (every 5 minutes)
scheduleClosingAlerts();
// Workflow 4: Check closing times (every 5 minutes)
scheduleClosingAlerts();
isRunning = true;
Console.println("✓ All scheduled workflows started\n");
}
isRunning = true;
log.info("✓ All scheduled workflows started\n");
}
/**
* Workflow 1: Import Scraper Data
* Frequency: Every 30 minutes
* Purpose: Import new auctions and lots from external scraper
*/
private void scheduleScraperDataImport() {
scheduler.scheduleAtFixedRate(() -> {
try {
Console.println("📥 [WORKFLOW 1] Importing scraper data...");
long start = System.currentTimeMillis();
/**
* Workflow 1: Import Scraper Data
* Frequency: Every 30 minutes
* Purpose: Import new auctions and lots from external scraper
*/
private void scheduleScraperDataImport() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("📥 [WORKFLOW 1] Importing scraper data...");
long start = System.currentTimeMillis();
// Import auctions
var auctions = db.importAuctionsFromScraper();
Console.println(" → Imported " + auctions.size() + " auctions");
// Import lots
var lots = db.importLotsFromScraper();
Console.println(" → Imported " + lots.size() + " lots");
// Import image URLs
var images = db.getUnprocessedImagesFromScraper();
Console.println(" → Found " + images.size() + " unprocessed images");
long duration = System.currentTimeMillis() - start;
Console.println(" ✓ Scraper import completed in " + duration + "ms\n");
// Trigger notification if significant data imported
if (auctions.size() > 0 || lots.size() > 10) {
notifier.sendNotification(
String.format("Imported %d auctions, %d lots", auctions.size(), lots.size()),
"Data Import Complete",
0
);
}
} catch (Exception e) {
Console.println(" ❌ Scraper import failed: " + e.getMessage());
}
}, 0, 30, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Scraper Data Import (every 30 min)");
}
/**
* Workflow 2: Process Pending Images
* Frequency: Every 1 hour
* Purpose: Download images and run object detection
*/
private void scheduleImageProcessing() {
scheduler.scheduleAtFixedRate(() -> {
try {
Console.println("🖼️ [WORKFLOW 2] Processing pending images...");
long start = System.currentTimeMillis();
// Get unprocessed images
var unprocessedImages = db.getUnprocessedImagesFromScraper();
if (unprocessedImages.isEmpty()) {
Console.println(" → No pending images to process\n");
return;
}
Console.println(" → Processing " + unprocessedImages.size() + " images");
int processed = 0;
int detected = 0;
for (var imageRecord : unprocessedImages) {
try {
// Download image
String filePath = imageProcessor.downloadImage(
imageRecord.url(),
imageRecord.saleId(),
imageRecord.lotId()
);
if (filePath != null) {
// Run object detection
var labels = detector.detectObjects(filePath);
// Save to database
db.insertImage(imageRecord.lotId(), imageRecord.url(),
filePath, labels);
processed++;
if (!labels.isEmpty()) {
detected++;
// Send notification for interesting detections
if (labels.size() >= 3) {
notifier.sendNotification(
String.format("Lot %d: Detected %s",
imageRecord.lotId(),
String.join(", ", labels)),
"Objects Detected",
0
);
}
}
}
// Rate limiting
Thread.sleep(500);
} catch (Exception e) {
Console.println(" ⚠️ Failed to process image: " + e.getMessage());
}
}
long duration = System.currentTimeMillis() - start;
Console.println(String.format(" ✓ Processed %d images, detected objects in %d (%.1fs)\n",
processed, detected, duration / 1000.0));
} catch (Exception e) {
Console.println(" ❌ Image processing failed: " + e.getMessage());
}
}, 5, 60, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Image Processing (every 1 hour)");
}
/**
* Workflow 3: Monitor Bids
* Frequency: Every 15 minutes
* Purpose: Check for bid changes and send notifications
*/
private void scheduleBidMonitoring() {
scheduler.scheduleAtFixedRate(() -> {
try {
Console.println("💰 [WORKFLOW 3] Monitoring bids...");
long start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
Console.println(" → Checking " + activeLots.size() + " active lots");
int bidChanges = 0;
for (var lot : activeLots) {
// Note: In production, this would call Troostwijk API
// For now, we just track what's in the database
// The external scraper updates bids, we just notify
}
long duration = System.currentTimeMillis() - start;
Console.println(String.format(" ✓ Bid monitoring completed in %dms\n", duration));
} catch (Exception e) {
Console.println(" ❌ Bid monitoring failed: " + e.getMessage());
}
}, 2, 15, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Bid Monitoring (every 15 min)");
}
/**
* Workflow 4: Check Closing Times
* Frequency: Every 5 minutes
* Purpose: Send alerts for lots closing soon
*/
private void scheduleClosingAlerts() {
scheduler.scheduleAtFixedRate(() -> {
try {
Console.println("⏰ [WORKFLOW 4] Checking closing times...");
long start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
int alertsSent = 0;
for (var lot : activeLots) {
if (lot.closingTime() == null) continue;
long minutesLeft = lot.minutesUntilClose();
// Alert for lots closing in 5 minutes
if (minutesLeft <= 5 && minutesLeft > 0 && !lot.closingNotified()) {
String message = String.format("Kavel %d sluit binnen %d min.",
lot.lotId(), minutesLeft);
notifier.sendNotification(message, "Lot Closing Soon", 1);
// Mark as notified
var updated = new Lot(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
lot.currentBid(), lot.currency(), lot.url(),
lot.closingTime(), true
);
db.updateLotNotificationFlags(updated);
alertsSent++;
}
}
long duration = System.currentTimeMillis() - start;
Console.println(String.format(" → Sent %d closing alerts in %dms\n",
alertsSent, duration));
} catch (Exception e) {
Console.println(" ❌ Closing alerts failed: " + e.getMessage());
}
}, 1, 5, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Closing Alerts (every 5 min)");
}
/**
* Manual trigger: Run complete workflow once
* Useful for testing or on-demand execution
*/
public void runCompleteWorkflowOnce() {
Console.println("\n🔄 Running Complete Workflow (Manual Trigger)...\n");
try {
// Step 1: Import data
Console.println("[1/4] Importing scraper data...");
// Import auctions
var auctions = db.importAuctionsFromScraper();
log.info(" → Imported " + auctions.size() + " auctions");
// Import lots
var lots = db.importLotsFromScraper();
Console.println(" ✓ Imported " + auctions.size() + " auctions, " + lots.size() + " lots");
log.info(" → Imported " + lots.size() + " lots");
// Step 2: Process images
Console.println("[2/4] Processing pending images...");
monitor.processPendingImages();
Console.println(" ✓ Image processing completed");
// Import image URLs
var images = db.getUnprocessedImagesFromScraper();
log.info(" → Found " + images.size() + " unprocessed images");
// Step 3: Check bids
Console.println("[3/4] Monitoring bids...");
var activeLots = db.getActiveLots();
Console.println(" ✓ Monitored " + activeLots.size() + " lots");
long duration = System.currentTimeMillis() - start;
log.info(" ✓ Scraper import completed in " + duration + "ms\n");
// Step 4: Check closing times
Console.println("[4/4] Checking closing times...");
int closingSoon = 0;
for (var lot : activeLots) {
if (lot.closingTime() != null && lot.minutesUntilClose() < 30) {
closingSoon++;
}
// Trigger notification if significant data imported
if (auctions.size() > 0 || lots.size() > 10) {
notifier.sendNotification(
String.format("Imported %d auctions, %d lots", auctions.size(), lots.size()),
"Data Import Complete",
0
);
}
Console.println(" ✓ Found " + closingSoon + " lots closing soon");
Console.println("\n✓ Complete workflow finished successfully\n");
} catch (Exception e) {
log.info(" ❌ Scraper import failed: " + e.getMessage());
}
}, 0, 30, TimeUnit.MINUTES);
} catch (Exception e) {
Console.println("\n❌ Workflow failed: " + e.getMessage() + "\n");
}
}
log.info(" ✓ Scheduled: Scraper Data Import (every 30 min)");
}
/**
* Event-driven trigger: New auction discovered
*/
public void onNewAuctionDiscovered(AuctionInfo auction) {
Console.println("📣 EVENT: New auction discovered - " + auction.title());
/**
* Workflow 2: Process Pending Images
* Frequency: Every 1 hour
* Purpose: Download images and run object detection
*/
private void scheduleImageProcessing() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("🖼️ [WORKFLOW 2] Processing pending images...");
long start = System.currentTimeMillis();
try {
db.upsertAuction(auction);
// Get unprocessed images
var unprocessedImages = db.getUnprocessedImagesFromScraper();
if (unprocessedImages.isEmpty()) {
log.info(" → No pending images to process\n");
return;
}
log.info(" → Processing " + unprocessedImages.size() + " images");
int processed = 0;
int detected = 0;
for (var imageRecord : unprocessedImages) {
try {
// Download image
String filePath = imageProcessor.downloadImage(
imageRecord.url(),
imageRecord.saleId(),
imageRecord.lotId()
);
if (filePath != null) {
// Run object detection
var labels = detector.detectObjects(filePath);
// Save to database
db.insertImage(imageRecord.lotId(), imageRecord.url(),
filePath, labels);
processed++;
if (!labels.isEmpty()) {
detected++;
// Send notification for interesting detections
if (labels.size() >= 3) {
notifier.sendNotification(
String.format("Lot %d: Detected %s",
imageRecord.lotId(),
String.join(", ", labels)),
"Objects Detected",
0
);
}
}
}
// Rate limiting
Thread.sleep(500);
} catch (Exception e) {
log.info(" ⚠️ Failed to process image: " + e.getMessage());
}
}
long duration = System.currentTimeMillis() - start;
log.info(String.format(" ✓ Processed %d images, detected objects in %d (%.1fs)\n",
processed, detected, duration / 1000.0));
} catch (Exception e) {
log.info(" ❌ Image processing failed: " + e.getMessage());
}
}, 5, 60, TimeUnit.MINUTES);
log.info(" ✓ Scheduled: Image Processing (every 1 hour)");
}
/**
* Workflow 3: Monitor Bids
* Frequency: Every 15 minutes
* Purpose: Check for bid changes and send notifications
*/
private void scheduleBidMonitoring() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("💰 [WORKFLOW 3] Monitoring bids...");
long start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
log.info(" → Checking " + activeLots.size() + " active lots");
int bidChanges = 0;
for (var lot : activeLots) {
// Note: In production, this would call Troostwijk API
// For now, we just track what's in the database
// The external scraper updates bids, we just notify
}
long duration = System.currentTimeMillis() - start;
log.info(String.format(" ✓ Bid monitoring completed in %dms\n", duration));
} catch (Exception e) {
log.info(" ❌ Bid monitoring failed: " + e.getMessage());
}
}, 2, 15, TimeUnit.MINUTES);
log.info(" ✓ Scheduled: Bid Monitoring (every 15 min)");
}
/**
* Workflow 4: Check Closing Times
* Frequency: Every 5 minutes
* Purpose: Send alerts for lots closing soon
*/
private void scheduleClosingAlerts() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("⏰ [WORKFLOW 4] Checking closing times...");
long start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
int alertsSent = 0;
for (var lot : activeLots) {
if (lot.closingTime() == null) continue;
long minutesLeft = lot.minutesUntilClose();
// Alert for lots closing in 5 minutes
if (minutesLeft <= 5 && minutesLeft > 0 && !lot.closingNotified()) {
String message = String.format("Kavel %d sluit binnen %d min.",
lot.lotId(), minutesLeft);
notifier.sendNotification(message, "Lot Closing Soon", 1);
// Mark as notified
var updated = new Lot(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
lot.currentBid(), lot.currency(), lot.url(),
lot.closingTime(), true
);
db.updateLotNotificationFlags(updated);
alertsSent++;
}
}
long duration = System.currentTimeMillis() - start;
log.info(String.format(" → Sent %d closing alerts in %dms\n",
alertsSent, duration));
} catch (Exception e) {
log.info(" ❌ Closing alerts failed: " + e.getMessage());
}
}, 1, 5, TimeUnit.MINUTES);
log.info(" ✓ Scheduled: Closing Alerts (every 5 min)");
}
/**
* Manual trigger: Run complete workflow once
* Useful for testing or on-demand execution
*/
public void runCompleteWorkflowOnce() {
log.info("\n🔄 Running Complete Workflow (Manual Trigger)...\n");
try {
// Step 1: Import data
log.info("[1/4] Importing scraper data...");
var auctions = db.importAuctionsFromScraper();
var lots = db.importLotsFromScraper();
log.info(" ✓ Imported " + auctions.size() + " auctions, " + lots.size() + " lots");
// Step 2: Process images
log.info("[2/4] Processing pending images...");
monitor.processPendingImages();
log.info(" ✓ Image processing completed");
// Step 3: Check bids
log.info("[3/4] Monitoring bids...");
var activeLots = db.getActiveLots();
log.info(" ✓ Monitored " + activeLots.size() + " lots");
// Step 4: Check closing times
log.info("[4/4] Checking closing times...");
int closingSoon = 0;
for (var lot : activeLots) {
if (lot.closingTime() != null && lot.minutesUntilClose() < 30) {
closingSoon++;
}
}
log.info(" ✓ Found " + closingSoon + " lots closing soon");
log.info("\n✓ Complete workflow finished successfully\n");
} catch (Exception e) {
log.info("\n❌ Workflow failed: " + e.getMessage() + "\n");
}
}
/**
* Event-driven trigger: New auction discovered
*/
public void onNewAuctionDiscovered(AuctionInfo auction) {
log.info("📣 EVENT: New auction discovered - " + auction.title());
try {
db.upsertAuction(auction);
notifier.sendNotification(
String.format("New auction: %s\nLocation: %s\nLots: %d",
auction.title(), auction.location(), auction.lotCount()),
"New Auction Discovered",
0
);
} catch (Exception e) {
log.info(" ❌ Failed to handle new auction: " + e.getMessage());
}
}
/**
* Event-driven trigger: Bid change detected
*/
public void onBidChange(Lot lot, double previousBid, double newBid) {
log.info(String.format("📣 EVENT: Bid change on lot %d (€%.2f → €%.2f)",
lot.lotId(), previousBid, newBid));
try {
db.updateLotCurrentBid(lot);
notifier.sendNotification(
String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previousBid),
"Kavel Bieding Update",
0
);
} catch (Exception e) {
log.info(" ❌ Failed to handle bid change: " + e.getMessage());
}
}
/**
* Event-driven trigger: Objects detected in image
*/
public void onObjectsDetected(int lotId, List<String> labels) {
log.info(String.format("📣 EVENT: Objects detected in lot %d - %s",
lotId, String.join(", ", labels)));
try {
if (labels.size() >= 2) {
notifier.sendNotification(
String.format("New auction: %s\nLocation: %s\nLots: %d",
auction.title(), auction.location(), auction.lotCount()),
"New Auction Discovered",
0
);
} catch (Exception e) {
Console.println(" ❌ Failed to handle new auction: " + e.getMessage());
}
}
/**
* Event-driven trigger: Bid change detected
*/
public void onBidChange(Lot lot, double previousBid, double newBid) {
Console.println(String.format("📣 EVENT: Bid change on lot %d (€%.2f → €%.2f)",
lot.lotId(), previousBid, newBid));
try {
db.updateLotCurrentBid(lot);
notifier.sendNotification(
String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previousBid),
"Kavel Bieding Update",
0
);
} catch (Exception e) {
Console.println(" ❌ Failed to handle bid change: " + e.getMessage());
}
}
/**
* Event-driven trigger: Objects detected in image
*/
public void onObjectsDetected(int lotId, List<String> labels) {
Console.println(String.format("📣 EVENT: Objects detected in lot %d - %s",
lotId, String.join(", ", labels)));
try {
if (labels.size() >= 2) {
notifier.sendNotification(
String.format("Lot %d contains: %s", lotId, String.join(", ", labels)),
"Objects Detected",
0
);
);
}
} catch (Exception e) {
log.info(" ❌ Failed to send detection notification: " + e.getMessage());
}
}
/**
* Prints current workflow status
*/
public void printStatus() {
log.info("\n📊 Workflow Status:");
log.info(" Running: " + (isRunning ? "Yes" : "No"));
try {
var auctions = db.getAllAuctions();
var lots = db.getAllLots();
int images = db.getImageCount();
log.info(" Auctions: " + auctions.size());
log.info(" Lots: " + lots.size());
log.info(" Images: " + images);
// Count closing soon
int closingSoon = 0;
for (var lot : lots) {
if (lot.closingTime() != null && lot.minutesUntilClose() < 30) {
closingSoon++;
}
} catch (Exception e) {
Console.println(" ❌ Failed to send detection notification: " + e.getMessage());
}
}
}
log.info(" Closing soon (< 30 min): " + closingSoon);
/**
* Prints current workflow status
*/
public void printStatus() {
Console.println("\n📊 Workflow Status:");
Console.println(" Running: " + (isRunning ? "Yes" : "No"));
} catch (Exception e) {
log.info(" ⚠️ Could not retrieve status: " + e.getMessage());
}
try {
var auctions = db.getAllAuctions();
var lots = db.getAllLots();
int images = db.getImageCount();
IO.println();
}
Console.println(" Auctions: " + auctions.size());
Console.println(" Lots: " + lots.size());
Console.println(" Images: " + images);
/**
* Gracefully shuts down all workflows
*/
public void shutdown() {
log.info("\n🛑 Shutting down workflows...");
// Count closing soon
int closingSoon = 0;
for (var lot : lots) {
if (lot.closingTime() != null && lot.minutesUntilClose() < 30) {
closingSoon++;
}
}
Console.println(" Closing soon (< 30 min): " + closingSoon);
isRunning = false;
scheduler.shutdown();
} catch (Exception e) {
Console.println(" ⚠️ Could not retrieve status: " + e.getMessage());
}
IO.println();
}
/**
* Gracefully shuts down all workflows
*/
public void shutdown() {
Console.println("\n🛑 Shutting down workflows...");
isRunning = false;
scheduler.shutdown();
try {
if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
scheduler.shutdownNow();
}
Console.println("✓ Workflows shut down successfully\n");
} catch (InterruptedException e) {
try {
if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
scheduler.shutdownNow();
Thread.currentThread().interrupt();
}
}
}
log.info("✓ Workflows shut down successfully\n");
} catch (InterruptedException e) {
scheduler.shutdownNow();
Thread.currentThread().interrupt();
}
}
}

19
src/main/resources/application.properties
View File

@@ -1,10 +1,21 @@
# Application Configuration
quarkus.application.name=auctiora
quarkus.application.version=1.0-SNAPSHOT
# Values will be injected from pom.xml during build
quarkus.application.name=${project.artifactId}
quarkus.application.version=${project.version}
# Custom properties for groupId if needed
application.groupId=${project.groupId}
application.artifactId=${project.artifactId}
application.version=${project.version}
# HTTP Configuration
quarkus.http.port=8081
quarkus.http.host=0.0.0.0
# ========== DEVELOPMENT (quarkus:dev) ==========
%dev.quarkus.http.host=127.0.0.1
# ========== PRODUCTION (Docker/JAR) ==========
%prod.quarkus.http.host=0.0.0.0
# ========== TEST PROFILE ==========
%test.quarkus.http.host=localhost
# Enable CORS for frontend development
quarkus.http.cors=true
@@ -26,7 +37,7 @@ quarkus.log.console.level=INFO
# Static resources
quarkus.http.enable-compression=true
quarkus.rest.path=/api
quarkus.rest.path=/
quarkus.http.root-path=/
# Auction Monitor Configuration

7
src/main/resources/resources/beans.xml Normal file
View File

@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="https://jakarta.ee/xml/ns/jakartaee"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="https://jakarta.ee/xml/ns/jakartaee https://jakarta.ee/xml/ns/jakartaee/beans_3_0.xsd
https://jakarta.ee/xml/ns/jakartaee "
version="3.0" bean-discovery-mode="all">
</beans>

224
src/main/resources/resources/index.html Normal file
View File

@@ -0,0 +1,224 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Scrape-UI 1 - Enterprise</title>
<script src="https://cdn.tailwindcss.com"></script>
<style>
.gradient-bg {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
}
.card-hover {
transition: all 0.3s ease;
}
.card-hover:hover {
transform: translateY(-5px);
box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04);
}
</style>
</head>
<body class="bg-gray-50 min-h-screen">
<!-- Header -->
<header class="gradient-bg text-white py-8">
<div class="container mx-auto px-4">
<h1 class="text-4xl font-bold mb-2">Scrape-UI Enterprise</h1>
<p class="text-xl opacity-90">Powered by Quarkus + Modern Frontend</p>
</div>
</header>
<!-- Main Content -->
<main class="container mx-auto px-4 py-8">
<!-- API Status Card -->
<!-- API & Build Status Card -->
<div class="bg-white rounded-lg shadow-md p-6 mb-8 card-hover">
<h2 class="text-2xl font-bold mb-4 text-gray-800">Build & Runtime Status</h2>
<div id="api-status" class="space-y-4">
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
<!-- Build Information -->
<div class="bg-blue-50 p-4 rounded-lg">
<h3 class="font-semibold text-blue-800 mb-2">📦 Maven Build</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
<span class="text-gray-600">Group:</span>
<span class="font-mono font-medium" id="build-group">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Artifact:</span>
<span class="font-mono font-medium" id="build-artifact">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Version:</span>
<span class="font-mono font-medium px-2 py-1 bg-blue-100 rounded" id="build-version">-</span>
</div>
</div>
</div>
<!-- Runtime Information -->
<div class="bg-green-50 p-4 rounded-lg">
<h3 class="font-semibold text-green-800 mb-2">🚀 Runtime</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
<span class="text-gray-600">Status:</span>
<span class="px-2 py-1 rounded-full text-xs font-semibold bg-green-100 text-green-800" id="runtime-status">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Java:</span>
<span class="font-mono" id="java-version">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Platform:</span>
<span class="font-mono" id="runtime-os">-</span>
</div>
</div>
</div>
</div>
<!-- Timestamp & Additional Info -->
<div class="pt-4 border-t">
<div class="flex justify-between items-center">
<div>
<p class="text-sm text-gray-500">Last Updated</p>
<p class="font-medium" id="last-updated">-</p>
</div>
<button onclick="fetchStatus()" class="bg-gray-100 hover:bg-gray-200 text-gray-700 px-4 py-2 rounded-lg text-sm transition-colors">
🔄 Refresh
</button>
</div>
</div>
</div>
</div>
<!-- API Response Card -->
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h2 class="text-2xl font-bold mb-4 text-gray-800">API Test</h2>
<button id="test-api" class="bg-blue-600 text-white px-6 py-2 rounded-lg hover:bg-blue-700 transition-colors mb-4">
Test Greeting API
</button>
<div id="api-response" class="bg-gray-100 p-4 rounded-lg">
<pre class="text-sm text-gray-700">Click the button to test the API</pre>
</div>
</div>
<!-- Features Grid -->
<div class="grid grid-cols-1 md:grid-cols-3 gap-6 mt-8">
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">⚡ Quarkus Backend</h3>
<p class="text-gray-600">Fast startup, low memory footprint, optimized for containers</p>
</div>
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">🚀 REST API</h3>
<p class="text-gray-600">RESTful endpoints with JSON responses</p>
</div>
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">🎨 Modern UI</h3>
<p class="text-gray-600">Responsive design with Tailwind CSS</p>
</div>
</div>
</main>
<script>
// Fetch API status on load
async function fetchStatus() {
try {
const response = await fetch('/api/status')
if (!response.ok) {
throw new Error(`HTTP ${ response.status }: ${ response.statusText }`)
}
const data = await response.json()
// Update Build Information
document.getElementById('build-group').textContent = data.groupId || 'N/A'
document.getElementById('build-artifact').textContent = data.artifactId || data.name || 'N/A'
document.getElementById('build-version').textContent = data.version || 'N/A'
// Update Runtime Information
document.getElementById('runtime-status').textContent = data.status || 'unknown'
document.getElementById('java-version').textContent = data.javaVersion || System.getProperty?.('java.version') || 'N/A'
document.getElementById('runtime-os').textContent = data.os || 'N/A'
// Update Timestamp
const timestamp = data.timestamp ? new Date(data.timestamp).toLocaleString() : 'N/A'
document.getElementById('last-updated').textContent = timestamp
// Update status badge color based on status
const statusBadge = document.getElementById('runtime-status')
if (data.status?.toLowerCase() === 'running') {
statusBadge.className = 'px-2 py-1 rounded-full text-xs font-semibold bg-green-100 text-green-800'
} else {
statusBadge.className = 'px-2 py-1 rounded-full text-xs font-semibold bg-yellow-100 text-yellow-800'
}
} catch (error) {
console.error('Error fetching status:', error)
document.getElementById('api-status').innerHTML = `
<div class="bg-red-50 border-l-4 border-red-500 p-4">
<div class="flex">
<div class="flex-shrink-0">
<svg class="h-5 w-5 text-red-400" viewBox="0 0 20 20" fill="currentColor">
<path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clip-rule="evenodd" />
</svg>
</div>
<div class="ml-3">
<p class="text-sm text-red-700">Failed to load status: ${ error.message }</p>
<button onclick="fetchStatus()" class="mt-2 text-sm text-red-700 hover:text-red-600 font-medium">
Retry →
</button>
</div>
</div>
</div>
`
}
}
// Fetch API status on load
async function fetchStatus3() {
try {
const response = await fetch('/api/status')
const data = await response.json()
document.getElementById('api-status').innerHTML = `
<p><strong>Application:</strong> ${ data.application }</p>
<p><strong>Status:</strong> <span class="text-green-600 font-semibold">${ data.status }</span></p>
<p><strong>Version:</strong> ${ data.version }</p>
<p><strong>Timestamp:</strong> ${ data.timestamp }</p>
`
} catch (error) {
document.getElementById('api-status').innerHTML = `
<p class="text-red-600">Error loading status: ${ error.message }</p>
`
}
}
// Test greeting API
document.getElementById('test-api').addEventListener('click', async () => {
try {
const response = await fetch('/api/hello')
const data = await response.json()
document.getElementById('api-response').innerHTML = `
<pre class="text-sm text-gray-700">${ JSON.stringify(data, null, 2) }</pre>
`
} catch (error) {
document.getElementById('api-response').innerHTML = `
<pre class="text-sm text-red-600">Error: ${ error.message }</pre>
`
}
})
// Auto-refresh every 30 seconds
let refreshInterval = setInterval(fetchStatus, 30000);
// Stop auto-refresh when page loses focus (optional)
document.addEventListener('visibilitychange', function() {
if (document.hidden) {
clearInterval(refreshInterval);
} else {
refreshInterval = setInterval(fetchStatus, 30000);
fetchStatus(); // Refresh immediately when returning to tab
}
});
// Load status on page load
fetchStatus()
</script>
</body>
</html>

26
src/test/java/auctiora/AuctionParsingTest.java
View File

@@ -1,5 +1,6 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@@ -19,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.*;
* Test auction parsing logic using saved HTML from test.html
* Tests the markup data extraction for each auction found
*/
@Slf4j
public class AuctionParsingTest {
private static String testHtml;
@@ -27,12 +29,12 @@ public class AuctionParsingTest {
public static void loadTestHtml() throws IOException {
// Load the test HTML file
testHtml = Files.readString(Paths.get("src/test/resources/test_auctions.html"));
System.out.println("Loaded test HTML (" + testHtml.length() + " characters)");
log.info("Loaded test HTML ({} characters)", testHtml.length());
}
@Test
public void testLocationPatternMatching() {
System.out.println("\n=== Location Pattern Tests ===");
log.info("\n=== Location Pattern Tests ===");
// Test different location formats
var testCases = new String[]{
@@ -48,16 +50,16 @@ public class AuctionParsingTest {
if (elem != null) {
var text = elem.text();
System.out.println("\nTest: " + testHtml);
System.out.println("Text: " + text);
log.info("\nTest: {}", testHtml);
log.info("Text: {}", text);
// Test regex pattern
if (text.matches(".*[A-Z]{2}$")) {
var countryCode = text.substring(text.length() - 2);
var cityPart = text.substring(0, text.length() - 2).trim().replaceAll("[,\\s]+$", "");
System.out.println("→ Extracted: " + cityPart + ", " + countryCode);
log.info("→ Extracted: {}, {}", cityPart, countryCode);
} else {
System.out.println("→ No match");
log.info("→ No match");
}
}
}
@@ -65,7 +67,7 @@ public class AuctionParsingTest {
@Test
public void testFullTextPatternMatching() {
System.out.println("\n=== Full Text Pattern Tests ===");
log.info("\n=== Full Text Pattern Tests ===");
// Test the complete auction text format
var testCases = new String[]{
@@ -75,7 +77,7 @@ public class AuctionParsingTest {
};
for (var testText : testCases) {
System.out.println("\nParsing: \"" + testText + "\"");
log.info("\nParsing: \"{}\"", testText);
// Simulated extraction
var remaining = testText;
@@ -84,7 +86,7 @@ public class AuctionParsingTest {
var timePattern = java.util.regex.Pattern.compile("(\\w+)\\s+om\\s+(\\d{1,2}:\\d{2})");
var timeMatcher = timePattern.matcher(remaining);
if (timeMatcher.find()) {
System.out.println(" Time: " + timeMatcher.group(1) + " om " + timeMatcher.group(2));
log.info(" Time: {} om {}", timeMatcher.group(1), timeMatcher.group(2));
remaining = remaining.substring(timeMatcher.end()).trim();
}
@@ -94,7 +96,7 @@ public class AuctionParsingTest {
);
var locMatcher = locPattern.matcher(remaining);
if (locMatcher.find()) {
System.out.println(" Location: " + locMatcher.group(1) + ", " + locMatcher.group(2));
log.info(" Location: {}, {}", locMatcher.group(1), locMatcher.group(2));
remaining = remaining.substring(0, locMatcher.start()).trim();
}
@@ -102,12 +104,12 @@ public class AuctionParsingTest {
var lotPattern = java.util.regex.Pattern.compile("^(\\d+)\\s+");
var lotMatcher = lotPattern.matcher(remaining);
if (lotMatcher.find()) {
System.out.println(" Lot count: " + lotMatcher.group(1));
log.info(" Lot count: {}", lotMatcher.group(1));
remaining = remaining.substring(lotMatcher.end()).trim();
}
// What remains is title
System.out.println(" Title: " + remaining);
log.info(" Title: {}", remaining);
}
}
}

4
src/test/java/auctiora/ImageProcessingServiceTest.java
View File

@@ -19,14 +19,14 @@ class ImageProcessingServiceTest {
private DatabaseService mockDb;
private ObjectDetectionService mockDetector;
private RateLimitedHttpClient mockHttpClient;
private RateLimitedHttpClient2 mockHttpClient;
private ImageProcessingService service;
@BeforeEach
void setUp() {
mockDb = mock(DatabaseService.class);
mockDetector = mock(ObjectDetectionService.class);
mockHttpClient = mock(RateLimitedHttpClient.class);
mockHttpClient = mock(RateLimitedHttpClient2.class);
service = new ImageProcessingService(mockDb, mockDetector, mockHttpClient);
}

4
src/test/java/auctiora/IntegrationTest.java
View File

@@ -40,7 +40,7 @@ class IntegrationTest {
db = new DatabaseService(testDbPath);
db.ensureSchema();
notifier = new NotificationService("desktop", "");
notifier = new NotificationService("desktop");
detector = new ObjectDetectionService(
"non_existent.cfg",
@@ -48,7 +48,7 @@ class IntegrationTest {
"non_existent.txt"
);
RateLimitedHttpClient httpClient = new RateLimitedHttpClient();
RateLimitedHttpClient2 httpClient = new RateLimitedHttpClient2();
imageProcessor = new ImageProcessingService(db, detector, httpClient);
monitor = new TroostwijkMonitor(

51
src/test/java/auctiora/NotificationServiceTest.java
View File

@@ -14,7 +14,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should initialize with desktop-only configuration")
void testDesktopOnlyConfiguration() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertNotNull(service);
}
@@ -22,8 +22,7 @@ class NotificationServiceTest {
@DisplayName("Should initialize with SMTP configuration")
void testSMTPConfiguration() {
NotificationService service = new NotificationService(
"smtp:test@gmail.com:app_password:recipient@example.com",
""
"smtp:test@gmail.com:app_password:recipient@example.com"
);
assertNotNull(service);
}
@@ -33,12 +32,12 @@ class NotificationServiceTest {
void testInvalidSMTPConfiguration() {
// Missing parts
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:incomplete", "")
new NotificationService("smtp:incomplete")
);
// Wrong format
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:only:two:parts", "")
new NotificationService("smtp:only:two:parts")
);
}
@@ -46,14 +45,14 @@ class NotificationServiceTest {
@DisplayName("Should reject unknown configuration type")
void testUnknownConfiguration() {
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("unknown_type", "")
new NotificationService("unknown_type")
);
}
@Test
@DisplayName("Should send desktop notification without error")
void testDesktopNotification() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
// Should not throw exception even if system tray not available
assertDoesNotThrow(() ->
@@ -64,7 +63,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should send high priority notification")
void testHighPriorityNotification() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("Urgent message", "High Priority", 1)
@@ -74,7 +73,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should send normal priority notification")
void testNormalPriorityNotification() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("Regular message", "Normal Priority", 0)
@@ -84,7 +83,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should handle notification when system tray not supported")
void testNoSystemTraySupport() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
// Should gracefully handle missing system tray
assertDoesNotThrow(() ->
@@ -98,8 +97,7 @@ class NotificationServiceTest {
// Note: This won't actually send email without valid credentials
// But it should initialize properly
NotificationService service = new NotificationService(
"smtp:test@gmail.com:fake_password:test@example.com",
""
"smtp:test@gmail.com:fake_password:test@example.com"
);
// Should not throw during initialization
@@ -115,8 +113,7 @@ class NotificationServiceTest {
@DisplayName("Should include both desktop and email when SMTP configured")
void testBothNotificationChannels() {
NotificationService service = new NotificationService(
"smtp:user@gmail.com:password:recipient@example.com",
""
"smtp:user@gmail.com:password:recipient@example.com"
);
// Both desktop and email should be attempted
@@ -128,7 +125,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should handle empty message gracefully")
void testEmptyMessage() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("", "", 0)
@@ -138,7 +135,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should handle very long message")
void testLongMessage() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
String longMessage = "A".repeat(1000);
assertDoesNotThrow(() ->
@@ -149,7 +146,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should handle special characters in message")
void testSpecialCharactersInMessage() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification(
@@ -164,9 +161,9 @@ class NotificationServiceTest {
@DisplayName("Should accept case-insensitive desktop config")
void testCaseInsensitiveDesktopConfig() {
assertDoesNotThrow(() -> {
new NotificationService("DESKTOP", "");
new NotificationService("Desktop", "");
new NotificationService("desktop", "");
new NotificationService("DESKTOP");
new NotificationService("Desktop");
new NotificationService("desktop");
});
}
@@ -175,19 +172,19 @@ class NotificationServiceTest {
void testSMTPConfigPartsValidation() {
// Too few parts
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:user:pass", "")
new NotificationService("smtp:user:pass")
);
// Too many parts should work (extras ignored in split)
assertDoesNotThrow(() ->
new NotificationService("smtp:user:pass:email:extra", "")
new NotificationService("smtp:user:pass:email:extra")
);
}
@Test
@DisplayName("Should handle multiple rapid notifications")
void testRapidNotifications() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() -> {
for (int i = 0; i < 5; i++) {
@@ -201,14 +198,14 @@ class NotificationServiceTest {
void testNullConfigParameter() {
// Second parameter can be empty string (kept for compatibility)
assertDoesNotThrow(() ->
new NotificationService("desktop", null)
new NotificationService("desktop")
);
}
@Test
@DisplayName("Should send bid change notification format")
void testBidChangeNotificationFormat() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
String message = "Nieuw bod op kavel 12345: €150.00 (was €125.00)";
String title = "Kavel bieding update";
@@ -221,7 +218,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should send closing alert notification format")
void testClosingAlertNotificationFormat() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
String message = "Kavel 12345 sluit binnen 5 min.";
String title = "Lot nearing closure";
@@ -234,7 +231,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should send object detection notification format")
void testObjectDetectionNotificationFormat() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
String message = "Lot contains: car, truck, machinery\nEstimated value: €5000";
String title = "Object Detected";

60
src/test/java/auctiora/Parser.java
View File

File diff suppressed because one or more lines are too long

62
src/test/java/auctiora/ParserTest.java Normal file
View File

File diff suppressed because one or more lines are too long

10
src/test/java/auctiora/ScraperDataAdapterTest.java
View File

@@ -55,9 +55,9 @@ class ScraperDataAdapterTest {
assertEquals("Cluj-Napoca", result.city());
assertEquals("RO", result.country());
assertEquals("https://example.com/auction/A7-39813", result.url());
assertEquals("A7", result.type());
assertEquals("A7", result.typePrefix());
assertEquals(150, result.lotCount());
assertNotNull(result.closingTime());
assertNotNull(result.firstLotClosingTime());
}
@Test
@@ -75,7 +75,7 @@ class ScraperDataAdapterTest {
assertEquals("Amsterdam", result.city());
assertEquals("", result.country());
assertNull(result.closingTime());
assertNull(result.firstLotClosingTime());
}
@Test
@@ -196,7 +196,7 @@ class ScraperDataAdapterTest {
when(rs1.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo auction1 = ScraperDataAdapter.fromScraperAuction(rs1);
assertEquals("A7", auction1.type());
assertEquals("A7", auction1.typePrefix());
ResultSet rs2 = mock(ResultSet.class);
when(rs2.getString("auction_id")).thenReturn("B1-12345");
@@ -207,7 +207,7 @@ class ScraperDataAdapterTest {
when(rs2.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo auction2 = ScraperDataAdapter.fromScraperAuction(rs2);
assertEquals("B1", auction2.type());
assertEquals("B1", auction2.typePrefix());
}
@Test

48
src/test/java/auctiora/TroostwijkMonitorTest.java
View File

@@ -43,7 +43,7 @@ class TroostwijkMonitorTest {
@DisplayName("Should initialize monitor successfully")
void testMonitorInitialization() {
assertNotNull(monitor);
assertNotNull(monitor.db);
assertNotNull(monitor.getDb());
}
@Test
@@ -61,8 +61,8 @@ class TroostwijkMonitorTest {
@Test
@DisplayName("Should handle empty database gracefully")
void testEmptyDatabaseHandling() throws SQLException {
var auctions = monitor.db.getAllAuctions();
var lots = monitor.db.getAllLots();
var auctions = monitor.getDb().getAllAuctions();
var lots = monitor.getDb().getAllLots();
assertNotNull(auctions);
assertNotNull(lots);
@@ -88,9 +88,9 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(lot);
monitor.getDb().upsertLot(lot);
var lots = monitor.db.getAllLots();
var lots = monitor.getDb().getAllLots();
assertTrue(lots.stream().anyMatch(l -> l.lotId() == 22222));
}
@@ -113,9 +113,9 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(closingSoon);
monitor.getDb().upsertLot(closingSoon);
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 44444)
.findFirst()
@@ -143,9 +143,9 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(futureLot);
monitor.getDb().upsertLot(futureLot);
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 66666)
.findFirst()
@@ -173,9 +173,9 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(noClosing);
monitor.getDb().upsertLot(noClosing);
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 88888)
.findFirst()
@@ -203,7 +203,7 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(lot);
monitor.getDb().upsertLot(lot);
// Update notification flag
var notified = new Lot(
@@ -221,9 +221,9 @@ class TroostwijkMonitorTest {
true
);
monitor.db.updateLotNotificationFlags(notified);
monitor.getDb().updateLotNotificationFlags(notified);
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 11110)
.findFirst()
@@ -251,7 +251,7 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(lot);
monitor.getDb().upsertLot(lot);
// Simulate bid increase
var higherBid = new Lot(
@@ -269,9 +269,9 @@ class TroostwijkMonitorTest {
false
);
monitor.db.updateLotCurrentBid(higherBid);
monitor.getDb().updateLotCurrentBid(higherBid);
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 13131)
.findFirst()
@@ -287,7 +287,7 @@ class TroostwijkMonitorTest {
Thread t1 = new Thread(() -> {
try {
for (int i = 0; i < 5; i++) {
monitor.db.upsertLot(new Lot(
monitor.getDb().upsertLot(new Lot(
20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
100.0, "EUR", "https://example.com/" + i, null, false
));
@@ -300,7 +300,7 @@ class TroostwijkMonitorTest {
Thread t2 = new Thread(() -> {
try {
for (int i = 5; i < 10; i++) {
monitor.db.upsertLot(new Lot(
monitor.getDb().upsertLot(new Lot(
20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
200.0, "EUR", "https://example.com/" + i, null, false
));
@@ -315,7 +315,7 @@ class TroostwijkMonitorTest {
t1.join();
t2.join();
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
long count = lots.stream()
.filter(l -> l.lotId() >= 30000 && l.lotId() < 30010)
.count();
@@ -351,7 +351,7 @@ class TroostwijkMonitorTest {
LocalDateTime.now().plusDays(2)
);
monitor.db.upsertAuction(auction);
monitor.getDb().upsertAuction(auction);
// Insert related lot
var lot = new Lot(
@@ -369,11 +369,11 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(lot);
monitor.getDb().upsertLot(lot);
// Verify
var auctions = monitor.db.getAllAuctions();
var lots = monitor.db.getAllLots();
var auctions = monitor.getDb().getAllAuctions();
var lots = monitor.getDb().getAllLots();
assertTrue(auctions.stream().anyMatch(a -> a.auctionId() == 40000));
assertTrue(lots.stream().anyMatch(l -> l.lotId() == 50000));

20
workflows/maven.yml Normal file
View File

@@ -0,0 +1,20 @@
name: Publish to Gitea Package Registry
on:
push:
tags:
- 'v*'
jobs:
publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up JDK 25
uses: actions/setup-java@v4
with:
java-version: '25'
distribution: 'temurin'
- name: Publish with Maven
run: mvn --batch-mode clean deploy
env:
GITEA_TOKEN: ${{ secrets.EA_PUBLISH_TOKEN }}