Tour/auctiora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix mock tests

Browse Source
This commit is contained in:
Tour
2025-12-04 19:38:38 +01:00
parent 9857f053a1
commit ed74bb5e93
34 changed files with 2312 additions and 2006 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
.idea/compiler.xml generated
View File

@@ -2,17 +2,37 @@
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile default="true" name="Default" enabled="true" />
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="troostwijk-scraper" />
</profile>
<profile name="Annotation profile for Troostwijk Auction Scraper" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<processorPath useClasspath="false">
<entry name="$MAVEN_REPOSITORY$/org/projectlombok/lombok/1.18.40/lombok-1.18.40.jar" />
<entry name="$MAVEN_REPOSITORY$/io/quarkus/quarkus-extension-processor/3.17.7/quarkus-extension-processor-3.17.7.jar" />
<entry name="$MAVEN_REPOSITORY$/org/jboss/jdeparser/jdeparser/2.0.3.Final/jdeparser-2.0.3.Final.jar" />
<entry name="$MAVEN_REPOSITORY$/org/jsoup/jsoup/1.15.3/jsoup-1.15.3.jar" />
<entry name="$MAVEN_REPOSITORY$/com/github/javaparser/javaparser-core/3.26.2/javaparser-core-3.26.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-databind/2.18.2/jackson-databind-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-annotations/2.18.2/jackson-annotations-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-core/2.18.2/jackson-core-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/dataformat/jackson-dataformat-yaml/2.18.2/jackson-dataformat-yaml-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/org/yaml/snakeyaml/2.3/snakeyaml-2.3.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/module/jackson-module-parameter-names/2.18.2/jackson-module-parameter-names-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/io/quarkus/quarkus-bootstrap-app-model/3.17.7/quarkus-bootstrap-app-model-3.17.7.jar" />
</processorPath>
<module name="auctiora" />
</profile>
</annotationProcessing>
</component>
<component name="JavacSettings">
<option name="ADDITIONAL_OPTIONS_OVERRIDE">
<module name="troostwijk-scraper" options="" />
<module name="auctiora" options="-Xdiags:verbose -Xlint:all -parameters" />
</option>
</component>
</project>

5
.idea/jarRepositories.xml generated
View File

@@ -1,6 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="RemoteRepositoriesConfiguration">
<remote-repository>
<option name="id" value="jitpack.io" />
<option name="name" value="jitpack.io" />
<option name="url" value="https://jitpack.io" />
</remote-repository>
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Apache Central Repository" />

67
Dockerfile
View File

@@ -1,52 +1,27 @@
# Multi-stage Dockerfile for Quarkus Auction Monitor
# Build stage
FROM maven:3.9-eclipse-temurin-25-alpine AS build
# ==================== BUILD STAGE ====================
FROM maven:3.9-eclipse-temurin-25-alpine AS builder
WORKDIR /app
# Copy POM first (allows for cached dependency layer)
COPY pom.xml .
# This will now work if the opencv dependency has no classifier
# -----LOCAL----
RUN mvn dependency:resolve -B
# -----LOCAL----
# RUN mvn dependency:go-offline -B
# Copy Maven files for dependency caching
COPY pom.xml ./
RUN mvn dependency:go-offline -B
# Copy source code
COPY src/ ./src/
# Build Quarkus application (fast-jar for production)
RUN mvn package -DskipTests -Dquarkus.package.jar.type=fast-jar
# Runtime stage
FROM eclipse-temurin:25-jre-alpine
COPY src ./src
# Updated with both properties to avoid the warning
RUN mvn package -DskipTests -Dquarkus.package.jar.type=uber-jar -Dquarkus.package.jar.enabled=true
# ==================== RUNTIME STAGE ====================
FROM eclipse-temurin:25-jre
WORKDIR /app
# Create non-root user
RUN addgroup -g 1001 quarkus && \
adduser -u 1001 -G quarkus -s /bin/sh -D quarkus
# Create directories for data
RUN mkdir -p /mnt/okcomputer/output/images && \
chown -R quarkus:quarkus /mnt/okcomputer
# Copy Quarkus fast-jar structure
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/lib/ /app/lib/
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/*.jar /app/
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/app/ /app/app/
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/quarkus/ /app/quarkus/
# Switch to non-root user
RUN groupadd -r quarkus && useradd -r -g quarkus quarkusa
COPY --from=builder --chown=quarkus:quarkus /app/target/scrape-ui-*.jar app.jar
USER quarkus
# Expose ports
EXPOSE 8081
# Set environment variables
ENV JAVA_OPTS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager"
ENV JAVA_APP_JAR="/app/quarkus-run.jar"
# Health check
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD wget --no-verbose --tries=1 --spider http://localhost:8081/health/live || exit 1
# Run the Quarkus application
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar $JAVA_APP_JAR"]
ENTRYPOINT ["java", \
"-Dio.netty.tryReflectionSetAccessible=true", \
"--enable-native-access=ALL-UNNAMED", \
"--sun-misc-unsafe-memory-access=allow", \
"-jar", "app.jar"]

11
auction.iml
View File

@@ -1,11 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

11
docker-compose.yml
View File

@@ -1,11 +1,8 @@
version: '3.8'
services:
auction-monitor:
sophena:
build:
context: .
dockerfile: Dockerfile
container_name: troostwijk-auction-monitor
dockerfile: .
container_name: sophena
ports:
- "8081:8081"
volumes:
@@ -39,6 +36,7 @@ services:
- AUCTION_WORKFLOW_IMAGE_PROCESSING_CRON=0 0 * * * ?
- AUCTION_WORKFLOW_BID_MONITORING_CRON=0 */15 * * * ?
- AUCTION_WORKFLOW_CLOSING_ALERTS_CRON=0 */5 * * * ?
- JAVA_TOOL_OPTIONS=-Dio.netty.tryReflectionSetAccessible=true --enable-native-access=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8081/health/live"]
@@ -52,6 +50,7 @@ services:
networks:
- auction-network
networks:
auction-network:
driver: bridge

204
pom.xml
View File

@@ -11,16 +11,39 @@
<name>Troostwijk Auction Scraper</name>
<description>Web scraper for Troostwijk Auctions with object detection and notifications</description>
<repositories>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
</repositories>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>25</maven.compiler.source>
<maven.compiler.target>25</maven.compiler.target>
<maven.compiler.release>25</maven.compiler.release>
<jackson.version>2.17.0</jackson.version>
<opencv.version>4.9.0-0</opencv.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<quarkus.platform.version>3.17.7</quarkus.platform.version>
<asm.version>9.8</asm.version>
<lombok.version>1.18.40</lombok.version>
<!--this is not a bug, its feature-->
<lombok-version>${lombok.version}</lombok-version>
<lombok-maven-version>1.18.20.0</lombok-maven-version>
<maven-compiler-plugin-version>3.14.0</maven-compiler-plugin-version>
<versions-maven-plugin.version>2.19.0</versions-maven-plugin.version>
<jandex-maven-plugin-version>3.5.0</jandex-maven-plugin-version>
<maven.compiler.args>
--enable-native-access=ALL-UNNAMED
--add-opens java.base/sun.misc=ALL-UNNAMED
-Xdiags:verbose
-Xlint:all
</maven.compiler.args>
<uberJar>true</uberJar> <!-- Your existing properties... -->
<quarkus.package.jar.type>uber-jar</quarkus.package.jar.type>
<quarkus.package.jar.enabled>true</quarkus.package.jar.enabled>
<maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
</properties>
<dependencyManagement>
@@ -53,10 +76,28 @@
<artifactId>asm-util</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>4.1.124.Final</version> <!-- This version has the fix -->
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.github.vladimir-bukhtoyarov.bucket4j</groupId>
<artifactId>bucket4j-core</artifactId>
<version>7.6.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.github.bucket4j/bucket4j -->
<!--<dependency>
<groupId>io.github.bucket4j</groupId>
<artifactId>bucket4j</artifactId>
<version>8.9.0</version>
</dependency>-->
<!-- JSoup for HTML parsing and HTTP client -->
<dependency>
<groupId>org.jsoup</groupId>
@@ -77,7 +118,19 @@
<artifactId>sqlite-jdbc</artifactId>
<version>3.45.1.0</version>
</dependency>
<!-- Lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven</artifactId>
<version>${lombok-maven-version}</version>
<type>pom</type>
</dependency>
<!-- JavaMail API for email notifications -->
<dependency>
<groupId>com.sun.mail</groupId>
@@ -85,12 +138,6 @@
<version>1.6.2</version>
</dependency>
<!-- OpenCV for image processing and object detection -->
<dependency>
<groupId>org.openpnp</groupId>
<artifactId>opencv</artifactId>
<version>${opencv.version}</version>
</dependency>
<dependency>
<groupId>com.microsoft.playwright</groupId>
<artifactId>playwright</artifactId>
@@ -167,6 +214,14 @@
<artifactId>quarkus-config-yaml</artifactId>
</dependency>
<!-- OSGi annotations -->
<dependency>
<groupId>org.osgi</groupId>
<artifactId>org.osgi.annotation.bundle</artifactId>
<version>2.0.0</version>
<scope>provided</scope>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>io.quarkus</groupId>
@@ -178,10 +233,23 @@
<artifactId>rest-assured</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.openpnp</groupId>
<artifactId>opencv</artifactId>
<version>4.9.0-0</version>
<!--<classifier>windows-x86_64</classifier>-->
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>io.quarkus.platform</groupId>
<artifactId>quarkus-maven-plugin</artifactId>
@@ -196,46 +264,74 @@
</goals>
</execution>
</executions>
</plugin>
<!-- Maven Compiler Plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.11.0</version>
<configuration>
<source>25</source>
<target>25</target>
<properties>
<build.timestamp>${maven.build.timestamp}</build.timestamp>
</properties>
</configuration>
</plugin>
<!-- Maven Exec Plugin for running with native access -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<mainClass>com.auction.TroostwijkAuctionExtractor</mainClass>
<cleanupDaemonThreads>false</cleanupDaemonThreads>
<arguments>
<argument>--max-visits</argument>
<argument>3</argument>
</arguments>
<additionalClasspathElements>
<!--suppress MavenModelInspection -->
<additionalClasspathElement>${project.build.outputDirectory}</additionalClasspathElement>
</additionalClasspathElements>
<commandlineArgs>--enable-native-access=ALL-UNNAMED</commandlineArgs>
</configuration>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven-plugin</artifactId>
<version>${lombok-maven-version}</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>java</goal>
<goal>delombok</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Maven Exec Plugin for running with native access -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin-version}</version>
<configuration>
<release>${maven.compiler.release}</release>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok-version}</version>
</path>
<path>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-extension-processor</artifactId>
<version>${quarkus.platform.version}</version>
</path>
</annotationProcessorPaths>
<compilerArgs>
<arg>-Xdiags:verbose</arg>
<arg>-Xlint:all</arg>
<arg>-parameters</arg>
</compilerArgs>
<fork>true</fork>
<excludes>
<exclude>module-info.java</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
<version>${versions-maven-plugin.version}</version>
</plugin>
<!-- Maven Surefire Plugin for tests with native access -->
<plugin>
<groupId>io.smallrye</groupId>
<artifactId>jandex-maven-plugin</artifactId>
<version>${jandex-maven-plugin-version}</version>
<executions>
<execution>
<id>make-index</id>
<goals>
<goal>jandex</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
@@ -262,31 +358,17 @@
</archive>
</configuration>
</plugin>-->
<!-- Maven Assembly Plugin for creating executable JAR with dependencies -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<archive>
<manifest>
<mainClass>com.auction.Main</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<!-- In your pom.xml, alongside <build> and <dependencies> -->
<distributionManagement>
<repository>
<id>gitea</id>
<url>https://git.appmodel.nl/api/packages/Tour/maven</url>
</repository>
<snapshotRepository>
<id>gitea</id>
<url>https://git.appmodel.nl/api/packages/Tour/maven</url>
</snapshotRepository>
</distributionManagement>
</project>

6
src/main/java/auctiora/AuctionInfo.java
View File

@@ -13,7 +13,7 @@ public record AuctionInfo(
String city, // City name
String country, // Country code (e.g., "NL")
String url, // Full auction URL
String type, // Auction type (A1 or A7)
String typePrefix, // Auction type (A1 or A7)
int lotCount, // Number of lots/kavels
LocalDateTime closingTime // Closing time if available
) {}
LocalDateTime firstLotClosingTime // Closing time if available
) { }

4
src/main/java/auctiora/AuctionMonitorProducer.java
View File

@@ -35,7 +35,7 @@ public class AuctionMonitorProducer {
@ConfigProperty(name = "auction.notification.config") String config) {
LOG.infof("Initializing NotificationService with config: %s", config);
return new NotificationService(config, "");
return new NotificationService(config);
}
@Produces
@@ -54,7 +54,7 @@ public class AuctionMonitorProducer {
public ImageProcessingService produceImageProcessingService(
DatabaseService db,
ObjectDetectionService detector,
RateLimitedHttpClient httpClient) {
RateLimitedHttpClient2 httpClient) {
LOG.infof("Initializing ImageProcessingService");
return new ImageProcessingService(db, detector, httpClient);

2
src/main/java/auctiora/AuctionMonitorResource.java
View File

@@ -33,7 +33,7 @@ public class AuctionMonitorResource {
NotificationService notifier;
@Inject
RateLimitedHttpClient httpClient;
RateLimitedHttpClient2 httpClient;
/**
* GET /api/monitor/status

10
src/main/java/auctiora/Console.java
View File

@@ -1,10 +0,0 @@
package auctiora;
/**
* Simple console output utility (renamed from IO to avoid Java 25 conflict)
*/
class Console {
static void println(String message) {
System.out.println(message);
}
}

15
src/main/java/auctiora/DatabaseService.java
View File

@@ -1,5 +1,7 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.io.Console;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.time.Instant;
@@ -12,6 +14,7 @@ import java.util.List;
* Data is typically populated by an external scraper process;
* this service enriches it with image processing and monitoring.
*/
@Slf4j
public class DatabaseService {
private final String url;
@@ -104,9 +107,9 @@ public class DatabaseService {
ps.setString(4, auction.city());
ps.setString(5, auction.country());
ps.setString(6, auction.url());
ps.setString(7, auction.type());
ps.setString(7, auction.typePrefix());
ps.setInt(8, auction.lotCount());
ps.setString(9, auction.closingTime() != null ? auction.closingTime().toString() : null);
ps.setString(9, auction.firstLotClosingTime() != null ? auction.firstLotClosingTime().toString() : null);
ps.setLong(10, Instant.now().getEpochSecond());
ps.executeUpdate();
}
@@ -383,7 +386,7 @@ public class DatabaseService {
}
} catch (SQLException e) {
// Table might not exist in scraper format - that's ok
Console.println("ℹ️ Scraper lots table not found or incompatible schema");
log.info("ℹ️ Scraper lots table not found or incompatible schema");
}
return imported;
@@ -421,7 +424,7 @@ public class DatabaseService {
));
}
} catch (SQLException e) {
Console.println("ℹ️ No unprocessed images found in scraper format");
log.info("ℹ️ No unprocessed images found in scraper format");
}
return images;
@@ -430,10 +433,10 @@ public class DatabaseService {
/**
* Simple record for image data from database
*/
record ImageRecord(int id, int lotId, String url, String filePath, String labels) {}
record ImageRecord(int id, int lotId, String url, String filePath, String labels) { }
/**
* Record for importing images from scraper format
*/
record ImageImportRecord(int lotId, int saleId, String url) {}
record ImageImportRecord(int lotId, int saleId, String url) { }
}

17
src/main/java/auctiora/ImageProcessingService.java
View File

@@ -1,5 +1,7 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.io.Console;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -13,13 +15,14 @@ import java.util.List;
* This separates image processing concerns from scraping, allowing this project
* to focus on enriching data scraped by the external process.
*/
@Slf4j
class ImageProcessingService {
private final RateLimitedHttpClient httpClient;
private final RateLimitedHttpClient2 httpClient;
private final DatabaseService db;
private final ObjectDetectionService detector;
ImageProcessingService(DatabaseService db, ObjectDetectionService detector, RateLimitedHttpClient httpClient) {
ImageProcessingService(DatabaseService db, ObjectDetectionService detector, RateLimitedHttpClient2 httpClient) {
this.httpClient = httpClient;
this.db = db;
this.detector = detector;
@@ -73,7 +76,7 @@ class ImageProcessingService {
* @param imageUrls list of image URLs to process
*/
void processImagesForLot(int lotId, int saleId, List<String> imageUrls) {
Console.println(" Processing " + imageUrls.size() + " images for lot " + lotId);
log.info(" Processing {} images for lot {}", imageUrls.size(), lotId);
for (var imgUrl : imageUrls) {
var fileName = downloadImage(imgUrl, saleId, lotId);
@@ -87,7 +90,7 @@ class ImageProcessingService {
db.insertImage(lotId, imgUrl, fileName, labels);
if (!labels.isEmpty()) {
Console.println(" Detected: " + String.join(", ", labels));
log.info(" Detected: {}", String.join(", ", labels));
}
} catch (SQLException e) {
System.err.println(" Failed to save image to database: " + e.getMessage());
@@ -101,18 +104,18 @@ class ImageProcessingService {
* Useful for processing images after the external scraper has populated lot data.
*/
void processPendingImages() {
Console.println("Processing pending images...");
log.info("Processing pending images...");
try {
var lots = db.getAllLots();
Console.println("Found " + lots.size() + " lots to check for images");
log.info("Found {} lots to check for images", lots.size());
for (var lot : lots) {
// Check if images already processed for this lot
var existingImages = db.getImagesForLot(lot.lotId());
if (existingImages.isEmpty()) {
Console.println(" Lot " + lot.lotId() + " has no images yet - needs external scraper data");
log.info(" Lot {} has no images yet - needs external scraper data", lot.lotId());
}
}

17
src/main/java/auctiora/Lot.java
View File

@@ -1,5 +1,6 @@
package auctiora;
import lombok.With;
import java.time.Duration;
import java.time.LocalDateTime;
@@ -8,6 +9,7 @@ import java.time.LocalDateTime;
* Data typically populated by the external scraper process.
* This project enriches the data with image analysis and monitoring.
*/
@With
record Lot(
int saleId,
int lotId,
@@ -23,8 +25,21 @@ record Lot(
LocalDateTime closingTime,
boolean closingNotified
) {
long minutesUntilClose() {
public long minutesUntilClose() {
if (closingTime == null) return Long.MAX_VALUE;
return Duration.between(LocalDateTime.now(), closingTime).toMinutes();
}
public Lot withCurrentBid(double newBid) {
return new Lot(saleId, lotId, title, description,
manufacturer, type, year, category,
newBid, currency, url, closingTime, closingNotified);
}
public Lot withClosingNotified(boolean flag) {
return new Lot(saleId, lotId, title, description,
manufacturer, type, year, category,
currentBid, currency, url, closingTime, flag);
}
}

84
src/main/java/auctiora/Main.java
View File

@@ -1,5 +1,6 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import org.opencv.core.Core;
/**
@@ -19,10 +20,11 @@ import org.opencv.core.Core;
* - Bid monitoring
* - Notifications
*/
@Slf4j
public class Main {
public static void main(String[] args) throws Exception {
Console.println("=== Troostwijk Auction Monitor ===\n");
log.info("=== Troostwijk Auction Monitor ===\n");
// Parse command line arguments
String mode = args.length > 0 ? args[0] : "workflow";
@@ -39,9 +41,9 @@ public class Main {
// Load native OpenCV library (only if models exist)
try {
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
Console.println("✓ OpenCV loaded");
log.info("✓ OpenCV loaded");
} catch (UnsatisfiedLinkError e) {
Console.println("⚠️ OpenCV not available - image detection disabled");
log.info("⚠️ OpenCV not available - image detection disabled");
}
switch (mode.toLowerCase()) {
@@ -75,7 +77,7 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
Console.println("🚀 Starting in WORKFLOW MODE (Orchestrated Scheduling)\n");
log.info("🚀 Starting in WORKFLOW MODE (Orchestrated Scheduling)\n");
WorkflowOrchestrator orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
@@ -87,16 +89,16 @@ public class Main {
// Start all scheduled workflows
orchestrator.startScheduledWorkflows();
Console.println("✓ All workflows are running");
Console.println(" - Scraper import: every 30 min");
Console.println(" - Image processing: every 1 hour");
Console.println(" - Bid monitoring: every 15 min");
Console.println(" - Closing alerts: every 5 min");
Console.println("\nPress Ctrl+C to stop.\n");
log.info("✓ All workflows are running");
log.info(" - Scraper import: every 30 min");
log.info(" - Image processing: every 1 hour");
log.info(" - Bid monitoring: every 15 min");
log.info(" - Closing alerts: every 5 min");
log.info("\nPress Ctrl+C to stop.\n");
// Add shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
Console.println("\n🛑 Shutdown signal received...");
log.info("\n🛑 Shutdown signal received...");
orchestrator.shutdown();
}));
@@ -117,7 +119,7 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
Console.println("🔄 Starting in ONCE MODE (Single Execution)\n");
log.info("🔄 Starting in ONCE MODE (Single Execution)\n");
WorkflowOrchestrator orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
@@ -125,7 +127,7 @@ public class Main {
orchestrator.runCompleteWorkflowOnce();
Console.println("✓ Workflow execution completed. Exiting.\n");
log.info("✓ Workflow execution completed. Exiting.\n");
}
/**
@@ -136,29 +138,29 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
Console.println("⚙️ Starting in LEGACY MODE\n");
log.info("⚙️ Starting in LEGACY MODE\n");
var monitor = new TroostwijkMonitor(dbPath, notifConfig,
yoloCfg, yoloWeights, yoloClasses);
Console.println("\n📊 Current Database State:");
log.info("\n📊 Current Database State:");
monitor.printDatabaseStats();
Console.println("\n[1/2] Processing images...");
log.info("\n[1/2] Processing images...");
monitor.processPendingImages();
Console.println("\n[2/2] Starting bid monitoring...");
log.info("\n[2/2] Starting bid monitoring...");
monitor.scheduleMonitoring();
Console.println("\n✓ Monitor is running. Press Ctrl+C to stop.\n");
Console.println("NOTE: This process expects auction/lot data from the external scraper.");
Console.println(" Make sure ARCHITECTURE-TROOSTWIJK-SCRAPER is running and populating the database.\n");
log.info("\n✓ Monitor is running. Press Ctrl+C to stop.\n");
log.info("NOTE: This process expects auction/lot data from the external scraper.");
log.info(" Make sure ARCHITECTURE-TROOSTWIJK-SCRAPER is running and populating the database.\n");
try {
Thread.sleep(Long.MAX_VALUE);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
Console.println("Monitor interrupted, exiting.");
log.info("Monitor interrupted, exiting.");
}
}
@@ -169,7 +171,7 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception {
Console.println("📊 Checking Status...\n");
log.info("📊 Checking Status...\n");
WorkflowOrchestrator orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
@@ -182,21 +184,21 @@ public class Main {
* Show usage information
*/
private static void showUsage() {
Console.println("Usage: java -jar troostwijk-monitor.jar [mode]\n");
Console.println("Modes:");
Console.println(" workflow - Run orchestrated scheduled workflows (default)");
Console.println(" once - Run complete workflow once and exit (for cron)");
Console.println(" legacy - Run original monitoring approach");
Console.println(" status - Show current status and exit");
Console.println("\nEnvironment Variables:");
Console.println(" DATABASE_FILE - Path to SQLite database");
Console.println(" (default: C:\\mnt\\okcomputer\\output\\cache.db)");
Console.println(" NOTIFICATION_CONFIG - 'desktop' or 'smtp:user:pass:email'");
Console.println(" (default: desktop)");
Console.println("\nExamples:");
Console.println(" java -jar troostwijk-monitor.jar workflow");
Console.println(" java -jar troostwijk-monitor.jar once");
Console.println(" java -jar troostwijk-monitor.jar status");
log.info("Usage: java -jar troostwijk-monitor.jar [mode]\n");
log.info("Modes:");
log.info(" workflow - Run orchestrated scheduled workflows (default)");
log.info(" once - Run complete workflow once and exit (for cron)");
log.info(" legacy - Run original monitoring approach");
log.info(" status - Show current status and exit");
log.info("\nEnvironment Variables:");
log.info(" DATABASE_FILE - Path to SQLite database");
log.info(" (default: C:\\mnt\\okcomputer\\output\\cache.db)");
log.info(" NOTIFICATION_CONFIG - 'desktop' or 'smtp:user:pass:email'");
log.info(" (default: desktop)");
log.info("\nExamples:");
log.info(" java -jar troostwijk-monitor.jar workflow");
log.info(" java -jar troostwijk-monitor.jar once");
log.info(" java -jar troostwijk-monitor.jar status");
IO.println();
}
@@ -206,18 +208,18 @@ public class Main {
*/
public static void main2(String[] args) {
if (args.length > 0) {
Console.println("Command mode - exiting to allow shell commands");
log.info("Command mode - exiting to allow shell commands");
return;
}
Console.println("Troostwijk Monitor container is running and healthy.");
Console.println("Use 'docker exec' or 'dokku run' to execute commands.");
log.info("Troostwijk Monitor container is running and healthy.");
log.info("Use 'docker exec' or 'dokku run' to execute commands.");
try {
Thread.sleep(Long.MAX_VALUE);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
Console.println("Container interrupted, exiting.");
log.info("Container interrupted, exiting.");
}
}
}

174
src/main/java/auctiora/NotificationService.java
View File

@@ -1,122 +1,48 @@
package auctiora;
import javax.mail.Authenticator;
import javax.mail.Message.RecipientType;
import javax.mail.PasswordAuthentication;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import java.awt.SystemTray;
import java.awt.Toolkit;
import java.awt.TrayIcon;
import java.awt.TrayIcon.MessageType;
import javax.mail.*;
import javax.mail.internet.*;
import lombok.extern.slf4j.Slf4j;
import java.awt.*;
import java.util.Date;
import java.util.Properties;
/**
* Service for sending notifications via desktop notifications and/or email.
* Supports free notification methods:
* 1. Desktop notifications (Windows/Linux/macOS system tray)
* 2. Email via Gmail SMTP (free, requires app password)
*
* Configuration:
* - For email: Set notificationEmail to your Gmail address
* - Enable 2FA in Gmail and create an App Password
* - Use format "smtp:username:appPassword:toEmail" for credentials
* - Or use "desktop" for desktop-only notifications
*/
class NotificationService {
private final boolean useDesktop;
private final boolean useEmail;
private final String smtpUsername;
private final String smtpPassword;
private final String toEmail;
@Slf4j
public class NotificationService {
/**
* Creates a notification service.
*
* @param config "desktop" for desktop only, or "smtp:username:password:toEmail" for email
* @param unusedParam Kept for compatibility (can pass empty string)
*/
NotificationService(String config, String unusedParam) {
private final Config config;
if ("desktop".equalsIgnoreCase(config)) {
this.useDesktop = true;
this.useEmail = false;
this.smtpUsername = null;
this.smtpPassword = null;
this.toEmail = null;
} else if (config.startsWith("smtp:")) {
var parts = config.split(":", 4);
if (parts.length != 4) {
throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'");
}
this.useDesktop = true; // Always include desktop
this.useEmail = true;
this.smtpUsername = parts[1];
this.smtpPassword = parts[2];
this.toEmail = parts[3];
} else {
throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'");
}
public NotificationService(String cfg) {
this.config = Config.parse(cfg);
}
/**
* Sends notification via configured channels.
*
* @param message The message body
* @param title Message title
* @param priority Priority level (0=normal, 1=high)
*/
void sendNotification(String message, String title, int priority) {
if (useDesktop) {
sendDesktopNotification(title, message, priority);
}
if (useEmail) {
sendEmailNotification(title, message, priority);
}
public void sendNotification(String message, String title, int priority) {
if (config.useDesktop()) sendDesktop(title, message, priority);
if (config.useEmail()) sendEmail(title, message, priority);
}
/**
* Sends a desktop notification using system tray.
* Works on Windows, macOS, and Linux with desktop environments.
*/
private void sendDesktopNotification(String title, String message, int priority) {
private void sendDesktop(String title, String msg, int prio) {
try {
if (SystemTray.isSupported()) {
if (!SystemTray.isSupported()) {
log.info("Desktop notifications not supported — " + title + " / " + msg);
return;
}
var tray = SystemTray.getSystemTray();
var image = Toolkit.getDefaultToolkit()
.createImage(new byte[0]); // Empty image
var trayIcon = new TrayIcon(image, "Troostwijk Scraper");
var image = Toolkit.getDefaultToolkit().createImage(new byte[0]);
var trayIcon = new TrayIcon(image, "NotificationService");
trayIcon.setImageAutoSize(true);
var messageType = priority > 0
? MessageType.WARNING
: MessageType.INFO;
var type = prio > 0 ? TrayIcon.MessageType.WARNING : TrayIcon.MessageType.INFO;
tray.add(trayIcon);
trayIcon.displayMessage(title, message, messageType);
// Remove icon after 2 seconds to avoid clutter
trayIcon.displayMessage(title, msg, type);
Thread.sleep(2000);
tray.remove(trayIcon);
Console.println("Desktop notification sent: " + title);
} else {
Console.println("Desktop notifications not supported, logging: " + title + " - " + message);
}
log.info("Desktop notification sent: " + title);
} catch (Exception e) {
System.err.println("Desktop notification failed: " + e.getMessage());
System.err.println("Desktop notification failed: " + e);
}
}
/**
* Sends email notification via Gmail SMTP (free).
* Uses Gmail's SMTP server with app password authentication.
*/
private void sendEmailNotification(String title, String message, int priority) {
private void sendEmail(String title, String msg, int prio) {
try {
var props = new Properties();
props.put("mail.smtp.auth", "true");
@@ -125,32 +51,48 @@ class NotificationService {
props.put("mail.smtp.port", "587");
props.put("mail.smtp.ssl.trust", "smtp.gmail.com");
var session = Session.getInstance(props,
new Authenticator() {
var session = Session.getInstance(props, new Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(smtpUsername, smtpPassword);
return new PasswordAuthentication(config.smtpUsername(), config.smtpPassword());
}
});
var msg = new MimeMessage(session);
msg.setFrom(new InternetAddress(smtpUsername));
msg.setRecipients(RecipientType.TO,
InternetAddress.parse(toEmail));
msg.setSubject("[Troostwijk] " + title);
msg.setText(message);
msg.setSentDate(new Date());
if (priority > 0) {
msg.setHeader("X-Priority", "1");
msg.setHeader("Importance", "High");
var m = new MimeMessage(session);
m.setFrom(new InternetAddress(config.smtpUsername()));
m.setRecipients(Message.RecipientType.TO, InternetAddress.parse(config.toEmail()));
m.setSubject("[Troostwijk] " + title);
m.setText(msg);
m.setSentDate(new Date());
if (prio > 0) {
m.setHeader("X-Priority", "1");
m.setHeader("Importance", "High");
}
Transport.send(m);
log.info("Email notification sent: " + title);
} catch (Exception e) {
log.info("Email notification failed: " + e);
}
}
Transport.send(msg);
Console.println("Email notification sent: " + title);
private record Config(
boolean useDesktop,
boolean useEmail,
String smtpUsername,
String smtpPassword,
String toEmail
) {
} catch (Exception e) {
System.err.println("Email notification failed: " + e.getMessage());
static Config parse(String cfg) {
if ("desktop".equalsIgnoreCase(cfg)) {
return new Config(true, false, null, null, null);
} else if (cfg.startsWith("smtp:")) {
var parts = cfg.split(":", 4);
if (parts.length != 4)
throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'");
return new Config(true, true, parts[1], parts[2], parts[3]);
}
throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'");
}
}
}

20
src/main/java/auctiora/ObjectDetectionService.java
View File

@@ -1,13 +1,16 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import org.opencv.core.Mat;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgcodecs.Imgcodecs;
import java.io.Console;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
@@ -24,7 +27,8 @@ import static org.opencv.dnn.Dnn.DNN_TARGET_CPU;
* If model files are not found, the service operates in disabled mode
* and returns empty lists.
*/
class ObjectDetectionService {
@Slf4j
public class ObjectDetectionService {
private final Net net;
private final List<String> classNames;
@@ -37,12 +41,12 @@ class ObjectDetectionService {
var classNamesFile = Paths.get(classNamesPath);
if (!Files.exists(cfgFile) || !Files.exists(weightsFile) || !Files.exists(classNamesFile)) {
Console.println("⚠️ Object detection disabled: YOLO model files not found");
Console.println(" Expected files:");
Console.println(" - " + cfgPath);
Console.println(" - " + weightsPath);
Console.println(" - " + classNamesPath);
Console.println(" Scraper will continue without image analysis.");
log.info("⚠️ Object detection disabled: YOLO model files not found");
log.info(" Expected files:");
log.info(" - " + cfgPath);
log.info(" - " + weightsPath);
log.info(" - " + classNamesPath);
log.info(" Scraper will continue without image analysis.");
this.enabled = false;
this.net = null;
this.classNames = new ArrayList<>();
@@ -57,7 +61,7 @@ class ObjectDetectionService {
// Load class names (one per line)
this.classNames = Files.readAllLines(classNamesFile);
this.enabled = true;
Console.println("✓ Object detection enabled with YOLO");
log.info("✓ Object detection enabled with YOLO");
} catch (Exception e) {
System.err.println("⚠️ Object detection disabled: " + e.getMessage());
throw new IOException("Failed to initialize object detection", e);

271
src/main/java/auctiora/RateLimitedHttpClient.java
View File

@@ -2,9 +2,7 @@ package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
@@ -12,259 +10,66 @@ import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import io.github.bucket4j.*;
/**
* Rate-limited HTTP client that enforces per-host request limits.
*
* Features:
* - Per-host rate limiting (configurable max requests per second)
* - Request counting and monitoring
* - Thread-safe using semaphores
* - Automatic host extraction from URLs
*
* This prevents overloading external services like Troostwijk and getting blocked.
*/
@ApplicationScoped
public class RateLimitedHttpClient {
private static final Logger LOG = Logger.getLogger(RateLimitedHttpClient.class);
private final HttpClient httpClient;
private final Map<String, RateLimiter> rateLimiters;
private final Map<String, RequestStats> requestStats;
private final HttpClient client = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(30))
.build();
@ConfigProperty(name = "auction.http.rate-limit.default-max-rps", defaultValue = "2")
int defaultMaxRequestsPerSecond;
int defaultRps;
@ConfigProperty(name = "auction.http.rate-limit.troostwijk-max-rps", defaultValue = "1")
int troostwijkMaxRequestsPerSecond;
int troostwijkRps;
@ConfigProperty(name = "auction.http.timeout-seconds", defaultValue = "30")
int timeoutSeconds;
public RateLimitedHttpClient() {
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(30))
.build();
this.rateLimiters = new ConcurrentHashMap<>();
this.requestStats = new ConcurrentHashMap<>();
}
private final Map<String, Bucket> buckets = new ConcurrentHashMap<>();
/**
* Sends a GET request with automatic rate limiting based on host.
*/
public HttpResponse<String> sendGet(String url) throws IOException, InterruptedException {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofString());
}
/**
* Sends a request for binary data (like images) with rate limiting.
*/
public HttpResponse<byte[]> sendGetBytes(String url) throws IOException, InterruptedException {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofByteArray());
}
/**
* Sends any HTTP request with automatic rate limiting.
*/
public <T> HttpResponse<T> send(HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler)
throws IOException, InterruptedException {
String host = extractHost(request.uri());
RateLimiter limiter = getRateLimiter(host);
RequestStats stats = getRequestStats(host);
// Enforce rate limit (blocks if necessary)
limiter.acquire();
// Track request
stats.incrementTotal();
long startTime = System.currentTimeMillis();
try {
HttpResponse<T> response = httpClient.send(request, bodyHandler);
long duration = System.currentTimeMillis() - startTime;
stats.recordSuccess(duration);
LOG.debugf("HTTP %d %s %s (%dms)",
response.statusCode(), request.method(), host, duration);
// Track rate limit violations (429 = Too Many Requests)
if (response.statusCode() == 429) {
stats.incrementRateLimited();
LOG.warnf("⚠️ Rate limited by %s (HTTP 429)", host);
}
return response;
} catch (IOException | InterruptedException e) {
stats.incrementFailed();
LOG.warnf("❌ HTTP request failed for %s: %s", host, e.getMessage());
throw e;
}
}
/**
* Gets or creates a rate limiter for a specific host.
*/
private RateLimiter getRateLimiter(String host) {
return rateLimiters.computeIfAbsent(host, h -> {
int maxRps = getMaxRequestsPerSecond(h);
LOG.infof("Initializing rate limiter for %s: %d req/s", h, maxRps);
return new RateLimiter(maxRps);
private Bucket bucketForHost(String host) {
return buckets.computeIfAbsent(host, h -> {
int rps = host.contains("troostwijk") ? troostwijkRps : defaultRps;
var limit = Bandwidth.simple(rps, Duration.ofSeconds(1));
return Bucket4j.builder().addLimit(limit).build();
});
}
/**
* Gets or creates request stats for a specific host.
*/
private RequestStats getRequestStats(String host) {
return requestStats.computeIfAbsent(host, h -> new RequestStats(h));
public HttpResponse<String> sendGet(String url) throws Exception {
var req = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(req, HttpResponse.BodyHandlers.ofString());
}
/**
* Determines max requests per second for a given host.
*/
private int getMaxRequestsPerSecond(String host) {
if (host.contains("troostwijk")) {
return troostwijkMaxRequestsPerSecond;
}
return defaultMaxRequestsPerSecond;
public HttpResponse<byte[]> sendGetBytes(String url) throws Exception {
var req = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(req, HttpResponse.BodyHandlers.ofByteArray());
}
/**
* Extracts host from URI (e.g., "api.troostwijkauctions.com").
*/
private String extractHost(URI uri) {
return uri.getHost() != null ? uri.getHost() : uri.toString();
}
public <T> HttpResponse<T> send(HttpRequest req,
HttpResponse.BodyHandler<T> handler) throws Exception {
String host = req.uri().getHost();
var bucket = bucketForHost(host);
bucket.asBlocking().consume(1);
/**
* Gets statistics for all hosts.
*/
public Map<String, RequestStats> getAllStats() {
return Map.copyOf(requestStats);
}
var start = System.currentTimeMillis();
var resp = client.send(req, handler);
var duration = System.currentTimeMillis() - start;
/**
* Gets statistics for a specific host.
*/
public RequestStats getStats(String host) {
return requestStats.get(host);
}
// (Optional) Logging
System.out.printf("HTTP %d %s %s in %d ms%n",
resp.statusCode(), req.method(), host, duration);
/**
* Rate limiter implementation using token bucket algorithm.
* Allows burst traffic up to maxRequestsPerSecond, then enforces steady rate.
*/
private static class RateLimiter {
private final Semaphore semaphore;
private final int maxRequestsPerSecond;
private final long intervalNanos;
RateLimiter(int maxRequestsPerSecond) {
this.maxRequestsPerSecond = maxRequestsPerSecond;
this.intervalNanos = TimeUnit.SECONDS.toNanos(1) / maxRequestsPerSecond;
this.semaphore = new Semaphore(maxRequestsPerSecond);
// Refill tokens periodically
startRefillThread();
}
void acquire() throws InterruptedException {
semaphore.acquire();
// Enforce minimum delay between requests
long delayMillis = intervalNanos / 1_000_000;
if (delayMillis > 0) {
Thread.sleep(delayMillis);
}
}
private void startRefillThread() {
Thread refillThread = new Thread(() -> {
while (!Thread.currentThread().isInterrupted()) {
try {
Thread.sleep(1000); // Refill every second
int toRelease = maxRequestsPerSecond - semaphore.availablePermits();
if (toRelease > 0) {
semaphore.release(toRelease);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}, "RateLimiter-Refill");
refillThread.setDaemon(true);
refillThread.start();
}
}
/**
* Statistics tracker for HTTP requests per host.
*/
public static class RequestStats {
private final String host;
private final AtomicLong totalRequests = new AtomicLong(0);
private final AtomicLong successfulRequests = new AtomicLong(0);
private final AtomicLong failedRequests = new AtomicLong(0);
private final AtomicLong rateLimitedRequests = new AtomicLong(0);
private final AtomicLong totalDurationMs = new AtomicLong(0);
RequestStats(String host) {
this.host = host;
}
void incrementTotal() {
totalRequests.incrementAndGet();
}
void recordSuccess(long durationMs) {
successfulRequests.incrementAndGet();
totalDurationMs.addAndGet(durationMs);
}
void incrementFailed() {
failedRequests.incrementAndGet();
}
void incrementRateLimited() {
rateLimitedRequests.incrementAndGet();
}
// Getters
public String getHost() { return host; }
public long getTotalRequests() { return totalRequests.get(); }
public long getSuccessfulRequests() { return successfulRequests.get(); }
public long getFailedRequests() { return failedRequests.get(); }
public long getRateLimitedRequests() { return rateLimitedRequests.get(); }
public long getAverageDurationMs() {
long successful = successfulRequests.get();
return successful > 0 ? totalDurationMs.get() / successful : 0;
}
@Override
public String toString() {
return String.format("%s: %d total, %d success, %d failed, %d rate-limited, avg %dms",
host, getTotalRequests(), getSuccessfulRequests(),
getFailedRequests(), getRateLimitedRequests(), getAverageDurationMs());
}
return resp;
}
}

270
src/main/java/auctiora/RateLimitedHttpClient2.java Normal file
View File

@@ -0,0 +1,270 @@
package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/**
* Rate-limited HTTP client that enforces per-host request limits.
*
* Features:
* - Per-host rate limiting (configurable max requests per second)
* - Request counting and monitoring
* - Thread-safe using semaphores
* - Automatic host extraction from URLs
*
* This prevents overloading external services like Troostwijk and getting blocked.
*/
@ApplicationScoped
public class RateLimitedHttpClient2 {
private static final Logger LOG = Logger.getLogger(RateLimitedHttpClient2.class);
private final HttpClient httpClient;
private final Map<String, RateLimiter> rateLimiters;
private final Map<String, RequestStats> requestStats;
@ConfigProperty(name = "auction.http.rate-limit.default-max-rps", defaultValue = "2")
int defaultMaxRequestsPerSecond;
@ConfigProperty(name = "auction.http.rate-limit.troostwijk-max-rps", defaultValue = "1")
int troostwijkMaxRequestsPerSecond;
@ConfigProperty(name = "auction.http.timeout-seconds", defaultValue = "30")
int timeoutSeconds;
public RateLimitedHttpClient2() {
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(30))
.build();
this.rateLimiters = new ConcurrentHashMap<>();
this.requestStats = new ConcurrentHashMap<>();
}
/**
* Sends a GET request with automatic rate limiting based on host.
*/
public HttpResponse<String> sendGet(String url) throws IOException, InterruptedException {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofString());
}
/**
* Sends a request for binary data (like images) with rate limiting.
*/
public HttpResponse<byte[]> sendGetBytes(String url) throws IOException, InterruptedException {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofByteArray());
}
/**
* Sends any HTTP request with automatic rate limiting.
*/
public <T> HttpResponse<T> send(HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler)
throws IOException, InterruptedException {
String host = extractHost(request.uri());
RateLimiter limiter = getRateLimiter(host);
RequestStats stats = getRequestStats(host);
// Enforce rate limit (blocks if necessary)
limiter.acquire();
// Track request
stats.incrementTotal();
long startTime = System.currentTimeMillis();
try {
HttpResponse<T> response = httpClient.send(request, bodyHandler);
long duration = System.currentTimeMillis() - startTime;
stats.recordSuccess(duration);
LOG.debugf("HTTP %d %s %s (%dms)",
response.statusCode(), request.method(), host, duration);
// Track rate limit violations (429 = Too Many Requests)
if (response.statusCode() == 429) {
stats.incrementRateLimited();
LOG.warnf("⚠️ Rate limited by %s (HTTP 429)", host);
}
return response;
} catch (IOException | InterruptedException e) {
stats.incrementFailed();
LOG.warnf("❌ HTTP request failed for %s: %s", host, e.getMessage());
throw e;
}
}
/**
* Gets or creates a rate limiter for a specific host.
*/
private RateLimiter getRateLimiter(String host) {
return rateLimiters.computeIfAbsent(host, h -> {
int maxRps = getMaxRequestsPerSecond(h);
LOG.infof("Initializing rate limiter for %s: %d req/s", h, maxRps);
return new RateLimiter(maxRps);
});
}
/**
* Gets or creates request stats for a specific host.
*/
private RequestStats getRequestStats(String host) {
return requestStats.computeIfAbsent(host, h -> new RequestStats(h));
}
/**
* Determines max requests per second for a given host.
*/
private int getMaxRequestsPerSecond(String host) {
if (host.contains("troostwijk")) {
return troostwijkMaxRequestsPerSecond;
}
return defaultMaxRequestsPerSecond;
}
/**
* Extracts host from URI (e.g., "api.troostwijkauctions.com").
*/
private String extractHost(URI uri) {
return uri.getHost() != null ? uri.getHost() : uri.toString();
}
/**
* Gets statistics for all hosts.
*/
public Map<String, RequestStats> getAllStats() {
return Map.copyOf(requestStats);
}
/**
* Gets statistics for a specific host.
*/
public RequestStats getStats(String host) {
return requestStats.get(host);
}
/**
* Rate limiter implementation using token bucket algorithm.
* Allows burst traffic up to maxRequestsPerSecond, then enforces steady rate.
*/
private static class RateLimiter {
private final Semaphore semaphore;
private final int maxRequestsPerSecond;
private final long intervalNanos;
RateLimiter(int maxRequestsPerSecond) {
this.maxRequestsPerSecond = maxRequestsPerSecond;
this.intervalNanos = TimeUnit.SECONDS.toNanos(1) / maxRequestsPerSecond;
this.semaphore = new Semaphore(maxRequestsPerSecond);
// Refill tokens periodically
startRefillThread();
}
void acquire() throws InterruptedException {
semaphore.acquire();
// Enforce minimum delay between requests
long delayMillis = intervalNanos / 1_000_000;
if (delayMillis > 0) {
Thread.sleep(delayMillis);
}
}
private void startRefillThread() {
Thread refillThread = new Thread(() -> {
while (!Thread.currentThread().isInterrupted()) {
try {
Thread.sleep(1000); // Refill every second
int toRelease = maxRequestsPerSecond - semaphore.availablePermits();
if (toRelease > 0) {
semaphore.release(toRelease);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}, "RateLimiter-Refill");
refillThread.setDaemon(true);
refillThread.start();
}
}
/**
* Statistics tracker for HTTP requests per host.
*/
public static class RequestStats {
private final String host;
private final AtomicLong totalRequests = new AtomicLong(0);
private final AtomicLong successfulRequests = new AtomicLong(0);
private final AtomicLong failedRequests = new AtomicLong(0);
private final AtomicLong rateLimitedRequests = new AtomicLong(0);
private final AtomicLong totalDurationMs = new AtomicLong(0);
RequestStats(String host) {
this.host = host;
}
void incrementTotal() {
totalRequests.incrementAndGet();
}
void recordSuccess(long durationMs) {
successfulRequests.incrementAndGet();
totalDurationMs.addAndGet(durationMs);
}
void incrementFailed() {
failedRequests.incrementAndGet();
}
void incrementRateLimited() {
rateLimitedRequests.incrementAndGet();
}
// Getters
public String getHost() { return host; }
public long getTotalRequests() { return totalRequests.get(); }
public long getSuccessfulRequests() { return successfulRequests.get(); }
public long getFailedRequests() { return failedRequests.get(); }
public long getRateLimitedRequests() { return rateLimitedRequests.get(); }
public long getAverageDurationMs() {
long successful = successfulRequests.get();
return successful > 0 ? totalDurationMs.get() / successful : 0;
}
@Override
public String toString() {
return String.format("%s: %d total, %d success, %d failed, %d rate-limited, avg %dms",
host, getTotalRequests(), getSuccessfulRequests(),
getFailedRequests(), getRateLimitedRequests(), getAverageDurationMs());
}
}
}

208
src/main/java/auctiora/ScraperDataAdapter.java
View File

@@ -1,23 +1,15 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
/**
* Adapter to convert data from the Python scraper's schema to the Monitor's schema.
*
* SCRAPER SCHEMA DIFFERENCES:
* - auction_id: TEXT ("A7-39813") vs INTEGER (39813)
* - lot_id: TEXT ("A1-28505-5") vs INTEGER (285055)
* - current_bid: TEXT ("€123.45") vs REAL (123.45)
* - Field names: lots_count vs lot_count, auction_id vs sale_id, etc.
*
* This adapter handles the translation between the two schemas.
*/
class ScraperDataAdapter {
@Slf4j
public class ScraperDataAdapter {
private static final DateTimeFormatter[] TIMESTAMP_FORMATS = {
DateTimeFormatter.ISO_LOCAL_DATE_TIME,
@@ -25,16 +17,6 @@ class ScraperDataAdapter {
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
};
/**
* Converts scraper's auction format to monitor's AuctionInfo record.
*
* Scraper format:
* - auction_id: "A7-39813" (TEXT)
* - location: "Cluj-Napoca, RO" (combined)
* - lots_count: INTEGER
* - first_lot_closing_time: TEXT
* - scraped_at: TEXT
*/
static AuctionInfo fromScraperAuction(ResultSet rs) throws SQLException {
// Parse "A7-39813" → auctionId=39813, type="A7"
String auctionIdStr = rs.getString("auction_id");
@@ -64,183 +46,91 @@ class ScraperDataAdapter {
);
}
/**
* Converts scraper's lot format to monitor's Lot record.
*
* Scraper format:
* - lot_id: "A1-28505-5" (TEXT)
* - auction_id: "A7-39813" (TEXT)
* - current_bid: "€123.45" or "No bids" (TEXT)
* - bid_count: INTEGER
* - closing_time: TEXT
*/
static Lot fromScraperLot(ResultSet rs) throws SQLException {
// Parse "A1-28505-5" → lotId=285055
String lotIdStr = rs.getString("lot_id");
int lotId = extractNumericId(lotIdStr);
public static Lot fromScraperLot(ResultSet rs) throws SQLException {
var lotId = extractNumericId(rs.getString("lot_id"));
var saleId = extractNumericId(rs.getString("auction_id"));
// Parse "A7-39813" → saleId=39813
String auctionIdStr = rs.getString("auction_id");
int saleId = extractNumericId(auctionIdStr);
var bidStr = getStringOrNull(rs, "current_bid");
var bid = parseBidAmount(bidStr);
var currency = parseBidCurrency(bidStr);
// Parse "€123.45" → currentBid=123.45, currency="EUR"
String currentBidStr = getStringOrNull(rs, "current_bid");
double currentBid = parseBidAmount(currentBidStr);
String currency = parseBidCurrency(currentBidStr);
// Parse timestamp
LocalDateTime closingTime = parseTimestamp(getStringOrNull(rs, "closing_time"));
var closing = parseTimestamp(getStringOrNull(rs, "closing_time"));
return new Lot(
saleId,
lotId,
rs.getString("title"),
getStringOrDefault(rs, "description", ""),
"", // manufacturer - not in scraper schema
"", // type - not in scraper schema
0, // year - not in scraper schema
"", "", 0,
getStringOrDefault(rs, "category", ""),
currentBid,
bid,
currency,
rs.getString("url"),
closingTime,
false // closing_notified - not yet notified
closing,
false
);
}
/**
* Extracts numeric ID from scraper's text format.
* Examples:
* - "A7-39813" → 39813
* - "A1-28505-5" → 285055 (concatenates all digits)
*/
static int extractNumericId(String id) {
if (id == null || id.isEmpty()) {
return 0;
}
String digits = id.replaceAll("[^0-9]", "");
public static int extractNumericId(String id) {
if (id == null || id.isBlank()) return 0;
var digits = id.replaceAll("\\D+", "");
return digits.isEmpty() ? 0 : Integer.parseInt(digits);
}
/**
* Extracts type prefix from scraper's auction/lot ID.
* Examples:
* - "A7-39813" → "A7"
* - "A1-28505-5" → "A1"
*/
private static String extractTypePrefix(String id) {
if (id == null || id.isEmpty()) {
return "";
}
int dashIndex = id.indexOf('-');
return dashIndex > 0 ? id.substring(0, dashIndex) : "";
if (id == null) return "";
var idx = id.indexOf('-');
return idx > 0 ? id.substring(0, idx) : "";
}
/**
* Parses location string into [city, country] array.
* Examples:
* - "Cluj-Napoca, RO" → ["Cluj-Napoca", "RO"]
* - "Amsterdam" → ["Amsterdam", ""]
*/
private static String[] parseLocation(String location) {
if (location == null || location.isEmpty()) {
return new String[]{"", ""};
if (location == null || location.isBlank()) return new String[]{ "", "" };
var parts = location.split(",\\s*");
var city = parts[0].trim();
var country = parts.length > 1 ? parts[parts.length - 1].trim() : "";
return new String[]{ city, country };
}
String[] parts = location.split(",\\s*");
String city = parts.length > 0 ? parts[0].trim() : "";
String country = parts.length > 1 ? parts[parts.length - 1].trim() : "";
return new String[]{city, country};
}
/**
* Parses bid amount from scraper's text format.
* Examples:
* - "€123.45" → 123.45
* - "$50.00" → 50.0
* - "No bids" → 0.0
* - "123.45" → 123.45
*/
private static double parseBidAmount(String bid) {
if (bid == null || bid.isEmpty() || bid.toLowerCase().contains("no")) {
return 0.0;
}
if (bid == null || bid.isBlank() || bid.toLowerCase().contains("no")) return 0.0;
var cleaned = bid.replaceAll("[^0-9.]", "");
try {
// Remove all non-numeric characters except decimal point
String cleanBid = bid.replaceAll("[^0-9.]", "");
return cleanBid.isEmpty() ? 0.0 : Double.parseDouble(cleanBid);
return cleaned.isEmpty() ? 0.0 : Double.parseDouble(cleaned);
} catch (NumberFormatException e) {
return 0.0;
}
}
/**
* Extracts currency from bid string.
* Examples:
* - "€123.45" → "EUR"
* - "$50.00" → "USD"
* - "123.45" → "EUR" (default)
*/
private static String parseBidCurrency(String bid) {
if (bid == null || bid.isEmpty()) {
return "EUR";
if (bid == null) return "EUR";
return bid.contains("€") ? "EUR"
: bid.contains("$") ? "USD"
: bid.contains("£") ? "GBP"
: "EUR";
}
if (bid.contains("€")) return "EUR";
if (bid.contains("$")) return "USD";
if (bid.contains("£")) return "GBP";
return "EUR"; // Default
private static LocalDateTime parseTimestamp(String ts) {
if (ts == null || ts.isBlank()) return null;
for (var fmt : TIMESTAMP_FORMATS) {
try {
return LocalDateTime.parse(ts, fmt);
} catch (DateTimeParseException ignored) { }
}
/**
* Parses timestamp from various formats used by the scraper.
* Tries multiple formats in order.
*/
private static LocalDateTime parseTimestamp(String timestamp) {
if (timestamp == null || timestamp.isEmpty()) {
log.info("Unable to parse timestamp: {}", ts);
return null;
}
for (DateTimeFormatter formatter : TIMESTAMP_FORMATS) {
try {
return LocalDateTime.parse(timestamp, formatter);
} catch (DateTimeParseException e) {
// Try next format
}
private static String getStringOrNull(ResultSet rs, String col) throws SQLException {
return rs.getString(col);
}
// Couldn't parse - return null
Console.println("⚠️ Could not parse timestamp: " + timestamp);
return null;
private static String getStringOrDefault(ResultSet rs, String col, String def) throws SQLException {
var v = rs.getString(col);
return v != null ? v : def;
}
// Helper methods for safe ResultSet access
private static String getStringOrNull(ResultSet rs, String column) throws SQLException {
try {
return rs.getString(column);
} catch (SQLException e) {
return null;
}
}
private static String getStringOrDefault(ResultSet rs, String column, String defaultValue) throws SQLException {
try {
String value = rs.getString(column);
return value != null ? value : defaultValue;
} catch (SQLException e) {
return defaultValue;
}
}
private static int getIntOrDefault(ResultSet rs, String column, int defaultValue) throws SQLException {
try {
return rs.getInt(column);
} catch (SQLException e) {
return defaultValue;
}
private static int getIntOrDefault(ResultSet rs, String col, int def) throws SQLException {
var v = rs.getInt(col);
return rs.wasNull() ? def : v;
}
}

84
src/main/java/auctiora/StatusResource.java Normal file
View File

@@ -0,0 +1,84 @@
package auctiora;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Map;
@Slf4j
@Path("/api")
public class StatusResource {
private static final DateTimeFormatter FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z")
.withZone(ZoneId.systemDefault());
@ConfigProperty(name = "application.version", defaultValue = "1.0-SNAPSHOT")
String appVersion;
@ConfigProperty(name = "application.groupId")
String groupId;
@ConfigProperty(name = "application.artifactId")
String artifactId;
@ConfigProperty(name = "application.version")
String version;
// Java 16+ Record for structured response
public record StatusResponse(
String groupId,
String artifactId,
String version,
String status,
String timestamp,
String mvnVersion,
String javaVersion,
String os,
String openCvVersion
) { }
@GET
@Path("/status")
@Produces(MediaType.APPLICATION_JSON)
public StatusResponse getStatus() {
log.info("Status endpoint called");
return new StatusResponse(groupId, artifactId, version,
"running",
FORMATTER.format(Instant.now()),
appVersion,
System.getProperty("java.version"),
System.getProperty("os.name") + " " + System.getProperty("os.arch"),
getOpenCvVersion()
);
}
@GET
@Path("/hello")
@Produces(MediaType.APPLICATION_JSON)
public Map<String, String> sayHello() {
log.info("hello endpoint called");
return Map.of(
"message", "Hello from Scrape-UI!",
"timestamp", FORMATTER.format(Instant.now()),
"openCvVersion", getOpenCvVersion()
);
}
private String getOpenCvVersion() {
try {
// Load OpenCV if not already loaded (safe to call multiple times)
nu.pattern.OpenCV.loadLocally();
return org.opencv.core.Core.VERSION;
} catch (Exception e) {
return "4.9.0 (default)";
}
}
}

180
src/main/java/auctiora/TroostwijkMonitor.java
View File

@@ -1,173 +1,135 @@
package auctiora;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.sql.SQLException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* Monitoring service for Troostwijk auction lots.
* This class focuses on:
* - Monitoring bid changes on lots (populated by external scraper)
* - Sending notifications for important events
* - Coordinating image processing
*
* Does NOT handle scraping - that's done by the external ARCHITECTURE-TROOSTWIJK-SCRAPER process.
*/
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Slf4j
public class TroostwijkMonitor {
private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list";
private final RateLimitedHttpClient httpClient;
private final ObjectMapper objectMapper;
public final DatabaseService db;
private final NotificationService notifier;
private final ObjectDetectionService detector;
private final ImageProcessingService imageProcessor;
RateLimitedHttpClient2 httpClient;
ObjectMapper objectMapper;
@Getter DatabaseService db;
NotificationService notifier;
ObjectDetectionService detector;
ImageProcessingService imageProcessor;
/**
* Constructor for the monitoring service.
*
* @param databasePath Path to SQLite database file (shared with external scraper)
* @param notificationConfig "desktop" or "smtp:user:pass:email"
* @param yoloCfgPath YOLO config file path
* @param yoloWeightsPath YOLO weights file path
* @param classNamesPath Class names file path
*/
public TroostwijkMonitor(String databasePath, String notificationConfig,
String yoloCfgPath, String yoloWeightsPath, String classNamesPath)
ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
var t = new Thread(r, "troostwijk-monitor-thread");
t.setDaemon(true);
return t;
});
public TroostwijkMonitor(String databasePath,
String notificationConfig,
String yoloCfgPath,
String yoloWeightsPath,
String classNamesPath)
throws SQLException, IOException {
this.httpClient = new RateLimitedHttpClient();
this.objectMapper = new ObjectMapper();
this.db = new DatabaseService(databasePath);
this.notifier = new NotificationService(notificationConfig, "");
this.detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
this.imageProcessor = new ImageProcessingService(db, detector, httpClient);
// Initialize database schema
httpClient = new RateLimitedHttpClient2();
objectMapper = new ObjectMapper();
db = new DatabaseService(databasePath);
notifier = new NotificationService(notificationConfig);
detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
imageProcessor = new ImageProcessingService(db, detector, httpClient);
db.ensureSchema();
}
/**
* Schedules periodic monitoring of all lots.
* Runs every hour to refresh bids and detect changes.
* Increases frequency for lots closing soon.
*/
public void scheduleMonitoring() {
var scheduler = Executors.newScheduledThreadPool(1);
scheduler.scheduleAtFixedRate(() -> {
scheduler.scheduleAtFixedRate(this::monitorAllLots, 0, 1, TimeUnit.HOURS);
log.info("✓ Monitoring service started");
}
private void monitorAllLots() {
try {
var activeLots = db.getActiveLots();
Console.println("Monitoring " + activeLots.size() + " active lots...");
log.info("Monitoring {} active lots …", activeLots.size());
for (var lot : activeLots) {
// Refresh lot bidding information
checkAndUpdateLot(lot);
}
} catch (SQLException e) {
log.error("Error during scheduled monitoring", e);
}
}
private void checkAndUpdateLot(Lot lot) {
refreshLotBid(lot);
// Check closing time
var minutesLeft = lot.minutesUntilClose();
if (minutesLeft < 30) {
// Send warning when within 5 minutes
if (minutesLeft <= 5 && !lot.closingNotified()) {
notifier.sendNotification(
"Kavel " + lot.lotId() + " sluit binnen " + minutesLeft + " min.",
"Lot nearing closure", 1);
// Update notification flag
var updated = new Lot(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
lot.currentBid(), lot.currency(), lot.url(),
lot.closingTime(), true
);
db.updateLotNotificationFlags(updated);
try {
db.updateLotNotificationFlags(lot.withClosingNotified(true));
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
// Schedule additional quick check
scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
}
}
} catch (SQLException e) {
System.err.println("Error during scheduled monitoring: " + e.getMessage());
}
}, 0, 1, TimeUnit.HOURS);
Console.println("✓ Monitoring service started");
}
/**
* Refreshes the bid for a single lot and sends notification if changed.
*
* @param lot the lot to refresh
*/
private void refreshLotBid(Lot lot) {
try {
var url = LOT_API + "?batchSize=1&listType=7&offset=0&sortOption=0&saleID=" + lot.saleId()
+ "&parentID=0&relationID=0&buildversion=201807311&lotID=" + lot.lotId();
var url = LOT_API +
"?batchSize=1&listType=7&offset=0&sortOption=0" +
"&saleID=" + lot.saleId() +
"&parentID=0&relationID=0&buildversion=201807311" +
"&lotID=" + lot.lotId();
var response = httpClient.sendGet(url);
var resp = httpClient.sendGet(url);
if (resp.statusCode() != 200) return;
if (response.statusCode() != 200) return;
var root = objectMapper.readTree(response.body());
var root = objectMapper.readTree(resp.body());
var results = root.path("results");
if (results.isArray() && !results.isEmpty()) {
var node = results.get(0);
var newBid = node.path("cb").asDouble();
if (results.isArray() && results.size() > 0) {
var newBid = results.get(0).path("cb").asDouble();
if (Double.compare(newBid, lot.currentBid()) > 0) {
var previous = lot.currentBid();
// Create updated lot with new bid
var updatedLot = new Lot(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
newBid, lot.currency(), lot.url(),
lot.closingTime(), lot.closingNotified()
);
var updatedLot = lot.withCurrentBid(newBid);
db.updateLotCurrentBid(updatedLot);
var msg = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)",
var msg = String.format(
"Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previous);
notifier.sendNotification(msg, "Kavel bieding update", 0);
}
}
} catch (IOException | InterruptedException | SQLException e) {
System.err.println("Failed to refresh bid for lot " + lot.lotId() + ": " + e.getMessage());
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
log.warn("Failed to refresh bid for lot {}", lot.lotId(), e);
if (e instanceof InterruptedException) Thread.currentThread().interrupt();
}
}
/**
* Prints statistics about the data in the database.
*/
public void printDatabaseStats() {
try {
var allLots = db.getAllLots();
var imageCount = db.getImageCount();
Console.println("📊 Database Summary:");
Console.println(" Total lots in database: " + allLots.size());
Console.println(" Total images processed: " + imageCount);
log.info("📊 Database Summary: total lots = {}, total images = {}",
allLots.size(), imageCount);
if (!allLots.isEmpty()) {
var totalBids = allLots.stream().mapToDouble(Lot::currentBid).sum();
Console.println(" Total current bids: €" + String.format("%.2f", totalBids));
var sum = allLots.stream().mapToDouble(Lot::currentBid).sum();
log.info("Total current bids: €{:.2f}", sum);
}
} catch (SQLException e) {
System.err.println(" ⚠️ Could not retrieve database stats: " + e.getMessage());
log.warn("Could not retrieve database stats", e);
}
}
/**
* Process pending images for lots in the database.
* This should be called after the external scraper has populated lot data.
*/
public void processPendingImages() {
imageProcessor.processPendingImages();
}

115
src/main/java/auctiora/WorkflowOrchestrator.java
View File

@@ -1,5 +1,7 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.io.Console;
import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
@@ -14,6 +16,7 @@ import java.util.concurrent.TimeUnit;
* This class coordinates all services and provides scheduled execution,
* event-driven triggers, and manual workflow execution.
*/
@Slf4j
public class WorkflowOrchestrator {
private final TroostwijkMonitor monitor;
@@ -32,15 +35,15 @@ public class WorkflowOrchestrator {
String yoloCfg, String yoloWeights, String yoloClasses)
throws SQLException, IOException {
Console.println("🔧 Initializing Workflow Orchestrator...");
log.info("🔧 Initializing Workflow Orchestrator...");
// Initialize core services
this.db = new DatabaseService(databasePath);
this.db.ensureSchema();
this.notifier = new NotificationService(notificationConfig, "");
this.notifier = new NotificationService(notificationConfig);
this.detector = new ObjectDetectionService(yoloCfg, yoloWeights, yoloClasses);
RateLimitedHttpClient httpClient = new RateLimitedHttpClient();
RateLimitedHttpClient2 httpClient = new RateLimitedHttpClient2();
this.imageProcessor = new ImageProcessingService(db, detector, httpClient);
this.monitor = new TroostwijkMonitor(databasePath, notificationConfig,
@@ -48,7 +51,7 @@ public class WorkflowOrchestrator {
this.scheduler = Executors.newScheduledThreadPool(3);
Console.println("✓ Workflow Orchestrator initialized");
log.info("✓ Workflow Orchestrator initialized");
}
/**
@@ -57,11 +60,11 @@ public class WorkflowOrchestrator {
*/
public void startScheduledWorkflows() {
if (isRunning) {
Console.println("⚠️ Workflows already running");
log.info("⚠️ Workflows already running");
return;
}
Console.println("\n🚀 Starting Scheduled Workflows...\n");
log.info("\n🚀 Starting Scheduled Workflows...\n");
// Workflow 1: Import scraper data (every 30 minutes)
scheduleScraperDataImport();
@@ -76,7 +79,7 @@ public class WorkflowOrchestrator {
scheduleClosingAlerts();
isRunning = true;
Console.println("✓ All scheduled workflows started\n");
log.info("✓ All scheduled workflows started\n");
}
/**
@@ -87,23 +90,23 @@ public class WorkflowOrchestrator {
private void scheduleScraperDataImport() {
scheduler.scheduleAtFixedRate(() -> {
try {
Console.println("📥 [WORKFLOW 1] Importing scraper data...");
log.info("📥 [WORKFLOW 1] Importing scraper data...");
long start = System.currentTimeMillis();
// Import auctions
var auctions = db.importAuctionsFromScraper();
Console.println(" → Imported " + auctions.size() + " auctions");
log.info(" → Imported " + auctions.size() + " auctions");
// Import lots
var lots = db.importLotsFromScraper();
Console.println(" → Imported " + lots.size() + " lots");
log.info(" → Imported " + lots.size() + " lots");
// Import image URLs
var images = db.getUnprocessedImagesFromScraper();
Console.println(" → Found " + images.size() + " unprocessed images");
log.info(" → Found " + images.size() + " unprocessed images");
long duration = System.currentTimeMillis() - start;
Console.println(" ✓ Scraper import completed in " + duration + "ms\n");
log.info(" ✓ Scraper import completed in " + duration + "ms\n");
// Trigger notification if significant data imported
if (auctions.size() > 0 || lots.size() > 10) {
@@ -115,11 +118,11 @@ public class WorkflowOrchestrator {
}
} catch (Exception e) {
Console.println(" ❌ Scraper import failed: " + e.getMessage());
log.info(" ❌ Scraper import failed: " + e.getMessage());
}
}, 0, 30, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Scraper Data Import (every 30 min)");
log.info(" ✓ Scheduled: Scraper Data Import (every 30 min)");
}
/**
@@ -130,18 +133,18 @@ public class WorkflowOrchestrator {
private void scheduleImageProcessing() {
scheduler.scheduleAtFixedRate(() -> {
try {
Console.println("🖼️ [WORKFLOW 2] Processing pending images...");
log.info("🖼️ [WORKFLOW 2] Processing pending images...");
long start = System.currentTimeMillis();
// Get unprocessed images
var unprocessedImages = db.getUnprocessedImagesFromScraper();
if (unprocessedImages.isEmpty()) {
Console.println(" → No pending images to process\n");
log.info(" → No pending images to process\n");
return;
}
Console.println(" → Processing " + unprocessedImages.size() + " images");
log.info(" → Processing " + unprocessedImages.size() + " images");
int processed = 0;
int detected = 0;
@@ -184,20 +187,20 @@ public class WorkflowOrchestrator {
Thread.sleep(500);
} catch (Exception e) {
Console.println(" ⚠️ Failed to process image: " + e.getMessage());
log.info(" ⚠️ Failed to process image: " + e.getMessage());
}
}
long duration = System.currentTimeMillis() - start;
Console.println(String.format(" ✓ Processed %d images, detected objects in %d (%.1fs)\n",
log.info(String.format(" ✓ Processed %d images, detected objects in %d (%.1fs)\n",
processed, detected, duration / 1000.0));
} catch (Exception e) {
Console.println(" ❌ Image processing failed: " + e.getMessage());
log.info(" ❌ Image processing failed: " + e.getMessage());
}
}, 5, 60, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Image Processing (every 1 hour)");
log.info(" ✓ Scheduled: Image Processing (every 1 hour)");
}
/**
@@ -208,11 +211,11 @@ public class WorkflowOrchestrator {
private void scheduleBidMonitoring() {
scheduler.scheduleAtFixedRate(() -> {
try {
Console.println("💰 [WORKFLOW 3] Monitoring bids...");
log.info("💰 [WORKFLOW 3] Monitoring bids...");
long start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
Console.println(" → Checking " + activeLots.size() + " active lots");
log.info(" → Checking " + activeLots.size() + " active lots");
int bidChanges = 0;
@@ -223,14 +226,14 @@ public class WorkflowOrchestrator {
}
long duration = System.currentTimeMillis() - start;
Console.println(String.format(" ✓ Bid monitoring completed in %dms\n", duration));
log.info(String.format(" ✓ Bid monitoring completed in %dms\n", duration));
} catch (Exception e) {
Console.println(" ❌ Bid monitoring failed: " + e.getMessage());
log.info(" ❌ Bid monitoring failed: " + e.getMessage());
}
}, 2, 15, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Bid Monitoring (every 15 min)");
log.info(" ✓ Scheduled: Bid Monitoring (every 15 min)");
}
/**
@@ -241,7 +244,7 @@ public class WorkflowOrchestrator {
private void scheduleClosingAlerts() {
scheduler.scheduleAtFixedRate(() -> {
try {
Console.println("⏰ [WORKFLOW 4] Checking closing times...");
log.info("⏰ [WORKFLOW 4] Checking closing times...");
long start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
@@ -273,15 +276,15 @@ public class WorkflowOrchestrator {
}
long duration = System.currentTimeMillis() - start;
Console.println(String.format(" → Sent %d closing alerts in %dms\n",
log.info(String.format(" → Sent %d closing alerts in %dms\n",
alertsSent, duration));
} catch (Exception e) {
Console.println(" ❌ Closing alerts failed: " + e.getMessage());
log.info(" ❌ Closing alerts failed: " + e.getMessage());
}
}, 1, 5, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Closing Alerts (every 5 min)");
log.info(" ✓ Scheduled: Closing Alerts (every 5 min)");
}
/**
@@ -289,39 +292,39 @@ public class WorkflowOrchestrator {
* Useful for testing or on-demand execution
*/
public void runCompleteWorkflowOnce() {
Console.println("\n🔄 Running Complete Workflow (Manual Trigger)...\n");
log.info("\n🔄 Running Complete Workflow (Manual Trigger)...\n");
try {
// Step 1: Import data
Console.println("[1/4] Importing scraper data...");
log.info("[1/4] Importing scraper data...");
var auctions = db.importAuctionsFromScraper();
var lots = db.importLotsFromScraper();
Console.println(" ✓ Imported " + auctions.size() + " auctions, " + lots.size() + " lots");
log.info(" ✓ Imported " + auctions.size() + " auctions, " + lots.size() + " lots");
// Step 2: Process images
Console.println("[2/4] Processing pending images...");
log.info("[2/4] Processing pending images...");
monitor.processPendingImages();
Console.println(" ✓ Image processing completed");
log.info(" ✓ Image processing completed");
// Step 3: Check bids
Console.println("[3/4] Monitoring bids...");
log.info("[3/4] Monitoring bids...");
var activeLots = db.getActiveLots();
Console.println(" ✓ Monitored " + activeLots.size() + " lots");
log.info(" ✓ Monitored " + activeLots.size() + " lots");
// Step 4: Check closing times
Console.println("[4/4] Checking closing times...");
log.info("[4/4] Checking closing times...");
int closingSoon = 0;
for (var lot : activeLots) {
if (lot.closingTime() != null && lot.minutesUntilClose() < 30) {
closingSoon++;
}
}
Console.println(" ✓ Found " + closingSoon + " lots closing soon");
log.info(" ✓ Found " + closingSoon + " lots closing soon");
Console.println("\n✓ Complete workflow finished successfully\n");
log.info("\n✓ Complete workflow finished successfully\n");
} catch (Exception e) {
Console.println("\n❌ Workflow failed: " + e.getMessage() + "\n");
log.info("\n❌ Workflow failed: " + e.getMessage() + "\n");
}
}
@@ -329,7 +332,7 @@ public class WorkflowOrchestrator {
* Event-driven trigger: New auction discovered
*/
public void onNewAuctionDiscovered(AuctionInfo auction) {
Console.println("📣 EVENT: New auction discovered - " + auction.title());
log.info("📣 EVENT: New auction discovered - " + auction.title());
try {
db.upsertAuction(auction);
@@ -342,7 +345,7 @@ public class WorkflowOrchestrator {
);
} catch (Exception e) {
Console.println(" ❌ Failed to handle new auction: " + e.getMessage());
log.info(" ❌ Failed to handle new auction: " + e.getMessage());
}
}
@@ -350,7 +353,7 @@ public class WorkflowOrchestrator {
* Event-driven trigger: Bid change detected
*/
public void onBidChange(Lot lot, double previousBid, double newBid) {
Console.println(String.format("📣 EVENT: Bid change on lot %d (€%.2f → €%.2f)",
log.info(String.format("📣 EVENT: Bid change on lot %d (€%.2f → €%.2f)",
lot.lotId(), previousBid, newBid));
try {
@@ -364,7 +367,7 @@ public class WorkflowOrchestrator {
);
} catch (Exception e) {
Console.println(" ❌ Failed to handle bid change: " + e.getMessage());
log.info(" ❌ Failed to handle bid change: " + e.getMessage());
}
}
@@ -372,7 +375,7 @@ public class WorkflowOrchestrator {
* Event-driven trigger: Objects detected in image
*/
public void onObjectsDetected(int lotId, List<String> labels) {
Console.println(String.format("📣 EVENT: Objects detected in lot %d - %s",
log.info(String.format("📣 EVENT: Objects detected in lot %d - %s",
lotId, String.join(", ", labels)));
try {
@@ -384,7 +387,7 @@ public class WorkflowOrchestrator {
);
}
} catch (Exception e) {
Console.println(" ❌ Failed to send detection notification: " + e.getMessage());
log.info(" ❌ Failed to send detection notification: " + e.getMessage());
}
}
@@ -392,17 +395,17 @@ public class WorkflowOrchestrator {
* Prints current workflow status
*/
public void printStatus() {
Console.println("\n📊 Workflow Status:");
Console.println(" Running: " + (isRunning ? "Yes" : "No"));
log.info("\n📊 Workflow Status:");
log.info(" Running: " + (isRunning ? "Yes" : "No"));
try {
var auctions = db.getAllAuctions();
var lots = db.getAllLots();
int images = db.getImageCount();
Console.println(" Auctions: " + auctions.size());
Console.println(" Lots: " + lots.size());
Console.println(" Images: " + images);
log.info(" Auctions: " + auctions.size());
log.info(" Lots: " + lots.size());
log.info(" Images: " + images);
// Count closing soon
int closingSoon = 0;
@@ -411,10 +414,10 @@ public class WorkflowOrchestrator {
closingSoon++;
}
}
Console.println(" Closing soon (< 30 min): " + closingSoon);
log.info(" Closing soon (< 30 min): " + closingSoon);
} catch (Exception e) {
Console.println(" ⚠️ Could not retrieve status: " + e.getMessage());
log.info(" ⚠️ Could not retrieve status: " + e.getMessage());
}
IO.println();
@@ -424,7 +427,7 @@ public class WorkflowOrchestrator {
* Gracefully shuts down all workflows
*/
public void shutdown() {
Console.println("\n🛑 Shutting down workflows...");
log.info("\n🛑 Shutting down workflows...");
isRunning = false;
scheduler.shutdown();
@@ -433,7 +436,7 @@ public class WorkflowOrchestrator {
if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
scheduler.shutdownNow();
}
Console.println("✓ Workflows shut down successfully\n");
log.info("✓ Workflows shut down successfully\n");
} catch (InterruptedException e) {
scheduler.shutdownNow();
Thread.currentThread().interrupt();

19
src/main/resources/application.properties
View File

@@ -1,10 +1,21 @@
# Application Configuration
quarkus.application.name=auctiora
quarkus.application.version=1.0-SNAPSHOT
# Values will be injected from pom.xml during build
quarkus.application.name=${project.artifactId}
quarkus.application.version=${project.version}
# Custom properties for groupId if needed
application.groupId=${project.groupId}
application.artifactId=${project.artifactId}
application.version=${project.version}
# HTTP Configuration
quarkus.http.port=8081
quarkus.http.host=0.0.0.0
# ========== DEVELOPMENT (quarkus:dev) ==========
%dev.quarkus.http.host=127.0.0.1
# ========== PRODUCTION (Docker/JAR) ==========
%prod.quarkus.http.host=0.0.0.0
# ========== TEST PROFILE ==========
%test.quarkus.http.host=localhost
# Enable CORS for frontend development
quarkus.http.cors=true
@@ -26,7 +37,7 @@ quarkus.log.console.level=INFO
# Static resources
quarkus.http.enable-compression=true
quarkus.rest.path=/api
quarkus.rest.path=/
quarkus.http.root-path=/
# Auction Monitor Configuration

7
src/main/resources/resources/beans.xml Normal file
View File

@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="https://jakarta.ee/xml/ns/jakartaee"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="https://jakarta.ee/xml/ns/jakartaee https://jakarta.ee/xml/ns/jakartaee/beans_3_0.xsd
https://jakarta.ee/xml/ns/jakartaee "
version="3.0" bean-discovery-mode="all">
</beans>

224
src/main/resources/resources/index.html Normal file
View File

@@ -0,0 +1,224 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Scrape-UI 1 - Enterprise</title>
<script src="https://cdn.tailwindcss.com"></script>
<style>
.gradient-bg {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
}
.card-hover {
transition: all 0.3s ease;
}
.card-hover:hover {
transform: translateY(-5px);
box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04);
}
</style>
</head>
<body class="bg-gray-50 min-h-screen">
<!-- Header -->
<header class="gradient-bg text-white py-8">
<div class="container mx-auto px-4">
<h1 class="text-4xl font-bold mb-2">Scrape-UI Enterprise</h1>
<p class="text-xl opacity-90">Powered by Quarkus + Modern Frontend</p>
</div>
</header>
<!-- Main Content -->
<main class="container mx-auto px-4 py-8">
<!-- API Status Card -->
<!-- API & Build Status Card -->
<div class="bg-white rounded-lg shadow-md p-6 mb-8 card-hover">
<h2 class="text-2xl font-bold mb-4 text-gray-800">Build & Runtime Status</h2>
<div id="api-status" class="space-y-4">
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
<!-- Build Information -->
<div class="bg-blue-50 p-4 rounded-lg">
<h3 class="font-semibold text-blue-800 mb-2">📦 Maven Build</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
<span class="text-gray-600">Group:</span>
<span class="font-mono font-medium" id="build-group">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Artifact:</span>
<span class="font-mono font-medium" id="build-artifact">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Version:</span>
<span class="font-mono font-medium px-2 py-1 bg-blue-100 rounded" id="build-version">-</span>
</div>
</div>
</div>
<!-- Runtime Information -->
<div class="bg-green-50 p-4 rounded-lg">
<h3 class="font-semibold text-green-800 mb-2">🚀 Runtime</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
<span class="text-gray-600">Status:</span>
<span class="px-2 py-1 rounded-full text-xs font-semibold bg-green-100 text-green-800" id="runtime-status">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Java:</span>
<span class="font-mono" id="java-version">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Platform:</span>
<span class="font-mono" id="runtime-os">-</span>
</div>
</div>
</div>
</div>
<!-- Timestamp & Additional Info -->
<div class="pt-4 border-t">
<div class="flex justify-between items-center">
<div>
<p class="text-sm text-gray-500">Last Updated</p>
<p class="font-medium" id="last-updated">-</p>
</div>
<button onclick="fetchStatus()" class="bg-gray-100 hover:bg-gray-200 text-gray-700 px-4 py-2 rounded-lg text-sm transition-colors">
🔄 Refresh
</button>
</div>
</div>
</div>
</div>
<!-- API Response Card -->
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h2 class="text-2xl font-bold mb-4 text-gray-800">API Test</h2>
<button id="test-api" class="bg-blue-600 text-white px-6 py-2 rounded-lg hover:bg-blue-700 transition-colors mb-4">
Test Greeting API
</button>
<div id="api-response" class="bg-gray-100 p-4 rounded-lg">
<pre class="text-sm text-gray-700">Click the button to test the API</pre>
</div>
</div>
<!-- Features Grid -->
<div class="grid grid-cols-1 md:grid-cols-3 gap-6 mt-8">
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">⚡ Quarkus Backend</h3>
<p class="text-gray-600">Fast startup, low memory footprint, optimized for containers</p>
</div>
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">🚀 REST API</h3>
<p class="text-gray-600">RESTful endpoints with JSON responses</p>
</div>
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">🎨 Modern UI</h3>
<p class="text-gray-600">Responsive design with Tailwind CSS</p>
</div>
</div>
</main>
<script>
// Fetch API status on load
async function fetchStatus() {
try {
const response = await fetch('/api/status')
if (!response.ok) {
throw new Error(`HTTP ${ response.status }: ${ response.statusText }`)
}
const data = await response.json()
// Update Build Information
document.getElementById('build-group').textContent = data.groupId || 'N/A'
document.getElementById('build-artifact').textContent = data.artifactId || data.name || 'N/A'
document.getElementById('build-version').textContent = data.version || 'N/A'
// Update Runtime Information
document.getElementById('runtime-status').textContent = data.status || 'unknown'
document.getElementById('java-version').textContent = data.javaVersion || System.getProperty?.('java.version') || 'N/A'
document.getElementById('runtime-os').textContent = data.os || 'N/A'
// Update Timestamp
const timestamp = data.timestamp ? new Date(data.timestamp).toLocaleString() : 'N/A'
document.getElementById('last-updated').textContent = timestamp
// Update status badge color based on status
const statusBadge = document.getElementById('runtime-status')
if (data.status?.toLowerCase() === 'running') {
statusBadge.className = 'px-2 py-1 rounded-full text-xs font-semibold bg-green-100 text-green-800'
} else {
statusBadge.className = 'px-2 py-1 rounded-full text-xs font-semibold bg-yellow-100 text-yellow-800'
}
} catch (error) {
console.error('Error fetching status:', error)
document.getElementById('api-status').innerHTML = `
<div class="bg-red-50 border-l-4 border-red-500 p-4">
<div class="flex">
<div class="flex-shrink-0">
<svg class="h-5 w-5 text-red-400" viewBox="0 0 20 20" fill="currentColor">
<path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clip-rule="evenodd" />
</svg>
</div>
<div class="ml-3">
<p class="text-sm text-red-700">Failed to load status: ${ error.message }</p>
<button onclick="fetchStatus()" class="mt-2 text-sm text-red-700 hover:text-red-600 font-medium">
Retry →
</button>
</div>
</div>
</div>
`
}
}
// Fetch API status on load
async function fetchStatus3() {
try {
const response = await fetch('/api/status')
const data = await response.json()
document.getElementById('api-status').innerHTML = `
<p><strong>Application:</strong> ${ data.application }</p>
<p><strong>Status:</strong> <span class="text-green-600 font-semibold">${ data.status }</span></p>
<p><strong>Version:</strong> ${ data.version }</p>
<p><strong>Timestamp:</strong> ${ data.timestamp }</p>
`
} catch (error) {
document.getElementById('api-status').innerHTML = `
<p class="text-red-600">Error loading status: ${ error.message }</p>
`
}
}
// Test greeting API
document.getElementById('test-api').addEventListener('click', async () => {
try {
const response = await fetch('/api/hello')
const data = await response.json()
document.getElementById('api-response').innerHTML = `
<pre class="text-sm text-gray-700">${ JSON.stringify(data, null, 2) }</pre>
`
} catch (error) {
document.getElementById('api-response').innerHTML = `
<pre class="text-sm text-red-600">Error: ${ error.message }</pre>
`
}
})
// Auto-refresh every 30 seconds
let refreshInterval = setInterval(fetchStatus, 30000);
// Stop auto-refresh when page loses focus (optional)
document.addEventListener('visibilitychange', function() {
if (document.hidden) {
clearInterval(refreshInterval);
} else {
refreshInterval = setInterval(fetchStatus, 30000);
fetchStatus(); // Refresh immediately when returning to tab
}
});
// Load status on page load
fetchStatus()
</script>
</body>
</html>

26
src/test/java/auctiora/AuctionParsingTest.java
View File

@@ -1,5 +1,6 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
@@ -19,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.*;
* Test auction parsing logic using saved HTML from test.html
* Tests the markup data extraction for each auction found
*/
@Slf4j
public class AuctionParsingTest {
private static String testHtml;
@@ -27,12 +29,12 @@ public class AuctionParsingTest {
public static void loadTestHtml() throws IOException {
// Load the test HTML file
testHtml = Files.readString(Paths.get("src/test/resources/test_auctions.html"));
System.out.println("Loaded test HTML (" + testHtml.length() + " characters)");
log.info("Loaded test HTML ({} characters)", testHtml.length());
}
@Test
public void testLocationPatternMatching() {
System.out.println("\n=== Location Pattern Tests ===");
log.info("\n=== Location Pattern Tests ===");
// Test different location formats
var testCases = new String[]{
@@ -48,16 +50,16 @@ public class AuctionParsingTest {
if (elem != null) {
var text = elem.text();
System.out.println("\nTest: " + testHtml);
System.out.println("Text: " + text);
log.info("\nTest: {}", testHtml);
log.info("Text: {}", text);
// Test regex pattern
if (text.matches(".*[A-Z]{2}$")) {
var countryCode = text.substring(text.length() - 2);
var cityPart = text.substring(0, text.length() - 2).trim().replaceAll("[,\\s]+$", "");
System.out.println("→ Extracted: " + cityPart + ", " + countryCode);
log.info("→ Extracted: {}, {}", cityPart, countryCode);
} else {
System.out.println("→ No match");
log.info("→ No match");
}
}
}
@@ -65,7 +67,7 @@ public class AuctionParsingTest {
@Test
public void testFullTextPatternMatching() {
System.out.println("\n=== Full Text Pattern Tests ===");
log.info("\n=== Full Text Pattern Tests ===");
// Test the complete auction text format
var testCases = new String[]{
@@ -75,7 +77,7 @@ public class AuctionParsingTest {
};
for (var testText : testCases) {
System.out.println("\nParsing: \"" + testText + "\"");
log.info("\nParsing: \"{}\"", testText);
// Simulated extraction
var remaining = testText;
@@ -84,7 +86,7 @@ public class AuctionParsingTest {
var timePattern = java.util.regex.Pattern.compile("(\\w+)\\s+om\\s+(\\d{1,2}:\\d{2})");
var timeMatcher = timePattern.matcher(remaining);
if (timeMatcher.find()) {
System.out.println(" Time: " + timeMatcher.group(1) + " om " + timeMatcher.group(2));
log.info(" Time: {} om {}", timeMatcher.group(1), timeMatcher.group(2));
remaining = remaining.substring(timeMatcher.end()).trim();
}
@@ -94,7 +96,7 @@ public class AuctionParsingTest {
);
var locMatcher = locPattern.matcher(remaining);
if (locMatcher.find()) {
System.out.println(" Location: " + locMatcher.group(1) + ", " + locMatcher.group(2));
log.info(" Location: {}, {}", locMatcher.group(1), locMatcher.group(2));
remaining = remaining.substring(0, locMatcher.start()).trim();
}
@@ -102,12 +104,12 @@ public class AuctionParsingTest {
var lotPattern = java.util.regex.Pattern.compile("^(\\d+)\\s+");
var lotMatcher = lotPattern.matcher(remaining);
if (lotMatcher.find()) {
System.out.println(" Lot count: " + lotMatcher.group(1));
log.info(" Lot count: {}", lotMatcher.group(1));
remaining = remaining.substring(lotMatcher.end()).trim();
}
// What remains is title
System.out.println(" Title: " + remaining);
log.info(" Title: {}", remaining);
}
}
}

4
src/test/java/auctiora/ImageProcessingServiceTest.java
View File

@@ -19,14 +19,14 @@ class ImageProcessingServiceTest {
private DatabaseService mockDb;
private ObjectDetectionService mockDetector;
private RateLimitedHttpClient mockHttpClient;
private RateLimitedHttpClient2 mockHttpClient;
private ImageProcessingService service;
@BeforeEach
void setUp() {
mockDb = mock(DatabaseService.class);
mockDetector = mock(ObjectDetectionService.class);
mockHttpClient = mock(RateLimitedHttpClient.class);
mockHttpClient = mock(RateLimitedHttpClient2.class);
service = new ImageProcessingService(mockDb, mockDetector, mockHttpClient);
}

4
src/test/java/auctiora/IntegrationTest.java
View File

@@ -40,7 +40,7 @@ class IntegrationTest {
db = new DatabaseService(testDbPath);
db.ensureSchema();
notifier = new NotificationService("desktop", "");
notifier = new NotificationService("desktop");
detector = new ObjectDetectionService(
"non_existent.cfg",
@@ -48,7 +48,7 @@ class IntegrationTest {
"non_existent.txt"
);
RateLimitedHttpClient httpClient = new RateLimitedHttpClient();
RateLimitedHttpClient2 httpClient = new RateLimitedHttpClient2();
imageProcessor = new ImageProcessingService(db, detector, httpClient);
monitor = new TroostwijkMonitor(

51
src/test/java/auctiora/NotificationServiceTest.java
View File

@@ -14,7 +14,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should initialize with desktop-only configuration")
void testDesktopOnlyConfiguration() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertNotNull(service);
}
@@ -22,8 +22,7 @@ class NotificationServiceTest {
@DisplayName("Should initialize with SMTP configuration")
void testSMTPConfiguration() {
NotificationService service = new NotificationService(
"smtp:test@gmail.com:app_password:recipient@example.com",
""
"smtp:test@gmail.com:app_password:recipient@example.com"
);
assertNotNull(service);
}
@@ -33,12 +32,12 @@ class NotificationServiceTest {
void testInvalidSMTPConfiguration() {
// Missing parts
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:incomplete", "")
new NotificationService("smtp:incomplete")
);
// Wrong format
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:only:two:parts", "")
new NotificationService("smtp:only:two:parts")
);
}
@@ -46,14 +45,14 @@ class NotificationServiceTest {
@DisplayName("Should reject unknown configuration type")
void testUnknownConfiguration() {
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("unknown_type", "")
new NotificationService("unknown_type")
);
}
@Test
@DisplayName("Should send desktop notification without error")
void testDesktopNotification() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
// Should not throw exception even if system tray not available
assertDoesNotThrow(() ->
@@ -64,7 +63,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should send high priority notification")
void testHighPriorityNotification() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("Urgent message", "High Priority", 1)
@@ -74,7 +73,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should send normal priority notification")
void testNormalPriorityNotification() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("Regular message", "Normal Priority", 0)
@@ -84,7 +83,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should handle notification when system tray not supported")
void testNoSystemTraySupport() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
// Should gracefully handle missing system tray
assertDoesNotThrow(() ->
@@ -98,8 +97,7 @@ class NotificationServiceTest {
// Note: This won't actually send email without valid credentials
// But it should initialize properly
NotificationService service = new NotificationService(
"smtp:test@gmail.com:fake_password:test@example.com",
""
"smtp:test@gmail.com:fake_password:test@example.com"
);
// Should not throw during initialization
@@ -115,8 +113,7 @@ class NotificationServiceTest {
@DisplayName("Should include both desktop and email when SMTP configured")
void testBothNotificationChannels() {
NotificationService service = new NotificationService(
"smtp:user@gmail.com:password:recipient@example.com",
""
"smtp:user@gmail.com:password:recipient@example.com"
);
// Both desktop and email should be attempted
@@ -128,7 +125,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should handle empty message gracefully")
void testEmptyMessage() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("", "", 0)
@@ -138,7 +135,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should handle very long message")
void testLongMessage() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
String longMessage = "A".repeat(1000);
assertDoesNotThrow(() ->
@@ -149,7 +146,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should handle special characters in message")
void testSpecialCharactersInMessage() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification(
@@ -164,9 +161,9 @@ class NotificationServiceTest {
@DisplayName("Should accept case-insensitive desktop config")
void testCaseInsensitiveDesktopConfig() {
assertDoesNotThrow(() -> {
new NotificationService("DESKTOP", "");
new NotificationService("Desktop", "");
new NotificationService("desktop", "");
new NotificationService("DESKTOP");
new NotificationService("Desktop");
new NotificationService("desktop");
});
}
@@ -175,19 +172,19 @@ class NotificationServiceTest {
void testSMTPConfigPartsValidation() {
// Too few parts
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:user:pass", "")
new NotificationService("smtp:user:pass")
);
// Too many parts should work (extras ignored in split)
assertDoesNotThrow(() ->
new NotificationService("smtp:user:pass:email:extra", "")
new NotificationService("smtp:user:pass:email:extra")
);
}
@Test
@DisplayName("Should handle multiple rapid notifications")
void testRapidNotifications() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() -> {
for (int i = 0; i < 5; i++) {
@@ -201,14 +198,14 @@ class NotificationServiceTest {
void testNullConfigParameter() {
// Second parameter can be empty string (kept for compatibility)
assertDoesNotThrow(() ->
new NotificationService("desktop", null)
new NotificationService("desktop")
);
}
@Test
@DisplayName("Should send bid change notification format")
void testBidChangeNotificationFormat() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
String message = "Nieuw bod op kavel 12345: €150.00 (was €125.00)";
String title = "Kavel bieding update";
@@ -221,7 +218,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should send closing alert notification format")
void testClosingAlertNotificationFormat() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
String message = "Kavel 12345 sluit binnen 5 min.";
String title = "Lot nearing closure";
@@ -234,7 +231,7 @@ class NotificationServiceTest {
@Test
@DisplayName("Should send object detection notification format")
void testObjectDetectionNotificationFormat() {
NotificationService service = new NotificationService("desktop", "");
NotificationService service = new NotificationService("desktop");
String message = "Lot contains: car, truck, machinery\nEstimated value: €5000";
String title = "Object Detected";

60
src/test/java/auctiora/Parser.java
View File

File diff suppressed because one or more lines are too long

62
src/test/java/auctiora/ParserTest.java Normal file
View File

File diff suppressed because one or more lines are too long

10
src/test/java/auctiora/ScraperDataAdapterTest.java
View File

@@ -55,9 +55,9 @@ class ScraperDataAdapterTest {
assertEquals("Cluj-Napoca", result.city());
assertEquals("RO", result.country());
assertEquals("https://example.com/auction/A7-39813", result.url());
assertEquals("A7", result.type());
assertEquals("A7", result.typePrefix());
assertEquals(150, result.lotCount());
assertNotNull(result.closingTime());
assertNotNull(result.firstLotClosingTime());
}
@Test
@@ -75,7 +75,7 @@ class ScraperDataAdapterTest {
assertEquals("Amsterdam", result.city());
assertEquals("", result.country());
assertNull(result.closingTime());
assertNull(result.firstLotClosingTime());
}
@Test
@@ -196,7 +196,7 @@ class ScraperDataAdapterTest {
when(rs1.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo auction1 = ScraperDataAdapter.fromScraperAuction(rs1);
assertEquals("A7", auction1.type());
assertEquals("A7", auction1.typePrefix());
ResultSet rs2 = mock(ResultSet.class);
when(rs2.getString("auction_id")).thenReturn("B1-12345");
@@ -207,7 +207,7 @@ class ScraperDataAdapterTest {
when(rs2.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo auction2 = ScraperDataAdapter.fromScraperAuction(rs2);
assertEquals("B1", auction2.type());
assertEquals("B1", auction2.typePrefix());
}
@Test

48
src/test/java/auctiora/TroostwijkMonitorTest.java
View File

@@ -43,7 +43,7 @@ class TroostwijkMonitorTest {
@DisplayName("Should initialize monitor successfully")
void testMonitorInitialization() {
assertNotNull(monitor);
assertNotNull(monitor.db);
assertNotNull(monitor.getDb());
}
@Test
@@ -61,8 +61,8 @@ class TroostwijkMonitorTest {
@Test
@DisplayName("Should handle empty database gracefully")
void testEmptyDatabaseHandling() throws SQLException {
var auctions = monitor.db.getAllAuctions();
var lots = monitor.db.getAllLots();
var auctions = monitor.getDb().getAllAuctions();
var lots = monitor.getDb().getAllLots();
assertNotNull(auctions);
assertNotNull(lots);
@@ -88,9 +88,9 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(lot);
monitor.getDb().upsertLot(lot);
var lots = monitor.db.getAllLots();
var lots = monitor.getDb().getAllLots();
assertTrue(lots.stream().anyMatch(l -> l.lotId() == 22222));
}
@@ -113,9 +113,9 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(closingSoon);
monitor.getDb().upsertLot(closingSoon);
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 44444)
.findFirst()
@@ -143,9 +143,9 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(futureLot);
monitor.getDb().upsertLot(futureLot);
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 66666)
.findFirst()
@@ -173,9 +173,9 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(noClosing);
monitor.getDb().upsertLot(noClosing);
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 88888)
.findFirst()
@@ -203,7 +203,7 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(lot);
monitor.getDb().upsertLot(lot);
// Update notification flag
var notified = new Lot(
@@ -221,9 +221,9 @@ class TroostwijkMonitorTest {
true
);
monitor.db.updateLotNotificationFlags(notified);
monitor.getDb().updateLotNotificationFlags(notified);
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 11110)
.findFirst()
@@ -251,7 +251,7 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(lot);
monitor.getDb().upsertLot(lot);
// Simulate bid increase
var higherBid = new Lot(
@@ -269,9 +269,9 @@ class TroostwijkMonitorTest {
false
);
monitor.db.updateLotCurrentBid(higherBid);
monitor.getDb().updateLotCurrentBid(higherBid);
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 13131)
.findFirst()
@@ -287,7 +287,7 @@ class TroostwijkMonitorTest {
Thread t1 = new Thread(() -> {
try {
for (int i = 0; i < 5; i++) {
monitor.db.upsertLot(new Lot(
monitor.getDb().upsertLot(new Lot(
20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
100.0, "EUR", "https://example.com/" + i, null, false
));
@@ -300,7 +300,7 @@ class TroostwijkMonitorTest {
Thread t2 = new Thread(() -> {
try {
for (int i = 5; i < 10; i++) {
monitor.db.upsertLot(new Lot(
monitor.getDb().upsertLot(new Lot(
20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
200.0, "EUR", "https://example.com/" + i, null, false
));
@@ -315,7 +315,7 @@ class TroostwijkMonitorTest {
t1.join();
t2.join();
var lots = monitor.db.getActiveLots();
var lots = monitor.getDb().getActiveLots();
long count = lots.stream()
.filter(l -> l.lotId() >= 30000 && l.lotId() < 30010)
.count();
@@ -351,7 +351,7 @@ class TroostwijkMonitorTest {
LocalDateTime.now().plusDays(2)
);
monitor.db.upsertAuction(auction);
monitor.getDb().upsertAuction(auction);
// Insert related lot
var lot = new Lot(
@@ -369,11 +369,11 @@ class TroostwijkMonitorTest {
false
);
monitor.db.upsertLot(lot);
monitor.getDb().upsertLot(lot);
// Verify
var auctions = monitor.db.getAllAuctions();
var lots = monitor.db.getAllLots();
var auctions = monitor.getDb().getAllAuctions();
var lots = monitor.getDb().getAllLots();
assertTrue(auctions.stream().anyMatch(a -> a.auctionId() == 40000));
assertTrue(lots.stream().anyMatch(l -> l.lotId() == 50000));

20
workflows/maven.yml Normal file
View File

@@ -0,0 +1,20 @@
name: Publish to Gitea Package Registry
on:
push:
tags:
- 'v*'
jobs:
publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up JDK 25
uses: actions/setup-java@v4
with:
java-version: '25'
distribution: 'temurin'
- name: Publish with Maven
run: mvn --batch-mode clean deploy
env:
GITEA_TOKEN: ${{ secrets.EA_PUBLISH_TOKEN }}