Tour/auctiora
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix mock tests

Browse Source
This commit is contained in:
Tour
2025-12-04 19:38:38 +01:00
parent 9857f053a1
commit ed74bb5e93
34 changed files with 2312 additions and 2006 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
.idea/compiler.xml generated
View File

@@ -2,17 +2,37 @@
<project version="4"> <project version="4">
<component name="CompilerConfiguration"> <component name="CompilerConfiguration">
<annotationProcessing> <annotationProcessing>
<profile default="true" name="Default" enabled="true" />
<profile name="Maven default annotation processors profile" enabled="true"> <profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" /> <sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" /> <sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" /> <outputRelativeToContentRoot value="true" />
<module name="troostwijk-scraper" /> </profile>
<profile name="Annotation profile for Troostwijk Auction Scraper" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<processorPath useClasspath="false">
<entry name="$MAVEN_REPOSITORY$/org/projectlombok/lombok/1.18.40/lombok-1.18.40.jar" />
<entry name="$MAVEN_REPOSITORY$/io/quarkus/quarkus-extension-processor/3.17.7/quarkus-extension-processor-3.17.7.jar" />
<entry name="$MAVEN_REPOSITORY$/org/jboss/jdeparser/jdeparser/2.0.3.Final/jdeparser-2.0.3.Final.jar" />
<entry name="$MAVEN_REPOSITORY$/org/jsoup/jsoup/1.15.3/jsoup-1.15.3.jar" />
<entry name="$MAVEN_REPOSITORY$/com/github/javaparser/javaparser-core/3.26.2/javaparser-core-3.26.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-databind/2.18.2/jackson-databind-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-annotations/2.18.2/jackson-annotations-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/core/jackson-core/2.18.2/jackson-core-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/dataformat/jackson-dataformat-yaml/2.18.2/jackson-dataformat-yaml-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/org/yaml/snakeyaml/2.3/snakeyaml-2.3.jar" />
<entry name="$MAVEN_REPOSITORY$/com/fasterxml/jackson/module/jackson-module-parameter-names/2.18.2/jackson-module-parameter-names-2.18.2.jar" />
<entry name="$MAVEN_REPOSITORY$/io/quarkus/quarkus-bootstrap-app-model/3.17.7/quarkus-bootstrap-app-model-3.17.7.jar" />
</processorPath>
<module name="auctiora" />
</profile> </profile>
</annotationProcessing> </annotationProcessing>
</component> </component>
<component name="JavacSettings"> <component name="JavacSettings">
<option name="ADDITIONAL_OPTIONS_OVERRIDE"> <option name="ADDITIONAL_OPTIONS_OVERRIDE">
<module name="troostwijk-scraper" options="" /> <module name="auctiora" options="-Xdiags:verbose -Xlint:all -parameters" />
</option> </option>
</component> </component>
</project> </project>

5
.idea/jarRepositories.xml generated
View File

@@ -1,6 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="RemoteRepositoriesConfiguration"> <component name="RemoteRepositoriesConfiguration">
<remote-repository>
<option name="id" value="jitpack.io" />
<option name="name" value="jitpack.io" />
<option name="url" value="https://jitpack.io" />
</remote-repository>
<remote-repository> <remote-repository>
<option name="id" value="central" /> <option name="id" value="central" />
<option name="name" value="Apache Central Repository" /> <option name="name" value="Apache Central Repository" />

67
Dockerfile
View File

@@ -1,52 +1,27 @@
# Multi-stage Dockerfile for Quarkus Auction Monitor # ==================== BUILD STAGE ====================
FROM maven:3.9-eclipse-temurin-25-alpine AS builder
# Build stage
FROM maven:3.9-eclipse-temurin-25-alpine AS build
WORKDIR /app WORKDIR /app
# Copy POM first (allows for cached dependency layer)
COPY pom.xml .
# This will now work if the opencv dependency has no classifier
# -----LOCAL----
RUN mvn dependency:resolve -B
# -----LOCAL----
# RUN mvn dependency:go-offline -B
# Copy Maven files for dependency caching COPY src ./src
COPY pom.xml ./ # Updated with both properties to avoid the warning
RUN mvn dependency:go-offline -B RUN mvn package -DskipTests -Dquarkus.package.jar.type=uber-jar -Dquarkus.package.jar.enabled=true
# Copy source code
COPY src/ ./src/
# Build Quarkus application (fast-jar for production)
RUN mvn package -DskipTests -Dquarkus.package.jar.type=fast-jar
# Runtime stage
FROM eclipse-temurin:25-jre-alpine
# ==================== RUNTIME STAGE ====================
FROM eclipse-temurin:25-jre
WORKDIR /app WORKDIR /app
RUN groupadd -r quarkus && useradd -r -g quarkus quarkusa
# Create non-root user COPY --from=builder --chown=quarkus:quarkus /app/target/scrape-ui-*.jar app.jar
RUN addgroup -g 1001 quarkus && \
adduser -u 1001 -G quarkus -s /bin/sh -D quarkus
# Create directories for data
RUN mkdir -p /mnt/okcomputer/output/images && \
chown -R quarkus:quarkus /mnt/okcomputer
# Copy Quarkus fast-jar structure
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/lib/ /app/lib/
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/*.jar /app/
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/app/ /app/app/
COPY --from=build --chown=quarkus:quarkus /app/target/quarkus-app/quarkus/ /app/quarkus/
# Switch to non-root user
USER quarkus USER quarkus
# Expose ports
EXPOSE 8081 EXPOSE 8081
ENTRYPOINT ["java", \
# Set environment variables "-Dio.netty.tryReflectionSetAccessible=true", \
ENV JAVA_OPTS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" "--enable-native-access=ALL-UNNAMED", \
ENV JAVA_APP_JAR="/app/quarkus-run.jar" "--sun-misc-unsafe-memory-access=allow", \
"-jar", "app.jar"]
# Health check
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD wget --no-verbose --tries=1 --spider http://localhost:8081/health/live || exit 1
# Run the Quarkus application
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar $JAVA_APP_JAR"]

11
auction.iml
View File

@@ -1,11 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

11
docker-compose.yml
View File

@@ -1,11 +1,8 @@
version: '3.8'
services: services:
auction-monitor: sophena:
build: build:
context: . dockerfile: .
dockerfile: Dockerfile container_name: sophena
container_name: troostwijk-auction-monitor
ports: ports:
- "8081:8081" - "8081:8081"
volumes: volumes:
@@ -39,6 +36,7 @@ services:
- AUCTION_WORKFLOW_IMAGE_PROCESSING_CRON=0 0 * * * ? - AUCTION_WORKFLOW_IMAGE_PROCESSING_CRON=0 0 * * * ?
- AUCTION_WORKFLOW_BID_MONITORING_CRON=0 */15 * * * ? - AUCTION_WORKFLOW_BID_MONITORING_CRON=0 */15 * * * ?
- AUCTION_WORKFLOW_CLOSING_ALERTS_CRON=0 */5 * * * ? - AUCTION_WORKFLOW_CLOSING_ALERTS_CRON=0 */5 * * * ?
- JAVA_TOOL_OPTIONS=-Dio.netty.tryReflectionSetAccessible=true --enable-native-access=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow
healthcheck: healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8081/health/live"] test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8081/health/live"]
@@ -52,6 +50,7 @@ services:
networks: networks:
- auction-network - auction-network
networks: networks:
auction-network: auction-network:
driver: bridge driver: bridge

204
pom.xml
View File

@@ -11,16 +11,39 @@
<name>Troostwijk Auction Scraper</name> <name>Troostwijk Auction Scraper</name>
<description>Web scraper for Troostwijk Auctions with object detection and notifications</description> <description>Web scraper for Troostwijk Auctions with object detection and notifications</description>
<repositories>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
</repositories>
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>25</maven.compiler.source> <maven.compiler.source>25</maven.compiler.source>
<maven.compiler.target>25</maven.compiler.target> <maven.compiler.target>25</maven.compiler.target>
<maven.compiler.release>25</maven.compiler.release>
<jackson.version>2.17.0</jackson.version> <jackson.version>2.17.0</jackson.version>
<opencv.version>4.9.0-0</opencv.version> <opencv.version>4.9.0-0</opencv.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<quarkus.platform.version>3.17.7</quarkus.platform.version> <quarkus.platform.version>3.17.7</quarkus.platform.version>
<asm.version>9.8</asm.version> <asm.version>9.8</asm.version>
<lombok.version>1.18.40</lombok.version>
<!--this is not a bug, its feature-->
<lombok-version>${lombok.version}</lombok-version>
<lombok-maven-version>1.18.20.0</lombok-maven-version>
<maven-compiler-plugin-version>3.14.0</maven-compiler-plugin-version>
<versions-maven-plugin.version>2.19.0</versions-maven-plugin.version>
<jandex-maven-plugin-version>3.5.0</jandex-maven-plugin-version>
<maven.compiler.args>
--enable-native-access=ALL-UNNAMED
--add-opens java.base/sun.misc=ALL-UNNAMED
-Xdiags:verbose
-Xlint:all
</maven.compiler.args>
<uberJar>true</uberJar> <!-- Your existing properties... -->
<quarkus.package.jar.type>uber-jar</quarkus.package.jar.type>
<quarkus.package.jar.enabled>true</quarkus.package.jar.enabled>
<maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
</properties> </properties>
<dependencyManagement> <dependencyManagement>
@@ -53,10 +76,28 @@
<artifactId>asm-util</artifactId> <artifactId>asm-util</artifactId>
<version>${asm.version}</version> <version>${asm.version}</version>
</dependency> </dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>4.1.124.Final</version> <!-- This version has the fix -->
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>
<dependencies> <dependencies>
<dependency>
<groupId>com.github.vladimir-bukhtoyarov.bucket4j</groupId>
<artifactId>bucket4j-core</artifactId>
<version>7.6.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.github.bucket4j/bucket4j -->
<!--<dependency>
<groupId>io.github.bucket4j</groupId>
<artifactId>bucket4j</artifactId>
<version>8.9.0</version>
</dependency>-->
<!-- JSoup for HTML parsing and HTTP client --> <!-- JSoup for HTML parsing and HTTP client -->
<dependency> <dependency>
<groupId>org.jsoup</groupId> <groupId>org.jsoup</groupId>
@@ -77,7 +118,19 @@
<artifactId>sqlite-jdbc</artifactId> <artifactId>sqlite-jdbc</artifactId>
<version>3.45.1.0</version> <version>3.45.1.0</version>
</dependency> </dependency>
<!-- Lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven</artifactId>
<version>${lombok-maven-version}</version>
<type>pom</type>
</dependency>
<!-- JavaMail API for email notifications --> <!-- JavaMail API for email notifications -->
<dependency> <dependency>
<groupId>com.sun.mail</groupId> <groupId>com.sun.mail</groupId>
@@ -85,12 +138,6 @@
<version>1.6.2</version> <version>1.6.2</version>
</dependency> </dependency>
<!-- OpenCV for image processing and object detection -->
<dependency>
<groupId>org.openpnp</groupId>
<artifactId>opencv</artifactId>
<version>${opencv.version}</version>
</dependency>
<dependency> <dependency>
<groupId>com.microsoft.playwright</groupId> <groupId>com.microsoft.playwright</groupId>
<artifactId>playwright</artifactId> <artifactId>playwright</artifactId>
@@ -167,6 +214,14 @@
<artifactId>quarkus-config-yaml</artifactId> <artifactId>quarkus-config-yaml</artifactId>
</dependency> </dependency>
<!-- OSGi annotations -->
<dependency>
<groupId>org.osgi</groupId>
<artifactId>org.osgi.annotation.bundle</artifactId>
<version>2.0.0</version>
<scope>provided</scope>
</dependency>
<!-- Test dependencies --> <!-- Test dependencies -->
<dependency> <dependency>
<groupId>io.quarkus</groupId> <groupId>io.quarkus</groupId>
@@ -178,10 +233,23 @@
<artifactId>rest-assured</artifactId> <artifactId>rest-assured</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.openpnp</groupId>
<artifactId>opencv</artifactId>
<version>4.9.0-0</version>
<!--<classifier>windows-x86_64</classifier>-->
</dependency>
</dependencies> </dependencies>
<build> <build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins> <plugins>
<plugin> <plugin>
<groupId>io.quarkus.platform</groupId> <groupId>io.quarkus.platform</groupId>
<artifactId>quarkus-maven-plugin</artifactId> <artifactId>quarkus-maven-plugin</artifactId>
@@ -196,46 +264,74 @@
</goals> </goals>
</execution> </execution>
</executions> </executions>
</plugin>
<!-- Maven Compiler Plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.11.0</version>
<configuration> <configuration>
<source>25</source> <properties>
<target>25</target> <build.timestamp>${maven.build.timestamp}</build.timestamp>
</properties>
</configuration> </configuration>
</plugin> </plugin>
<!-- Maven Exec Plugin for running with native access -->
<plugin> <plugin>
<groupId>org.codehaus.mojo</groupId> <groupId>org.projectlombok</groupId>
<artifactId>exec-maven-plugin</artifactId> <artifactId>lombok-maven-plugin</artifactId>
<version>3.1.0</version> <version>${lombok-maven-version}</version>
<configuration>
<mainClass>com.auction.TroostwijkAuctionExtractor</mainClass>
<cleanupDaemonThreads>false</cleanupDaemonThreads>
<arguments>
<argument>--max-visits</argument>
<argument>3</argument>
</arguments>
<additionalClasspathElements>
<!--suppress MavenModelInspection -->
<additionalClasspathElement>${project.build.outputDirectory}</additionalClasspathElement>
</additionalClasspathElements>
<commandlineArgs>--enable-native-access=ALL-UNNAMED</commandlineArgs>
</configuration>
<executions> <executions>
<execution> <execution>
<phase>generate-sources</phase>
<goals> <goals>
<goal>java</goal> <goal>delombok</goal>
</goals> </goals>
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
<!-- Maven Exec Plugin for running with native access -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin-version}</version>
<configuration>
<release>${maven.compiler.release}</release>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok-version}</version>
</path>
<path>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-extension-processor</artifactId>
<version>${quarkus.platform.version}</version>
</path>
</annotationProcessorPaths>
<compilerArgs>
<arg>-Xdiags:verbose</arg>
<arg>-Xlint:all</arg>
<arg>-parameters</arg>
</compilerArgs>
<fork>true</fork>
<excludes>
<exclude>module-info.java</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
<version>${versions-maven-plugin.version}</version>
</plugin>
<!-- Maven Surefire Plugin for tests with native access --> <!-- Maven Surefire Plugin for tests with native access -->
<plugin>
<groupId>io.smallrye</groupId>
<artifactId>jandex-maven-plugin</artifactId>
<version>${jandex-maven-plugin-version}</version>
<executions>
<execution>
<id>make-index</id>
<goals>
<goal>jandex</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
@@ -262,31 +358,17 @@
</archive> </archive>
</configuration> </configuration>
</plugin>--> </plugin>-->
<!-- Maven Assembly Plugin for creating executable JAR with dependencies -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<archive>
<manifest>
<mainClass>com.auction.Main</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins> </plugins>
</build> </build>
<!-- In your pom.xml, alongside <build> and <dependencies> -->
<distributionManagement>
<repository>
<id>gitea</id>
<url>https://git.appmodel.nl/api/packages/Tour/maven</url>
</repository>
<snapshotRepository>
<id>gitea</id>
<url>https://git.appmodel.nl/api/packages/Tour/maven</url>
</snapshotRepository>
</distributionManagement>
</project> </project>

6
src/main/java/auctiora/AuctionInfo.java
View File

@@ -13,7 +13,7 @@ public record AuctionInfo(
String city, // City name String city, // City name
String country, // Country code (e.g., "NL") String country, // Country code (e.g., "NL")
String url, // Full auction URL String url, // Full auction URL
String type, // Auction type (A1 or A7) String typePrefix, // Auction type (A1 or A7)
int lotCount, // Number of lots/kavels int lotCount, // Number of lots/kavels
LocalDateTime closingTime // Closing time if available LocalDateTime firstLotClosingTime // Closing time if available
) {} ) { }

4
src/main/java/auctiora/AuctionMonitorProducer.java
View File

@@ -35,7 +35,7 @@ public class AuctionMonitorProducer {
@ConfigProperty(name = "auction.notification.config") String config) { @ConfigProperty(name = "auction.notification.config") String config) {
LOG.infof("Initializing NotificationService with config: %s", config); LOG.infof("Initializing NotificationService with config: %s", config);
return new NotificationService(config, ""); return new NotificationService(config);
} }
@Produces @Produces
@@ -54,7 +54,7 @@ public class AuctionMonitorProducer {
public ImageProcessingService produceImageProcessingService( public ImageProcessingService produceImageProcessingService(
DatabaseService db, DatabaseService db,
ObjectDetectionService detector, ObjectDetectionService detector,
RateLimitedHttpClient httpClient) { RateLimitedHttpClient2 httpClient) {
LOG.infof("Initializing ImageProcessingService"); LOG.infof("Initializing ImageProcessingService");
return new ImageProcessingService(db, detector, httpClient); return new ImageProcessingService(db, detector, httpClient);

2
src/main/java/auctiora/AuctionMonitorResource.java
View File

@@ -33,7 +33,7 @@ public class AuctionMonitorResource {
NotificationService notifier; NotificationService notifier;
@Inject @Inject
RateLimitedHttpClient httpClient; RateLimitedHttpClient2 httpClient;
/** /**
* GET /api/monitor/status * GET /api/monitor/status

10
src/main/java/auctiora/Console.java
View File

@@ -1,10 +0,0 @@
package auctiora;
/**
* Simple console output utility (renamed from IO to avoid Java 25 conflict)
*/
class Console {
static void println(String message) {
System.out.println(message);
}
}

15
src/main/java/auctiora/DatabaseService.java
View File

@@ -1,5 +1,7 @@
package auctiora; package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.io.Console;
import java.sql.DriverManager; import java.sql.DriverManager;
import java.sql.SQLException; import java.sql.SQLException;
import java.time.Instant; import java.time.Instant;
@@ -12,6 +14,7 @@ import java.util.List;
* Data is typically populated by an external scraper process; * Data is typically populated by an external scraper process;
* this service enriches it with image processing and monitoring. * this service enriches it with image processing and monitoring.
*/ */
@Slf4j
public class DatabaseService { public class DatabaseService {
private final String url; private final String url;
@@ -104,9 +107,9 @@ public class DatabaseService {
ps.setString(4, auction.city()); ps.setString(4, auction.city());
ps.setString(5, auction.country()); ps.setString(5, auction.country());
ps.setString(6, auction.url()); ps.setString(6, auction.url());
ps.setString(7, auction.type()); ps.setString(7, auction.typePrefix());
ps.setInt(8, auction.lotCount()); ps.setInt(8, auction.lotCount());
ps.setString(9, auction.closingTime() != null ? auction.closingTime().toString() : null); ps.setString(9, auction.firstLotClosingTime() != null ? auction.firstLotClosingTime().toString() : null);
ps.setLong(10, Instant.now().getEpochSecond()); ps.setLong(10, Instant.now().getEpochSecond());
ps.executeUpdate(); ps.executeUpdate();
} }
@@ -383,7 +386,7 @@ public class DatabaseService {
} }
} catch (SQLException e) { } catch (SQLException e) {
// Table might not exist in scraper format - that's ok // Table might not exist in scraper format - that's ok
Console.println("ℹ️ Scraper lots table not found or incompatible schema"); log.info("ℹ️ Scraper lots table not found or incompatible schema");
} }
return imported; return imported;
@@ -421,7 +424,7 @@ public class DatabaseService {
)); ));
} }
} catch (SQLException e) { } catch (SQLException e) {
Console.println("ℹ️ No unprocessed images found in scraper format"); log.info("ℹ️ No unprocessed images found in scraper format");
} }
return images; return images;
@@ -430,10 +433,10 @@ public class DatabaseService {
/** /**
* Simple record for image data from database * Simple record for image data from database
*/ */
record ImageRecord(int id, int lotId, String url, String filePath, String labels) {} record ImageRecord(int id, int lotId, String url, String filePath, String labels) { }
/** /**
* Record for importing images from scraper format * Record for importing images from scraper format
*/ */
record ImageImportRecord(int lotId, int saleId, String url) {} record ImageImportRecord(int lotId, int saleId, String url) { }
} }

17
src/main/java/auctiora/ImageProcessingService.java
View File

@@ -1,5 +1,7 @@
package auctiora; package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.io.Console;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
@@ -13,13 +15,14 @@ import java.util.List;
* This separates image processing concerns from scraping, allowing this project * This separates image processing concerns from scraping, allowing this project
* to focus on enriching data scraped by the external process. * to focus on enriching data scraped by the external process.
*/ */
@Slf4j
class ImageProcessingService { class ImageProcessingService {
private final RateLimitedHttpClient httpClient; private final RateLimitedHttpClient2 httpClient;
private final DatabaseService db; private final DatabaseService db;
private final ObjectDetectionService detector; private final ObjectDetectionService detector;
ImageProcessingService(DatabaseService db, ObjectDetectionService detector, RateLimitedHttpClient httpClient) { ImageProcessingService(DatabaseService db, ObjectDetectionService detector, RateLimitedHttpClient2 httpClient) {
this.httpClient = httpClient; this.httpClient = httpClient;
this.db = db; this.db = db;
this.detector = detector; this.detector = detector;
@@ -73,7 +76,7 @@ class ImageProcessingService {
* @param imageUrls list of image URLs to process * @param imageUrls list of image URLs to process
*/ */
void processImagesForLot(int lotId, int saleId, List<String> imageUrls) { void processImagesForLot(int lotId, int saleId, List<String> imageUrls) {
Console.println(" Processing " + imageUrls.size() + " images for lot " + lotId); log.info(" Processing {} images for lot {}", imageUrls.size(), lotId);
for (var imgUrl : imageUrls) { for (var imgUrl : imageUrls) {
var fileName = downloadImage(imgUrl, saleId, lotId); var fileName = downloadImage(imgUrl, saleId, lotId);
@@ -87,7 +90,7 @@ class ImageProcessingService {
db.insertImage(lotId, imgUrl, fileName, labels); db.insertImage(lotId, imgUrl, fileName, labels);
if (!labels.isEmpty()) { if (!labels.isEmpty()) {
Console.println(" Detected: " + String.join(", ", labels)); log.info(" Detected: {}", String.join(", ", labels));
} }
} catch (SQLException e) { } catch (SQLException e) {
System.err.println(" Failed to save image to database: " + e.getMessage()); System.err.println(" Failed to save image to database: " + e.getMessage());
@@ -101,18 +104,18 @@ class ImageProcessingService {
* Useful for processing images after the external scraper has populated lot data. * Useful for processing images after the external scraper has populated lot data.
*/ */
void processPendingImages() { void processPendingImages() {
Console.println("Processing pending images..."); log.info("Processing pending images...");
try { try {
var lots = db.getAllLots(); var lots = db.getAllLots();
Console.println("Found " + lots.size() + " lots to check for images"); log.info("Found {} lots to check for images", lots.size());
for (var lot : lots) { for (var lot : lots) {
// Check if images already processed for this lot // Check if images already processed for this lot
var existingImages = db.getImagesForLot(lot.lotId()); var existingImages = db.getImagesForLot(lot.lotId());
if (existingImages.isEmpty()) { if (existingImages.isEmpty()) {
Console.println(" Lot " + lot.lotId() + " has no images yet - needs external scraper data"); log.info(" Lot {} has no images yet - needs external scraper data", lot.lotId());
} }
} }

17
src/main/java/auctiora/Lot.java
View File

@@ -1,5 +1,6 @@
package auctiora; package auctiora;
import lombok.With;
import java.time.Duration; import java.time.Duration;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@@ -8,6 +9,7 @@ import java.time.LocalDateTime;
* Data typically populated by the external scraper process. * Data typically populated by the external scraper process.
* This project enriches the data with image analysis and monitoring. * This project enriches the data with image analysis and monitoring.
*/ */
@With
record Lot( record Lot(
int saleId, int saleId,
int lotId, int lotId,
@@ -23,8 +25,21 @@ record Lot(
LocalDateTime closingTime, LocalDateTime closingTime,
boolean closingNotified boolean closingNotified
) { ) {
long minutesUntilClose() {
public long minutesUntilClose() {
if (closingTime == null) return Long.MAX_VALUE; if (closingTime == null) return Long.MAX_VALUE;
return Duration.between(LocalDateTime.now(), closingTime).toMinutes(); return Duration.between(LocalDateTime.now(), closingTime).toMinutes();
} }
public Lot withCurrentBid(double newBid) {
return new Lot(saleId, lotId, title, description,
manufacturer, type, year, category,
newBid, currency, url, closingTime, closingNotified);
}
public Lot withClosingNotified(boolean flag) {
return new Lot(saleId, lotId, title, description,
manufacturer, type, year, category,
currentBid, currency, url, closingTime, flag);
}
} }

84
src/main/java/auctiora/Main.java
View File

@@ -1,5 +1,6 @@
package auctiora; package auctiora;
import lombok.extern.slf4j.Slf4j;
import org.opencv.core.Core; import org.opencv.core.Core;
/** /**
@@ -19,10 +20,11 @@ import org.opencv.core.Core;
* - Bid monitoring * - Bid monitoring
* - Notifications * - Notifications
*/ */
@Slf4j
public class Main { public class Main {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
Console.println("=== Troostwijk Auction Monitor ===\n"); log.info("=== Troostwijk Auction Monitor ===\n");
// Parse command line arguments // Parse command line arguments
String mode = args.length > 0 ? args[0] : "workflow"; String mode = args.length > 0 ? args[0] : "workflow";
@@ -39,9 +41,9 @@ public class Main {
// Load native OpenCV library (only if models exist) // Load native OpenCV library (only if models exist)
try { try {
System.loadLibrary(Core.NATIVE_LIBRARY_NAME); System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
Console.println("✓ OpenCV loaded"); log.info("✓ OpenCV loaded");
} catch (UnsatisfiedLinkError e) { } catch (UnsatisfiedLinkError e) {
Console.println("⚠️ OpenCV not available - image detection disabled"); log.info("⚠️ OpenCV not available - image detection disabled");
} }
switch (mode.toLowerCase()) { switch (mode.toLowerCase()) {
@@ -75,7 +77,7 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses) String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception { throws Exception {
Console.println("🚀 Starting in WORKFLOW MODE (Orchestrated Scheduling)\n"); log.info("🚀 Starting in WORKFLOW MODE (Orchestrated Scheduling)\n");
WorkflowOrchestrator orchestrator = new WorkflowOrchestrator( WorkflowOrchestrator orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
@@ -87,16 +89,16 @@ public class Main {
// Start all scheduled workflows // Start all scheduled workflows
orchestrator.startScheduledWorkflows(); orchestrator.startScheduledWorkflows();
Console.println("✓ All workflows are running"); log.info("✓ All workflows are running");
Console.println(" - Scraper import: every 30 min"); log.info(" - Scraper import: every 30 min");
Console.println(" - Image processing: every 1 hour"); log.info(" - Image processing: every 1 hour");
Console.println(" - Bid monitoring: every 15 min"); log.info(" - Bid monitoring: every 15 min");
Console.println(" - Closing alerts: every 5 min"); log.info(" - Closing alerts: every 5 min");
Console.println("\nPress Ctrl+C to stop.\n"); log.info("\nPress Ctrl+C to stop.\n");
// Add shutdown hook // Add shutdown hook
Runtime.getRuntime().addShutdownHook(new Thread(() -> { Runtime.getRuntime().addShutdownHook(new Thread(() -> {
Console.println("\n🛑 Shutdown signal received..."); log.info("\n🛑 Shutdown signal received...");
orchestrator.shutdown(); orchestrator.shutdown();
})); }));
@@ -117,7 +119,7 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses) String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception { throws Exception {
Console.println("🔄 Starting in ONCE MODE (Single Execution)\n"); log.info("🔄 Starting in ONCE MODE (Single Execution)\n");
WorkflowOrchestrator orchestrator = new WorkflowOrchestrator( WorkflowOrchestrator orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
@@ -125,7 +127,7 @@ public class Main {
orchestrator.runCompleteWorkflowOnce(); orchestrator.runCompleteWorkflowOnce();
Console.println("✓ Workflow execution completed. Exiting.\n"); log.info("✓ Workflow execution completed. Exiting.\n");
} }
/** /**
@@ -136,29 +138,29 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses) String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception { throws Exception {
Console.println("⚙️ Starting in LEGACY MODE\n"); log.info("⚙️ Starting in LEGACY MODE\n");
var monitor = new TroostwijkMonitor(dbPath, notifConfig, var monitor = new TroostwijkMonitor(dbPath, notifConfig,
yoloCfg, yoloWeights, yoloClasses); yoloCfg, yoloWeights, yoloClasses);
Console.println("\n📊 Current Database State:"); log.info("\n📊 Current Database State:");
monitor.printDatabaseStats(); monitor.printDatabaseStats();
Console.println("\n[1/2] Processing images..."); log.info("\n[1/2] Processing images...");
monitor.processPendingImages(); monitor.processPendingImages();
Console.println("\n[2/2] Starting bid monitoring..."); log.info("\n[2/2] Starting bid monitoring...");
monitor.scheduleMonitoring(); monitor.scheduleMonitoring();
Console.println("\n✓ Monitor is running. Press Ctrl+C to stop.\n"); log.info("\n✓ Monitor is running. Press Ctrl+C to stop.\n");
Console.println("NOTE: This process expects auction/lot data from the external scraper."); log.info("NOTE: This process expects auction/lot data from the external scraper.");
Console.println(" Make sure ARCHITECTURE-TROOSTWIJK-SCRAPER is running and populating the database.\n"); log.info(" Make sure ARCHITECTURE-TROOSTWIJK-SCRAPER is running and populating the database.\n");
try { try {
Thread.sleep(Long.MAX_VALUE); Thread.sleep(Long.MAX_VALUE);
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
Console.println("Monitor interrupted, exiting."); log.info("Monitor interrupted, exiting.");
} }
} }
@@ -169,7 +171,7 @@ public class Main {
String yoloCfg, String yoloWeights, String yoloClasses) String yoloCfg, String yoloWeights, String yoloClasses)
throws Exception { throws Exception {
Console.println("📊 Checking Status...\n"); log.info("📊 Checking Status...\n");
WorkflowOrchestrator orchestrator = new WorkflowOrchestrator( WorkflowOrchestrator orchestrator = new WorkflowOrchestrator(
dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses dbPath, notifConfig, yoloCfg, yoloWeights, yoloClasses
@@ -182,21 +184,21 @@ public class Main {
* Show usage information * Show usage information
*/ */
private static void showUsage() { private static void showUsage() {
Console.println("Usage: java -jar troostwijk-monitor.jar [mode]\n"); log.info("Usage: java -jar troostwijk-monitor.jar [mode]\n");
Console.println("Modes:"); log.info("Modes:");
Console.println(" workflow - Run orchestrated scheduled workflows (default)"); log.info(" workflow - Run orchestrated scheduled workflows (default)");
Console.println(" once - Run complete workflow once and exit (for cron)"); log.info(" once - Run complete workflow once and exit (for cron)");
Console.println(" legacy - Run original monitoring approach"); log.info(" legacy - Run original monitoring approach");
Console.println(" status - Show current status and exit"); log.info(" status - Show current status and exit");
Console.println("\nEnvironment Variables:"); log.info("\nEnvironment Variables:");
Console.println(" DATABASE_FILE - Path to SQLite database"); log.info(" DATABASE_FILE - Path to SQLite database");
Console.println(" (default: C:\\mnt\\okcomputer\\output\\cache.db)"); log.info(" (default: C:\\mnt\\okcomputer\\output\\cache.db)");
Console.println(" NOTIFICATION_CONFIG - 'desktop' or 'smtp:user:pass:email'"); log.info(" NOTIFICATION_CONFIG - 'desktop' or 'smtp:user:pass:email'");
Console.println(" (default: desktop)"); log.info(" (default: desktop)");
Console.println("\nExamples:"); log.info("\nExamples:");
Console.println(" java -jar troostwijk-monitor.jar workflow"); log.info(" java -jar troostwijk-monitor.jar workflow");
Console.println(" java -jar troostwijk-monitor.jar once"); log.info(" java -jar troostwijk-monitor.jar once");
Console.println(" java -jar troostwijk-monitor.jar status"); log.info(" java -jar troostwijk-monitor.jar status");
IO.println(); IO.println();
} }
@@ -206,18 +208,18 @@ public class Main {
*/ */
public static void main2(String[] args) { public static void main2(String[] args) {
if (args.length > 0) { if (args.length > 0) {
Console.println("Command mode - exiting to allow shell commands"); log.info("Command mode - exiting to allow shell commands");
return; return;
} }
Console.println("Troostwijk Monitor container is running and healthy."); log.info("Troostwijk Monitor container is running and healthy.");
Console.println("Use 'docker exec' or 'dokku run' to execute commands."); log.info("Use 'docker exec' or 'dokku run' to execute commands.");
try { try {
Thread.sleep(Long.MAX_VALUE); Thread.sleep(Long.MAX_VALUE);
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
Console.println("Container interrupted, exiting."); log.info("Container interrupted, exiting.");
} }
} }
} }

174
src/main/java/auctiora/NotificationService.java
View File

@@ -1,122 +1,48 @@
package auctiora; package auctiora;
import javax.mail.Authenticator; import javax.mail.*;
import javax.mail.Message.RecipientType; import javax.mail.internet.*;
import javax.mail.PasswordAuthentication; import lombok.extern.slf4j.Slf4j;
import javax.mail.Session; import java.awt.*;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import java.awt.SystemTray;
import java.awt.Toolkit;
import java.awt.TrayIcon;
import java.awt.TrayIcon.MessageType;
import java.util.Date; import java.util.Date;
import java.util.Properties; import java.util.Properties;
/**
* Service for sending notifications via desktop notifications and/or email.
* Supports free notification methods:
* 1. Desktop notifications (Windows/Linux/macOS system tray)
* 2. Email via Gmail SMTP (free, requires app password)
*
* Configuration:
* - For email: Set notificationEmail to your Gmail address
* - Enable 2FA in Gmail and create an App Password
* - Use format "smtp:username:appPassword:toEmail" for credentials
* - Or use "desktop" for desktop-only notifications
*/
class NotificationService {
private final boolean useDesktop; @Slf4j
private final boolean useEmail; public class NotificationService {
private final String smtpUsername;
private final String smtpPassword;
private final String toEmail;
/** private final Config config;
* Creates a notification service.
*
* @param config "desktop" for desktop only, or "smtp:username:password:toEmail" for email
* @param unusedParam Kept for compatibility (can pass empty string)
*/
NotificationService(String config, String unusedParam) {
if ("desktop".equalsIgnoreCase(config)) { public NotificationService(String cfg) {
this.useDesktop = true; this.config = Config.parse(cfg);
this.useEmail = false;
this.smtpUsername = null;
this.smtpPassword = null;
this.toEmail = null;
} else if (config.startsWith("smtp:")) {
var parts = config.split(":", 4);
if (parts.length != 4) {
throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'");
}
this.useDesktop = true; // Always include desktop
this.useEmail = true;
this.smtpUsername = parts[1];
this.smtpPassword = parts[2];
this.toEmail = parts[3];
} else {
throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'");
}
} }
/** public void sendNotification(String message, String title, int priority) {
* Sends notification via configured channels. if (config.useDesktop()) sendDesktop(title, message, priority);
* if (config.useEmail()) sendEmail(title, message, priority);
* @param message The message body
* @param title Message title
* @param priority Priority level (0=normal, 1=high)
*/
void sendNotification(String message, String title, int priority) {
if (useDesktop) {
sendDesktopNotification(title, message, priority);
}
if (useEmail) {
sendEmailNotification(title, message, priority);
}
} }
/** private void sendDesktop(String title, String msg, int prio) {
* Sends a desktop notification using system tray.
* Works on Windows, macOS, and Linux with desktop environments.
*/
private void sendDesktopNotification(String title, String message, int priority) {
try { try {
if (SystemTray.isSupported()) { if (!SystemTray.isSupported()) {
log.info("Desktop notifications not supported — " + title + " / " + msg);
return;
}
var tray = SystemTray.getSystemTray(); var tray = SystemTray.getSystemTray();
var image = Toolkit.getDefaultToolkit() var image = Toolkit.getDefaultToolkit().createImage(new byte[0]);
.createImage(new byte[0]); // Empty image var trayIcon = new TrayIcon(image, "NotificationService");
var trayIcon = new TrayIcon(image, "Troostwijk Scraper");
trayIcon.setImageAutoSize(true); trayIcon.setImageAutoSize(true);
var type = prio > 0 ? TrayIcon.MessageType.WARNING : TrayIcon.MessageType.INFO;
var messageType = priority > 0
? MessageType.WARNING
: MessageType.INFO;
tray.add(trayIcon); tray.add(trayIcon);
trayIcon.displayMessage(title, message, messageType); trayIcon.displayMessage(title, msg, type);
// Remove icon after 2 seconds to avoid clutter
Thread.sleep(2000); Thread.sleep(2000);
tray.remove(trayIcon); tray.remove(trayIcon);
log.info("Desktop notification sent: " + title);
Console.println("Desktop notification sent: " + title);
} else {
Console.println("Desktop notifications not supported, logging: " + title + " - " + message);
}
} catch (Exception e) { } catch (Exception e) {
System.err.println("Desktop notification failed: " + e.getMessage()); System.err.println("Desktop notification failed: " + e);
} }
} }
/** private void sendEmail(String title, String msg, int prio) {
* Sends email notification via Gmail SMTP (free).
* Uses Gmail's SMTP server with app password authentication.
*/
private void sendEmailNotification(String title, String message, int priority) {
try { try {
var props = new Properties(); var props = new Properties();
props.put("mail.smtp.auth", "true"); props.put("mail.smtp.auth", "true");
@@ -125,32 +51,48 @@ class NotificationService {
props.put("mail.smtp.port", "587"); props.put("mail.smtp.port", "587");
props.put("mail.smtp.ssl.trust", "smtp.gmail.com"); props.put("mail.smtp.ssl.trust", "smtp.gmail.com");
var session = Session.getInstance(props, var session = Session.getInstance(props, new Authenticator() {
new Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() { protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(smtpUsername, smtpPassword); return new PasswordAuthentication(config.smtpUsername(), config.smtpPassword());
} }
}); });
var msg = new MimeMessage(session); var m = new MimeMessage(session);
msg.setFrom(new InternetAddress(smtpUsername)); m.setFrom(new InternetAddress(config.smtpUsername()));
msg.setRecipients(RecipientType.TO, m.setRecipients(Message.RecipientType.TO, InternetAddress.parse(config.toEmail()));
InternetAddress.parse(toEmail)); m.setSubject("[Troostwijk] " + title);
msg.setSubject("[Troostwijk] " + title); m.setText(msg);
msg.setText(message); m.setSentDate(new Date());
msg.setSentDate(new Date()); if (prio > 0) {
m.setHeader("X-Priority", "1");
if (priority > 0) { m.setHeader("Importance", "High");
msg.setHeader("X-Priority", "1"); }
msg.setHeader("Importance", "High"); Transport.send(m);
log.info("Email notification sent: " + title);
} catch (Exception e) {
log.info("Email notification failed: " + e);
}
} }
Transport.send(msg); private record Config(
Console.println("Email notification sent: " + title); boolean useDesktop,
boolean useEmail,
String smtpUsername,
String smtpPassword,
String toEmail
) {
} catch (Exception e) { static Config parse(String cfg) {
System.err.println("Email notification failed: " + e.getMessage()); if ("desktop".equalsIgnoreCase(cfg)) {
return new Config(true, false, null, null, null);
} else if (cfg.startsWith("smtp:")) {
var parts = cfg.split(":", 4);
if (parts.length != 4)
throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'");
return new Config(true, true, parts[1], parts[2], parts[3]);
}
throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'");
} }
} }
} }

20
src/main/java/auctiora/ObjectDetectionService.java
View File

@@ -1,13 +1,16 @@
package auctiora; package auctiora;
import lombok.extern.slf4j.Slf4j;
import org.opencv.core.Mat; import org.opencv.core.Mat;
import org.opencv.core.Scalar; import org.opencv.core.Scalar;
import org.opencv.core.Size; import org.opencv.core.Size;
import org.opencv.dnn.Dnn; import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net; import org.opencv.dnn.Net;
import org.opencv.imgcodecs.Imgcodecs; import org.opencv.imgcodecs.Imgcodecs;
import java.io.Console;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@@ -24,7 +27,8 @@ import static org.opencv.dnn.Dnn.DNN_TARGET_CPU;
* If model files are not found, the service operates in disabled mode * If model files are not found, the service operates in disabled mode
* and returns empty lists. * and returns empty lists.
*/ */
class ObjectDetectionService { @Slf4j
public class ObjectDetectionService {
private final Net net; private final Net net;
private final List<String> classNames; private final List<String> classNames;
@@ -37,12 +41,12 @@ class ObjectDetectionService {
var classNamesFile = Paths.get(classNamesPath); var classNamesFile = Paths.get(classNamesPath);
if (!Files.exists(cfgFile) || !Files.exists(weightsFile) || !Files.exists(classNamesFile)) { if (!Files.exists(cfgFile) || !Files.exists(weightsFile) || !Files.exists(classNamesFile)) {
Console.println("⚠️ Object detection disabled: YOLO model files not found"); log.info("⚠️ Object detection disabled: YOLO model files not found");
Console.println(" Expected files:"); log.info(" Expected files:");
Console.println(" - " + cfgPath); log.info(" - " + cfgPath);
Console.println(" - " + weightsPath); log.info(" - " + weightsPath);
Console.println(" - " + classNamesPath); log.info(" - " + classNamesPath);
Console.println(" Scraper will continue without image analysis."); log.info(" Scraper will continue without image analysis.");
this.enabled = false; this.enabled = false;
this.net = null; this.net = null;
this.classNames = new ArrayList<>(); this.classNames = new ArrayList<>();
@@ -57,7 +61,7 @@ class ObjectDetectionService {
// Load class names (one per line) // Load class names (one per line)
this.classNames = Files.readAllLines(classNamesFile); this.classNames = Files.readAllLines(classNamesFile);
this.enabled = true; this.enabled = true;
Console.println("✓ Object detection enabled with YOLO"); log.info("✓ Object detection enabled with YOLO");
} catch (Exception e) { } catch (Exception e) {
System.err.println("⚠️ Object detection disabled: " + e.getMessage()); System.err.println("⚠️ Object detection disabled: " + e.getMessage());
throw new IOException("Failed to initialize object detection", e); throw new IOException("Failed to initialize object detection", e);

271
src/main/java/auctiora/RateLimitedHttpClient.java
View File

@@ -2,9 +2,7 @@ package auctiora;
import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.ApplicationScoped;
import org.eclipse.microprofile.config.inject.ConfigProperty; import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.http.HttpClient; import java.net.http.HttpClient;
import java.net.http.HttpRequest; import java.net.http.HttpRequest;
@@ -12,259 +10,66 @@ import java.net.http.HttpResponse;
import java.time.Duration; import java.time.Duration;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore; import io.github.bucket4j.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/**
* Rate-limited HTTP client that enforces per-host request limits.
*
* Features:
* - Per-host rate limiting (configurable max requests per second)
* - Request counting and monitoring
* - Thread-safe using semaphores
* - Automatic host extraction from URLs
*
* This prevents overloading external services like Troostwijk and getting blocked.
*/
@ApplicationScoped @ApplicationScoped
public class RateLimitedHttpClient { public class RateLimitedHttpClient {
private static final Logger LOG = Logger.getLogger(RateLimitedHttpClient.class); private final HttpClient client = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(30))
private final HttpClient httpClient; .build();
private final Map<String, RateLimiter> rateLimiters;
private final Map<String, RequestStats> requestStats;
@ConfigProperty(name = "auction.http.rate-limit.default-max-rps", defaultValue = "2") @ConfigProperty(name = "auction.http.rate-limit.default-max-rps", defaultValue = "2")
int defaultMaxRequestsPerSecond; int defaultRps;
@ConfigProperty(name = "auction.http.rate-limit.troostwijk-max-rps", defaultValue = "1") @ConfigProperty(name = "auction.http.rate-limit.troostwijk-max-rps", defaultValue = "1")
int troostwijkMaxRequestsPerSecond; int troostwijkRps;
@ConfigProperty(name = "auction.http.timeout-seconds", defaultValue = "30") @ConfigProperty(name = "auction.http.timeout-seconds", defaultValue = "30")
int timeoutSeconds; int timeoutSeconds;
public RateLimitedHttpClient() { private final Map<String, Bucket> buckets = new ConcurrentHashMap<>();
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(30))
.build();
this.rateLimiters = new ConcurrentHashMap<>();
this.requestStats = new ConcurrentHashMap<>();
}
/** private Bucket bucketForHost(String host) {
* Sends a GET request with automatic rate limiting based on host. return buckets.computeIfAbsent(host, h -> {
*/ int rps = host.contains("troostwijk") ? troostwijkRps : defaultRps;
public HttpResponse<String> sendGet(String url) throws IOException, InterruptedException { var limit = Bandwidth.simple(rps, Duration.ofSeconds(1));
HttpRequest request = HttpRequest.newBuilder() return Bucket4j.builder().addLimit(limit).build();
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofString());
}
/**
* Sends a request for binary data (like images) with rate limiting.
*/
public HttpResponse<byte[]> sendGetBytes(String url) throws IOException, InterruptedException {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofByteArray());
}
/**
* Sends any HTTP request with automatic rate limiting.
*/
public <T> HttpResponse<T> send(HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler)
throws IOException, InterruptedException {
String host = extractHost(request.uri());
RateLimiter limiter = getRateLimiter(host);
RequestStats stats = getRequestStats(host);
// Enforce rate limit (blocks if necessary)
limiter.acquire();
// Track request
stats.incrementTotal();
long startTime = System.currentTimeMillis();
try {
HttpResponse<T> response = httpClient.send(request, bodyHandler);
long duration = System.currentTimeMillis() - startTime;
stats.recordSuccess(duration);
LOG.debugf("HTTP %d %s %s (%dms)",
response.statusCode(), request.method(), host, duration);
// Track rate limit violations (429 = Too Many Requests)
if (response.statusCode() == 429) {
stats.incrementRateLimited();
LOG.warnf("⚠️ Rate limited by %s (HTTP 429)", host);
}
return response;
} catch (IOException | InterruptedException e) {
stats.incrementFailed();
LOG.warnf("❌ HTTP request failed for %s: %s", host, e.getMessage());
throw e;
}
}
/**
* Gets or creates a rate limiter for a specific host.
*/
private RateLimiter getRateLimiter(String host) {
return rateLimiters.computeIfAbsent(host, h -> {
int maxRps = getMaxRequestsPerSecond(h);
LOG.infof("Initializing rate limiter for %s: %d req/s", h, maxRps);
return new RateLimiter(maxRps);
}); });
} }
/** public HttpResponse<String> sendGet(String url) throws Exception {
* Gets or creates request stats for a specific host. var req = HttpRequest.newBuilder()
*/ .uri(URI.create(url))
private RequestStats getRequestStats(String host) { .timeout(Duration.ofSeconds(timeoutSeconds))
return requestStats.computeIfAbsent(host, h -> new RequestStats(h)); .GET()
.build();
return send(req, HttpResponse.BodyHandlers.ofString());
} }
/** public HttpResponse<byte[]> sendGetBytes(String url) throws Exception {
* Determines max requests per second for a given host. var req = HttpRequest.newBuilder()
*/ .uri(URI.create(url))
private int getMaxRequestsPerSecond(String host) { .timeout(Duration.ofSeconds(timeoutSeconds))
if (host.contains("troostwijk")) { .GET()
return troostwijkMaxRequestsPerSecond; .build();
} return send(req, HttpResponse.BodyHandlers.ofByteArray());
return defaultMaxRequestsPerSecond;
} }
/** public <T> HttpResponse<T> send(HttpRequest req,
* Extracts host from URI (e.g., "api.troostwijkauctions.com"). HttpResponse.BodyHandler<T> handler) throws Exception {
*/ String host = req.uri().getHost();
private String extractHost(URI uri) { var bucket = bucketForHost(host);
return uri.getHost() != null ? uri.getHost() : uri.toString(); bucket.asBlocking().consume(1);
}
/** var start = System.currentTimeMillis();
* Gets statistics for all hosts. var resp = client.send(req, handler);
*/ var duration = System.currentTimeMillis() - start;
public Map<String, RequestStats> getAllStats() {
return Map.copyOf(requestStats);
}
/** // (Optional) Logging
* Gets statistics for a specific host. System.out.printf("HTTP %d %s %s in %d ms%n",
*/ resp.statusCode(), req.method(), host, duration);
public RequestStats getStats(String host) {
return requestStats.get(host);
}
/** return resp;
* Rate limiter implementation using token bucket algorithm.
* Allows burst traffic up to maxRequestsPerSecond, then enforces steady rate.
*/
private static class RateLimiter {
private final Semaphore semaphore;
private final int maxRequestsPerSecond;
private final long intervalNanos;
RateLimiter(int maxRequestsPerSecond) {
this.maxRequestsPerSecond = maxRequestsPerSecond;
this.intervalNanos = TimeUnit.SECONDS.toNanos(1) / maxRequestsPerSecond;
this.semaphore = new Semaphore(maxRequestsPerSecond);
// Refill tokens periodically
startRefillThread();
}
void acquire() throws InterruptedException {
semaphore.acquire();
// Enforce minimum delay between requests
long delayMillis = intervalNanos / 1_000_000;
if (delayMillis > 0) {
Thread.sleep(delayMillis);
}
}
private void startRefillThread() {
Thread refillThread = new Thread(() -> {
while (!Thread.currentThread().isInterrupted()) {
try {
Thread.sleep(1000); // Refill every second
int toRelease = maxRequestsPerSecond - semaphore.availablePermits();
if (toRelease > 0) {
semaphore.release(toRelease);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}, "RateLimiter-Refill");
refillThread.setDaemon(true);
refillThread.start();
}
}
/**
* Statistics tracker for HTTP requests per host.
*/
public static class RequestStats {
private final String host;
private final AtomicLong totalRequests = new AtomicLong(0);
private final AtomicLong successfulRequests = new AtomicLong(0);
private final AtomicLong failedRequests = new AtomicLong(0);
private final AtomicLong rateLimitedRequests = new AtomicLong(0);
private final AtomicLong totalDurationMs = new AtomicLong(0);
RequestStats(String host) {
this.host = host;
}
void incrementTotal() {
totalRequests.incrementAndGet();
}
void recordSuccess(long durationMs) {
successfulRequests.incrementAndGet();
totalDurationMs.addAndGet(durationMs);
}
void incrementFailed() {
failedRequests.incrementAndGet();
}
void incrementRateLimited() {
rateLimitedRequests.incrementAndGet();
}
// Getters
public String getHost() { return host; }
public long getTotalRequests() { return totalRequests.get(); }
public long getSuccessfulRequests() { return successfulRequests.get(); }
public long getFailedRequests() { return failedRequests.get(); }
public long getRateLimitedRequests() { return rateLimitedRequests.get(); }
public long getAverageDurationMs() {
long successful = successfulRequests.get();
return successful > 0 ? totalDurationMs.get() / successful : 0;
}
@Override
public String toString() {
return String.format("%s: %d total, %d success, %d failed, %d rate-limited, avg %dms",
host, getTotalRequests(), getSuccessfulRequests(),
getFailedRequests(), getRateLimitedRequests(), getAverageDurationMs());
}
} }
} }

270
src/main/java/auctiora/RateLimitedHttpClient2.java Normal file
View File

@@ -0,0 +1,270 @@
package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/**
* Rate-limited HTTP client that enforces per-host request limits.
*
* Features:
* - Per-host rate limiting (configurable max requests per second)
* - Request counting and monitoring
* - Thread-safe using semaphores
* - Automatic host extraction from URLs
*
* This prevents overloading external services like Troostwijk and getting blocked.
*/
@ApplicationScoped
public class RateLimitedHttpClient2 {
private static final Logger LOG = Logger.getLogger(RateLimitedHttpClient2.class);
private final HttpClient httpClient;
private final Map<String, RateLimiter> rateLimiters;
private final Map<String, RequestStats> requestStats;
@ConfigProperty(name = "auction.http.rate-limit.default-max-rps", defaultValue = "2")
int defaultMaxRequestsPerSecond;
@ConfigProperty(name = "auction.http.rate-limit.troostwijk-max-rps", defaultValue = "1")
int troostwijkMaxRequestsPerSecond;
@ConfigProperty(name = "auction.http.timeout-seconds", defaultValue = "30")
int timeoutSeconds;
public RateLimitedHttpClient2() {
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(30))
.build();
this.rateLimiters = new ConcurrentHashMap<>();
this.requestStats = new ConcurrentHashMap<>();
}
/**
* Sends a GET request with automatic rate limiting based on host.
*/
public HttpResponse<String> sendGet(String url) throws IOException, InterruptedException {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofString());
}
/**
* Sends a request for binary data (like images) with rate limiting.
*/
public HttpResponse<byte[]> sendGetBytes(String url) throws IOException, InterruptedException {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofByteArray());
}
/**
* Sends any HTTP request with automatic rate limiting.
*/
public <T> HttpResponse<T> send(HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler)
throws IOException, InterruptedException {
String host = extractHost(request.uri());
RateLimiter limiter = getRateLimiter(host);
RequestStats stats = getRequestStats(host);
// Enforce rate limit (blocks if necessary)
limiter.acquire();
// Track request
stats.incrementTotal();
long startTime = System.currentTimeMillis();
try {
HttpResponse<T> response = httpClient.send(request, bodyHandler);
long duration = System.currentTimeMillis() - startTime;
stats.recordSuccess(duration);
LOG.debugf("HTTP %d %s %s (%dms)",
response.statusCode(), request.method(), host, duration);
// Track rate limit violations (429 = Too Many Requests)
if (response.statusCode() == 429) {
stats.incrementRateLimited();
LOG.warnf("⚠️ Rate limited by %s (HTTP 429)", host);
}
return response;
} catch (IOException | InterruptedException e) {
stats.incrementFailed();
LOG.warnf("❌ HTTP request failed for %s: %s", host, e.getMessage());
throw e;
}
}
/**
* Gets or creates a rate limiter for a specific host.
*/
private RateLimiter getRateLimiter(String host) {
return rateLimiters.computeIfAbsent(host, h -> {
int maxRps = getMaxRequestsPerSecond(h);
LOG.infof("Initializing rate limiter for %s: %d req/s", h, maxRps);
return new RateLimiter(maxRps);
});
}
/**
* Gets or creates request stats for a specific host.
*/
private RequestStats getRequestStats(String host) {
return requestStats.computeIfAbsent(host, h -> new RequestStats(h));
}
/**
* Determines max requests per second for a given host.
*/
private int getMaxRequestsPerSecond(String host) {
if (host.contains("troostwijk")) {
return troostwijkMaxRequestsPerSecond;
}
return defaultMaxRequestsPerSecond;
}
/**
* Extracts host from URI (e.g., "api.troostwijkauctions.com").
*/
private String extractHost(URI uri) {
return uri.getHost() != null ? uri.getHost() : uri.toString();
}
/**
* Gets statistics for all hosts.
*/
public Map<String, RequestStats> getAllStats() {
return Map.copyOf(requestStats);
}
/**
* Gets statistics for a specific host.
*/
public RequestStats getStats(String host) {
return requestStats.get(host);
}
/**
* Rate limiter implementation using token bucket algorithm.
* Allows burst traffic up to maxRequestsPerSecond, then enforces steady rate.
*/
private static class RateLimiter {
private final Semaphore semaphore;
private final int maxRequestsPerSecond;
private final long intervalNanos;
RateLimiter(int maxRequestsPerSecond) {
this.maxRequestsPerSecond = maxRequestsPerSecond;
this.intervalNanos = TimeUnit.SECONDS.toNanos(1) / maxRequestsPerSecond;
this.semaphore = new Semaphore(maxRequestsPerSecond);
// Refill tokens periodically
startRefillThread();
}
void acquire() throws InterruptedException {
semaphore.acquire();
// Enforce minimum delay between requests
long delayMillis = intervalNanos / 1_000_000;
if (delayMillis > 0) {
Thread.sleep(delayMillis);
}
}
private void startRefillThread() {
Thread refillThread = new Thread(() -> {
while (!Thread.currentThread().isInterrupted()) {
try {
Thread.sleep(1000); // Refill every second
int toRelease = maxRequestsPerSecond - semaphore.availablePermits();
if (toRelease > 0) {
semaphore.release(toRelease);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}, "RateLimiter-Refill");
refillThread.setDaemon(true);
refillThread.start();
}
}
/**
* Statistics tracker for HTTP requests per host.
*/
public static class RequestStats {
private final String host;
private final AtomicLong totalRequests = new AtomicLong(0);
private final AtomicLong successfulRequests = new AtomicLong(0);
private final AtomicLong failedRequests = new AtomicLong(0);
private final AtomicLong rateLimitedRequests = new AtomicLong(0);
private final AtomicLong totalDurationMs = new AtomicLong(0);
RequestStats(String host) {
this.host = host;
}
void incrementTotal() {
totalRequests.incrementAndGet();
}
void recordSuccess(long durationMs) {
successfulRequests.incrementAndGet();
totalDurationMs.addAndGet(durationMs);
}
void incrementFailed() {
failedRequests.incrementAndGet();
}
void incrementRateLimited() {
rateLimitedRequests.incrementAndGet();
}
// Getters
public String getHost() { return host; }
public long getTotalRequests() { return totalRequests.get(); }
public long getSuccessfulRequests() { return successfulRequests.get(); }
public long getFailedRequests() { return failedRequests.get(); }
public long getRateLimitedRequests() { return rateLimitedRequests.get(); }
public long getAverageDurationMs() {
long successful = successfulRequests.get();
return successful > 0 ? totalDurationMs.get() / successful : 0;
}
@Override
public String toString() {
return String.format("%s: %d total, %d success, %d failed, %d rate-limited, avg %dms",
host, getTotalRequests(), getSuccessfulRequests(),
getFailedRequests(), getRateLimitedRequests(), getAverageDurationMs());
}
}
}

208
src/main/java/auctiora/ScraperDataAdapter.java
View File

@@ -1,23 +1,15 @@
package auctiora; package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException; import java.time.format.DateTimeParseException;
/** @Slf4j
* Adapter to convert data from the Python scraper's schema to the Monitor's schema. public class ScraperDataAdapter {
*
* SCRAPER SCHEMA DIFFERENCES:
* - auction_id: TEXT ("A7-39813") vs INTEGER (39813)
* - lot_id: TEXT ("A1-28505-5") vs INTEGER (285055)
* - current_bid: TEXT ("€123.45") vs REAL (123.45)
* - Field names: lots_count vs lot_count, auction_id vs sale_id, etc.
*
* This adapter handles the translation between the two schemas.
*/
class ScraperDataAdapter {
private static final DateTimeFormatter[] TIMESTAMP_FORMATS = { private static final DateTimeFormatter[] TIMESTAMP_FORMATS = {
DateTimeFormatter.ISO_LOCAL_DATE_TIME, DateTimeFormatter.ISO_LOCAL_DATE_TIME,
@@ -25,16 +17,6 @@ class ScraperDataAdapter {
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
}; };
/**
* Converts scraper's auction format to monitor's AuctionInfo record.
*
* Scraper format:
* - auction_id: "A7-39813" (TEXT)
* - location: "Cluj-Napoca, RO" (combined)
* - lots_count: INTEGER
* - first_lot_closing_time: TEXT
* - scraped_at: TEXT
*/
static AuctionInfo fromScraperAuction(ResultSet rs) throws SQLException { static AuctionInfo fromScraperAuction(ResultSet rs) throws SQLException {
// Parse "A7-39813" → auctionId=39813, type="A7" // Parse "A7-39813" → auctionId=39813, type="A7"
String auctionIdStr = rs.getString("auction_id"); String auctionIdStr = rs.getString("auction_id");
@@ -64,183 +46,91 @@ class ScraperDataAdapter {
); );
} }
/** public static Lot fromScraperLot(ResultSet rs) throws SQLException {
* Converts scraper's lot format to monitor's Lot record. var lotId = extractNumericId(rs.getString("lot_id"));
* var saleId = extractNumericId(rs.getString("auction_id"));
* Scraper format:
* - lot_id: "A1-28505-5" (TEXT)
* - auction_id: "A7-39813" (TEXT)
* - current_bid: "€123.45" or "No bids" (TEXT)
* - bid_count: INTEGER
* - closing_time: TEXT
*/
static Lot fromScraperLot(ResultSet rs) throws SQLException {
// Parse "A1-28505-5" → lotId=285055
String lotIdStr = rs.getString("lot_id");
int lotId = extractNumericId(lotIdStr);
// Parse "A7-39813" → saleId=39813 var bidStr = getStringOrNull(rs, "current_bid");
String auctionIdStr = rs.getString("auction_id"); var bid = parseBidAmount(bidStr);
int saleId = extractNumericId(auctionIdStr); var currency = parseBidCurrency(bidStr);
// Parse "€123.45" → currentBid=123.45, currency="EUR" var closing = parseTimestamp(getStringOrNull(rs, "closing_time"));
String currentBidStr = getStringOrNull(rs, "current_bid");
double currentBid = parseBidAmount(currentBidStr);
String currency = parseBidCurrency(currentBidStr);
// Parse timestamp
LocalDateTime closingTime = parseTimestamp(getStringOrNull(rs, "closing_time"));
return new Lot( return new Lot(
saleId, saleId,
lotId, lotId,
rs.getString("title"), rs.getString("title"),
getStringOrDefault(rs, "description", ""), getStringOrDefault(rs, "description", ""),
"", // manufacturer - not in scraper schema "", "", 0,
"", // type - not in scraper schema
0, // year - not in scraper schema
getStringOrDefault(rs, "category", ""), getStringOrDefault(rs, "category", ""),
currentBid, bid,
currency, currency,
rs.getString("url"), rs.getString("url"),
closingTime, closing,
false // closing_notified - not yet notified false
); );
} }
/** public static int extractNumericId(String id) {
* Extracts numeric ID from scraper's text format. if (id == null || id.isBlank()) return 0;
* Examples: var digits = id.replaceAll("\\D+", "");
* - "A7-39813" → 39813
* - "A1-28505-5" → 285055 (concatenates all digits)
*/
static int extractNumericId(String id) {
if (id == null || id.isEmpty()) {
return 0;
}
String digits = id.replaceAll("[^0-9]", "");
return digits.isEmpty() ? 0 : Integer.parseInt(digits); return digits.isEmpty() ? 0 : Integer.parseInt(digits);
} }
/**
* Extracts type prefix from scraper's auction/lot ID.
* Examples:
* - "A7-39813" → "A7"
* - "A1-28505-5" → "A1"
*/
private static String extractTypePrefix(String id) { private static String extractTypePrefix(String id) {
if (id == null || id.isEmpty()) { if (id == null) return "";
return ""; var idx = id.indexOf('-');
} return idx > 0 ? id.substring(0, idx) : "";
int dashIndex = id.indexOf('-');
return dashIndex > 0 ? id.substring(0, dashIndex) : "";
} }
/**
* Parses location string into [city, country] array.
* Examples:
* - "Cluj-Napoca, RO" → ["Cluj-Napoca", "RO"]
* - "Amsterdam" → ["Amsterdam", ""]
*/
private static String[] parseLocation(String location) { private static String[] parseLocation(String location) {
if (location == null || location.isEmpty()) { if (location == null || location.isBlank()) return new String[]{ "", "" };
return new String[]{"", ""}; var parts = location.split(",\\s*");
var city = parts[0].trim();
var country = parts.length > 1 ? parts[parts.length - 1].trim() : "";
return new String[]{ city, country };
} }
String[] parts = location.split(",\\s*");
String city = parts.length > 0 ? parts[0].trim() : "";
String country = parts.length > 1 ? parts[parts.length - 1].trim() : "";
return new String[]{city, country};
}
/**
* Parses bid amount from scraper's text format.
* Examples:
* - "€123.45" → 123.45
* - "$50.00" → 50.0
* - "No bids" → 0.0
* - "123.45" → 123.45
*/
private static double parseBidAmount(String bid) { private static double parseBidAmount(String bid) {
if (bid == null || bid.isEmpty() || bid.toLowerCase().contains("no")) { if (bid == null || bid.isBlank() || bid.toLowerCase().contains("no")) return 0.0;
return 0.0; var cleaned = bid.replaceAll("[^0-9.]", "");
}
try { try {
// Remove all non-numeric characters except decimal point return cleaned.isEmpty() ? 0.0 : Double.parseDouble(cleaned);
String cleanBid = bid.replaceAll("[^0-9.]", "");
return cleanBid.isEmpty() ? 0.0 : Double.parseDouble(cleanBid);
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
return 0.0; return 0.0;
} }
} }
/**
* Extracts currency from bid string.
* Examples:
* - "€123.45" → "EUR"
* - "$50.00" → "USD"
* - "123.45" → "EUR" (default)
*/
private static String parseBidCurrency(String bid) { private static String parseBidCurrency(String bid) {
if (bid == null || bid.isEmpty()) { if (bid == null) return "EUR";
return "EUR"; return bid.contains("€") ? "EUR"
: bid.contains("$") ? "USD"
: bid.contains("£") ? "GBP"
: "EUR";
} }
if (bid.contains("€")) return "EUR"; private static LocalDateTime parseTimestamp(String ts) {
if (bid.contains("$")) return "USD"; if (ts == null || ts.isBlank()) return null;
if (bid.contains("£")) return "GBP"; for (var fmt : TIMESTAMP_FORMATS) {
try {
return "EUR"; // Default return LocalDateTime.parse(ts, fmt);
} catch (DateTimeParseException ignored) { }
} }
log.info("Unable to parse timestamp: {}", ts);
/**
* Parses timestamp from various formats used by the scraper.
* Tries multiple formats in order.
*/
private static LocalDateTime parseTimestamp(String timestamp) {
if (timestamp == null || timestamp.isEmpty()) {
return null; return null;
} }
for (DateTimeFormatter formatter : TIMESTAMP_FORMATS) { private static String getStringOrNull(ResultSet rs, String col) throws SQLException {
try { return rs.getString(col);
return LocalDateTime.parse(timestamp, formatter);
} catch (DateTimeParseException e) {
// Try next format
}
} }
// Couldn't parse - return null private static String getStringOrDefault(ResultSet rs, String col, String def) throws SQLException {
Console.println("⚠️ Could not parse timestamp: " + timestamp); var v = rs.getString(col);
return null; return v != null ? v : def;
} }
// Helper methods for safe ResultSet access private static int getIntOrDefault(ResultSet rs, String col, int def) throws SQLException {
var v = rs.getInt(col);
private static String getStringOrNull(ResultSet rs, String column) throws SQLException { return rs.wasNull() ? def : v;
try {
return rs.getString(column);
} catch (SQLException e) {
return null;
}
}
private static String getStringOrDefault(ResultSet rs, String column, String defaultValue) throws SQLException {
try {
String value = rs.getString(column);
return value != null ? value : defaultValue;
} catch (SQLException e) {
return defaultValue;
}
}
private static int getIntOrDefault(ResultSet rs, String column, int defaultValue) throws SQLException {
try {
return rs.getInt(column);
} catch (SQLException e) {
return defaultValue;
}
} }
} }

84
src/main/java/auctiora/StatusResource.java Normal file
View File

@@ -0,0 +1,84 @@
package auctiora;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Map;
@Slf4j
@Path("/api")
public class StatusResource {
private static final DateTimeFormatter FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z")
.withZone(ZoneId.systemDefault());
@ConfigProperty(name = "application.version", defaultValue = "1.0-SNAPSHOT")
String appVersion;
@ConfigProperty(name = "application.groupId")
String groupId;
@ConfigProperty(name = "application.artifactId")
String artifactId;
@ConfigProperty(name = "application.version")
String version;
// Java 16+ Record for structured response
public record StatusResponse(
String groupId,
String artifactId,
String version,
String status,
String timestamp,
String mvnVersion,
String javaVersion,
String os,
String openCvVersion
) { }
@GET
@Path("/status")
@Produces(MediaType.APPLICATION_JSON)
public StatusResponse getStatus() {
log.info("Status endpoint called");
return new StatusResponse(groupId, artifactId, version,
"running",
FORMATTER.format(Instant.now()),
appVersion,
System.getProperty("java.version"),
System.getProperty("os.name") + " " + System.getProperty("os.arch"),
getOpenCvVersion()
);
}
@GET
@Path("/hello")
@Produces(MediaType.APPLICATION_JSON)
public Map<String, String> sayHello() {
log.info("hello endpoint called");
return Map.of(
"message", "Hello from Scrape-UI!",
"timestamp", FORMATTER.format(Instant.now()),
"openCvVersion", getOpenCvVersion()
);
}
private String getOpenCvVersion() {
try {
// Load OpenCV if not already loaded (safe to call multiple times)
nu.pattern.OpenCV.loadLocally();
return org.opencv.core.Core.VERSION;
} catch (Exception e) {
return "4.9.0 (default)";
}
}
}

180
src/main/java/auctiora/TroostwijkMonitor.java
View File

@@ -1,173 +1,135 @@
package auctiora; package auctiora;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException; import java.io.IOException;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
/** @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
* Monitoring service for Troostwijk auction lots. @Slf4j
* This class focuses on:
* - Monitoring bid changes on lots (populated by external scraper)
* - Sending notifications for important events
* - Coordinating image processing
*
* Does NOT handle scraping - that's done by the external ARCHITECTURE-TROOSTWIJK-SCRAPER process.
*/
public class TroostwijkMonitor { public class TroostwijkMonitor {
private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list"; private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list";
private final RateLimitedHttpClient httpClient; RateLimitedHttpClient2 httpClient;
private final ObjectMapper objectMapper; ObjectMapper objectMapper;
public final DatabaseService db; @Getter DatabaseService db;
private final NotificationService notifier; NotificationService notifier;
private final ObjectDetectionService detector; ObjectDetectionService detector;
private final ImageProcessingService imageProcessor; ImageProcessingService imageProcessor;
/** ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
* Constructor for the monitoring service. var t = new Thread(r, "troostwijk-monitor-thread");
* t.setDaemon(true);
* @param databasePath Path to SQLite database file (shared with external scraper) return t;
* @param notificationConfig "desktop" or "smtp:user:pass:email" });
* @param yoloCfgPath YOLO config file path
* @param yoloWeightsPath YOLO weights file path public TroostwijkMonitor(String databasePath,
* @param classNamesPath Class names file path String notificationConfig,
*/ String yoloCfgPath,
public TroostwijkMonitor(String databasePath, String notificationConfig, String yoloWeightsPath,
String yoloCfgPath, String yoloWeightsPath, String classNamesPath) String classNamesPath)
throws SQLException, IOException { throws SQLException, IOException {
this.httpClient = new RateLimitedHttpClient();
this.objectMapper = new ObjectMapper();
this.db = new DatabaseService(databasePath);
this.notifier = new NotificationService(notificationConfig, "");
this.detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
this.imageProcessor = new ImageProcessingService(db, detector, httpClient);
// Initialize database schema httpClient = new RateLimitedHttpClient2();
objectMapper = new ObjectMapper();
db = new DatabaseService(databasePath);
notifier = new NotificationService(notificationConfig);
detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
imageProcessor = new ImageProcessingService(db, detector, httpClient);
db.ensureSchema(); db.ensureSchema();
} }
/**
* Schedules periodic monitoring of all lots.
* Runs every hour to refresh bids and detect changes.
* Increases frequency for lots closing soon.
*/
public void scheduleMonitoring() { public void scheduleMonitoring() {
var scheduler = Executors.newScheduledThreadPool(1); scheduler.scheduleAtFixedRate(this::monitorAllLots, 0, 1, TimeUnit.HOURS);
scheduler.scheduleAtFixedRate(() -> { log.info("✓ Monitoring service started");
}
private void monitorAllLots() {
try { try {
var activeLots = db.getActiveLots(); var activeLots = db.getActiveLots();
Console.println("Monitoring " + activeLots.size() + " active lots..."); log.info("Monitoring {} active lots …", activeLots.size());
for (var lot : activeLots) { for (var lot : activeLots) {
// Refresh lot bidding information checkAndUpdateLot(lot);
}
} catch (SQLException e) {
log.error("Error during scheduled monitoring", e);
}
}
private void checkAndUpdateLot(Lot lot) {
refreshLotBid(lot); refreshLotBid(lot);
// Check closing time
var minutesLeft = lot.minutesUntilClose(); var minutesLeft = lot.minutesUntilClose();
if (minutesLeft < 30) { if (minutesLeft < 30) {
// Send warning when within 5 minutes
if (minutesLeft <= 5 && !lot.closingNotified()) { if (minutesLeft <= 5 && !lot.closingNotified()) {
notifier.sendNotification( notifier.sendNotification(
"Kavel " + lot.lotId() + " sluit binnen " + minutesLeft + " min.", "Kavel " + lot.lotId() + " sluit binnen " + minutesLeft + " min.",
"Lot nearing closure", 1); "Lot nearing closure", 1);
try {
// Update notification flag db.updateLotNotificationFlags(lot.withClosingNotified(true));
var updated = new Lot( } catch (SQLException e) {
lot.saleId(), lot.lotId(), lot.title(), lot.description(), throw new RuntimeException(e);
lot.manufacturer(), lot.type(), lot.year(), lot.category(), }
lot.currentBid(), lot.currency(), lot.url(),
lot.closingTime(), true
);
db.updateLotNotificationFlags(updated);
} }
// Schedule additional quick check
scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES); scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
} }
} }
} catch (SQLException e) {
System.err.println("Error during scheduled monitoring: " + e.getMessage());
}
}, 0, 1, TimeUnit.HOURS);
Console.println("✓ Monitoring service started");
}
/**
* Refreshes the bid for a single lot and sends notification if changed.
*
* @param lot the lot to refresh
*/
private void refreshLotBid(Lot lot) { private void refreshLotBid(Lot lot) {
try { try {
var url = LOT_API + "?batchSize=1&listType=7&offset=0&sortOption=0&saleID=" + lot.saleId() var url = LOT_API +
+ "&parentID=0&relationID=0&buildversion=201807311&lotID=" + lot.lotId(); "?batchSize=1&listType=7&offset=0&sortOption=0" +
"&saleID=" + lot.saleId() +
"&parentID=0&relationID=0&buildversion=201807311" +
"&lotID=" + lot.lotId();
var response = httpClient.sendGet(url); var resp = httpClient.sendGet(url);
if (resp.statusCode() != 200) return;
if (response.statusCode() != 200) return; var root = objectMapper.readTree(resp.body());
var root = objectMapper.readTree(response.body());
var results = root.path("results"); var results = root.path("results");
if (results.isArray() && results.size() > 0) {
if (results.isArray() && !results.isEmpty()) { var newBid = results.get(0).path("cb").asDouble();
var node = results.get(0);
var newBid = node.path("cb").asDouble();
if (Double.compare(newBid, lot.currentBid()) > 0) { if (Double.compare(newBid, lot.currentBid()) > 0) {
var previous = lot.currentBid(); var previous = lot.currentBid();
var updatedLot = lot.withCurrentBid(newBid);
// Create updated lot with new bid
var updatedLot = new Lot(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
newBid, lot.currency(), lot.url(),
lot.closingTime(), lot.closingNotified()
);
db.updateLotCurrentBid(updatedLot); db.updateLotCurrentBid(updatedLot);
var msg = String.format(
var msg = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)", "Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previous); lot.lotId(), newBid, previous);
notifier.sendNotification(msg, "Kavel bieding update", 0); notifier.sendNotification(msg, "Kavel bieding update", 0);
} }
} }
} catch (IOException | InterruptedException | SQLException e) { } catch (IOException | InterruptedException | SQLException e) {
System.err.println("Failed to refresh bid for lot " + lot.lotId() + ": " + e.getMessage()); log.warn("Failed to refresh bid for lot {}", lot.lotId(), e);
if (e instanceof InterruptedException) { if (e instanceof InterruptedException) Thread.currentThread().interrupt();
Thread.currentThread().interrupt();
}
} }
} }
/**
* Prints statistics about the data in the database.
*/
public void printDatabaseStats() { public void printDatabaseStats() {
try { try {
var allLots = db.getAllLots(); var allLots = db.getAllLots();
var imageCount = db.getImageCount(); var imageCount = db.getImageCount();
log.info("📊 Database Summary: total lots = {}, total images = {}",
Console.println("📊 Database Summary:"); allLots.size(), imageCount);
Console.println(" Total lots in database: " + allLots.size());
Console.println(" Total images processed: " + imageCount);
if (!allLots.isEmpty()) { if (!allLots.isEmpty()) {
var totalBids = allLots.stream().mapToDouble(Lot::currentBid).sum(); var sum = allLots.stream().mapToDouble(Lot::currentBid).sum();
Console.println(" Total current bids: €" + String.format("%.2f", totalBids)); log.info("Total current bids: €{:.2f}", sum);
} }
} catch (SQLException e) { } catch (SQLException e) {
System.err.println(" ⚠️ Could not retrieve database stats: " + e.getMessage()); log.warn("Could not retrieve database stats", e);
} }
} }
/**
* Process pending images for lots in the database.
* This should be called after the external scraper has populated lot data.
*/
public void processPendingImages() { public void processPendingImages() {
imageProcessor.processPendingImages(); imageProcessor.processPendingImages();
} }

115
src/main/java/auctiora/WorkflowOrchestrator.java
View File

@@ -1,5 +1,7 @@
package auctiora; package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.io.Console;
import java.io.IOException; import java.io.IOException;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.List; import java.util.List;
@@ -14,6 +16,7 @@ import java.util.concurrent.TimeUnit;
* This class coordinates all services and provides scheduled execution, * This class coordinates all services and provides scheduled execution,
* event-driven triggers, and manual workflow execution. * event-driven triggers, and manual workflow execution.
*/ */
@Slf4j
public class WorkflowOrchestrator { public class WorkflowOrchestrator {
private final TroostwijkMonitor monitor; private final TroostwijkMonitor monitor;
@@ -32,15 +35,15 @@ public class WorkflowOrchestrator {
String yoloCfg, String yoloWeights, String yoloClasses) String yoloCfg, String yoloWeights, String yoloClasses)
throws SQLException, IOException { throws SQLException, IOException {
Console.println("🔧 Initializing Workflow Orchestrator..."); log.info("🔧 Initializing Workflow Orchestrator...");
// Initialize core services // Initialize core services
this.db = new DatabaseService(databasePath); this.db = new DatabaseService(databasePath);
this.db.ensureSchema(); this.db.ensureSchema();
this.notifier = new NotificationService(notificationConfig, ""); this.notifier = new NotificationService(notificationConfig);
this.detector = new ObjectDetectionService(yoloCfg, yoloWeights, yoloClasses); this.detector = new ObjectDetectionService(yoloCfg, yoloWeights, yoloClasses);
RateLimitedHttpClient httpClient = new RateLimitedHttpClient(); RateLimitedHttpClient2 httpClient = new RateLimitedHttpClient2();
this.imageProcessor = new ImageProcessingService(db, detector, httpClient); this.imageProcessor = new ImageProcessingService(db, detector, httpClient);
this.monitor = new TroostwijkMonitor(databasePath, notificationConfig, this.monitor = new TroostwijkMonitor(databasePath, notificationConfig,
@@ -48,7 +51,7 @@ public class WorkflowOrchestrator {
this.scheduler = Executors.newScheduledThreadPool(3); this.scheduler = Executors.newScheduledThreadPool(3);
Console.println("✓ Workflow Orchestrator initialized"); log.info("✓ Workflow Orchestrator initialized");
} }
/** /**
@@ -57,11 +60,11 @@ public class WorkflowOrchestrator {
*/ */
public void startScheduledWorkflows() { public void startScheduledWorkflows() {
if (isRunning) { if (isRunning) {
Console.println("⚠️ Workflows already running"); log.info("⚠️ Workflows already running");
return; return;
} }
Console.println("\n🚀 Starting Scheduled Workflows...\n"); log.info("\n🚀 Starting Scheduled Workflows...\n");
// Workflow 1: Import scraper data (every 30 minutes) // Workflow 1: Import scraper data (every 30 minutes)
scheduleScraperDataImport(); scheduleScraperDataImport();
@@ -76,7 +79,7 @@ public class WorkflowOrchestrator {
scheduleClosingAlerts(); scheduleClosingAlerts();
isRunning = true; isRunning = true;
Console.println("✓ All scheduled workflows started\n"); log.info("✓ All scheduled workflows started\n");
} }
/** /**
@@ -87,23 +90,23 @@ public class WorkflowOrchestrator {
private void scheduleScraperDataImport() { private void scheduleScraperDataImport() {
scheduler.scheduleAtFixedRate(() -> { scheduler.scheduleAtFixedRate(() -> {
try { try {
Console.println("📥 [WORKFLOW 1] Importing scraper data..."); log.info("📥 [WORKFLOW 1] Importing scraper data...");
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
// Import auctions // Import auctions
var auctions = db.importAuctionsFromScraper(); var auctions = db.importAuctionsFromScraper();
Console.println(" → Imported " + auctions.size() + " auctions"); log.info(" → Imported " + auctions.size() + " auctions");
// Import lots // Import lots
var lots = db.importLotsFromScraper(); var lots = db.importLotsFromScraper();
Console.println(" → Imported " + lots.size() + " lots"); log.info(" → Imported " + lots.size() + " lots");
// Import image URLs // Import image URLs
var images = db.getUnprocessedImagesFromScraper(); var images = db.getUnprocessedImagesFromScraper();
Console.println(" → Found " + images.size() + " unprocessed images"); log.info(" → Found " + images.size() + " unprocessed images");
long duration = System.currentTimeMillis() - start; long duration = System.currentTimeMillis() - start;
Console.println(" ✓ Scraper import completed in " + duration + "ms\n"); log.info(" ✓ Scraper import completed in " + duration + "ms\n");
// Trigger notification if significant data imported // Trigger notification if significant data imported
if (auctions.size() > 0 || lots.size() > 10) { if (auctions.size() > 0 || lots.size() > 10) {
@@ -115,11 +118,11 @@ public class WorkflowOrchestrator {
} }
} catch (Exception e) { } catch (Exception e) {
Console.println(" ❌ Scraper import failed: " + e.getMessage()); log.info(" ❌ Scraper import failed: " + e.getMessage());
} }
}, 0, 30, TimeUnit.MINUTES); }, 0, 30, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Scraper Data Import (every 30 min)"); log.info(" ✓ Scheduled: Scraper Data Import (every 30 min)");
} }
/** /**
@@ -130,18 +133,18 @@ public class WorkflowOrchestrator {
private void scheduleImageProcessing() { private void scheduleImageProcessing() {
scheduler.scheduleAtFixedRate(() -> { scheduler.scheduleAtFixedRate(() -> {
try { try {
Console.println("🖼️ [WORKFLOW 2] Processing pending images..."); log.info("🖼️ [WORKFLOW 2] Processing pending images...");
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
// Get unprocessed images // Get unprocessed images
var unprocessedImages = db.getUnprocessedImagesFromScraper(); var unprocessedImages = db.getUnprocessedImagesFromScraper();
if (unprocessedImages.isEmpty()) { if (unprocessedImages.isEmpty()) {
Console.println(" → No pending images to process\n"); log.info(" → No pending images to process\n");
return; return;
} }
Console.println(" → Processing " + unprocessedImages.size() + " images"); log.info(" → Processing " + unprocessedImages.size() + " images");
int processed = 0; int processed = 0;
int detected = 0; int detected = 0;
@@ -184,20 +187,20 @@ public class WorkflowOrchestrator {
Thread.sleep(500); Thread.sleep(500);
} catch (Exception e) { } catch (Exception e) {
Console.println(" ⚠️ Failed to process image: " + e.getMessage()); log.info(" ⚠️ Failed to process image: " + e.getMessage());
} }
} }
long duration = System.currentTimeMillis() - start; long duration = System.currentTimeMillis() - start;
Console.println(String.format(" ✓ Processed %d images, detected objects in %d (%.1fs)\n", log.info(String.format(" ✓ Processed %d images, detected objects in %d (%.1fs)\n",
processed, detected, duration / 1000.0)); processed, detected, duration / 1000.0));
} catch (Exception e) { } catch (Exception e) {
Console.println(" ❌ Image processing failed: " + e.getMessage()); log.info(" ❌ Image processing failed: " + e.getMessage());
} }
}, 5, 60, TimeUnit.MINUTES); }, 5, 60, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Image Processing (every 1 hour)"); log.info(" ✓ Scheduled: Image Processing (every 1 hour)");
} }
/** /**
@@ -208,11 +211,11 @@ public class WorkflowOrchestrator {
private void scheduleBidMonitoring() { private void scheduleBidMonitoring() {
scheduler.scheduleAtFixedRate(() -> { scheduler.scheduleAtFixedRate(() -> {
try { try {
Console.println("💰 [WORKFLOW 3] Monitoring bids..."); log.info("💰 [WORKFLOW 3] Monitoring bids...");
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
var activeLots = db.getActiveLots(); var activeLots = db.getActiveLots();
Console.println(" → Checking " + activeLots.size() + " active lots"); log.info(" → Checking " + activeLots.size() + " active lots");
int bidChanges = 0; int bidChanges = 0;
@@ -223,14 +226,14 @@ public class WorkflowOrchestrator {
} }
long duration = System.currentTimeMillis() - start; long duration = System.currentTimeMillis() - start;
Console.println(String.format(" ✓ Bid monitoring completed in %dms\n", duration)); log.info(String.format(" ✓ Bid monitoring completed in %dms\n", duration));
} catch (Exception e) { } catch (Exception e) {
Console.println(" ❌ Bid monitoring failed: " + e.getMessage()); log.info(" ❌ Bid monitoring failed: " + e.getMessage());
} }
}, 2, 15, TimeUnit.MINUTES); }, 2, 15, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Bid Monitoring (every 15 min)"); log.info(" ✓ Scheduled: Bid Monitoring (every 15 min)");
} }
/** /**
@@ -241,7 +244,7 @@ public class WorkflowOrchestrator {
private void scheduleClosingAlerts() { private void scheduleClosingAlerts() {
scheduler.scheduleAtFixedRate(() -> { scheduler.scheduleAtFixedRate(() -> {
try { try {
Console.println("⏰ [WORKFLOW 4] Checking closing times..."); log.info("⏰ [WORKFLOW 4] Checking closing times...");
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
var activeLots = db.getActiveLots(); var activeLots = db.getActiveLots();
@@ -273,15 +276,15 @@ public class WorkflowOrchestrator {
} }
long duration = System.currentTimeMillis() - start; long duration = System.currentTimeMillis() - start;
Console.println(String.format(" → Sent %d closing alerts in %dms\n", log.info(String.format(" → Sent %d closing alerts in %dms\n",
alertsSent, duration)); alertsSent, duration));
} catch (Exception e) { } catch (Exception e) {
Console.println(" ❌ Closing alerts failed: " + e.getMessage()); log.info(" ❌ Closing alerts failed: " + e.getMessage());
} }
}, 1, 5, TimeUnit.MINUTES); }, 1, 5, TimeUnit.MINUTES);
Console.println(" ✓ Scheduled: Closing Alerts (every 5 min)"); log.info(" ✓ Scheduled: Closing Alerts (every 5 min)");
} }
/** /**
@@ -289,39 +292,39 @@ public class WorkflowOrchestrator {
* Useful for testing or on-demand execution * Useful for testing or on-demand execution
*/ */
public void runCompleteWorkflowOnce() { public void runCompleteWorkflowOnce() {
Console.println("\n🔄 Running Complete Workflow (Manual Trigger)...\n"); log.info("\n🔄 Running Complete Workflow (Manual Trigger)...\n");
try { try {
// Step 1: Import data // Step 1: Import data
Console.println("[1/4] Importing scraper data..."); log.info("[1/4] Importing scraper data...");
var auctions = db.importAuctionsFromScraper(); var auctions = db.importAuctionsFromScraper();
var lots = db.importLotsFromScraper(); var lots = db.importLotsFromScraper();
Console.println(" ✓ Imported " + auctions.size() + " auctions, " + lots.size() + " lots"); log.info(" ✓ Imported " + auctions.size() + " auctions, " + lots.size() + " lots");
// Step 2: Process images // Step 2: Process images
Console.println("[2/4] Processing pending images..."); log.info("[2/4] Processing pending images...");
monitor.processPendingImages(); monitor.processPendingImages();
Console.println(" ✓ Image processing completed"); log.info(" ✓ Image processing completed");
// Step 3: Check bids // Step 3: Check bids
Console.println("[3/4] Monitoring bids..."); log.info("[3/4] Monitoring bids...");
var activeLots = db.getActiveLots(); var activeLots = db.getActiveLots();
Console.println(" ✓ Monitored " + activeLots.size() + " lots"); log.info(" ✓ Monitored " + activeLots.size() + " lots");
// Step 4: Check closing times // Step 4: Check closing times
Console.println("[4/4] Checking closing times..."); log.info("[4/4] Checking closing times...");
int closingSoon = 0; int closingSoon = 0;
for (var lot : activeLots) { for (var lot : activeLots) {
if (lot.closingTime() != null && lot.minutesUntilClose() < 30) { if (lot.closingTime() != null && lot.minutesUntilClose() < 30) {
closingSoon++; closingSoon++;
} }
} }
Console.println(" ✓ Found " + closingSoon + " lots closing soon"); log.info(" ✓ Found " + closingSoon + " lots closing soon");
Console.println("\n✓ Complete workflow finished successfully\n"); log.info("\n✓ Complete workflow finished successfully\n");
} catch (Exception e) { } catch (Exception e) {
Console.println("\n❌ Workflow failed: " + e.getMessage() + "\n"); log.info("\n❌ Workflow failed: " + e.getMessage() + "\n");
} }
} }
@@ -329,7 +332,7 @@ public class WorkflowOrchestrator {
* Event-driven trigger: New auction discovered * Event-driven trigger: New auction discovered
*/ */
public void onNewAuctionDiscovered(AuctionInfo auction) { public void onNewAuctionDiscovered(AuctionInfo auction) {
Console.println("📣 EVENT: New auction discovered - " + auction.title()); log.info("📣 EVENT: New auction discovered - " + auction.title());
try { try {
db.upsertAuction(auction); db.upsertAuction(auction);
@@ -342,7 +345,7 @@ public class WorkflowOrchestrator {
); );
} catch (Exception e) { } catch (Exception e) {
Console.println(" ❌ Failed to handle new auction: " + e.getMessage()); log.info(" ❌ Failed to handle new auction: " + e.getMessage());
} }
} }
@@ -350,7 +353,7 @@ public class WorkflowOrchestrator {
* Event-driven trigger: Bid change detected * Event-driven trigger: Bid change detected
*/ */
public void onBidChange(Lot lot, double previousBid, double newBid) { public void onBidChange(Lot lot, double previousBid, double newBid) {
Console.println(String.format("📣 EVENT: Bid change on lot %d (€%.2f → €%.2f)", log.info(String.format("📣 EVENT: Bid change on lot %d (€%.2f → €%.2f)",
lot.lotId(), previousBid, newBid)); lot.lotId(), previousBid, newBid));
try { try {
@@ -364,7 +367,7 @@ public class WorkflowOrchestrator {
); );
} catch (Exception e) { } catch (Exception e) {
Console.println(" ❌ Failed to handle bid change: " + e.getMessage()); log.info(" ❌ Failed to handle bid change: " + e.getMessage());
} }
} }
@@ -372,7 +375,7 @@ public class WorkflowOrchestrator {
* Event-driven trigger: Objects detected in image * Event-driven trigger: Objects detected in image
*/ */
public void onObjectsDetected(int lotId, List<String> labels) { public void onObjectsDetected(int lotId, List<String> labels) {
Console.println(String.format("📣 EVENT: Objects detected in lot %d - %s", log.info(String.format("📣 EVENT: Objects detected in lot %d - %s",
lotId, String.join(", ", labels))); lotId, String.join(", ", labels)));
try { try {
@@ -384,7 +387,7 @@ public class WorkflowOrchestrator {
); );
} }
} catch (Exception e) { } catch (Exception e) {
Console.println(" ❌ Failed to send detection notification: " + e.getMessage()); log.info(" ❌ Failed to send detection notification: " + e.getMessage());
} }
} }
@@ -392,17 +395,17 @@ public class WorkflowOrchestrator {
* Prints current workflow status * Prints current workflow status
*/ */
public void printStatus() { public void printStatus() {
Console.println("\n📊 Workflow Status:"); log.info("\n📊 Workflow Status:");
Console.println(" Running: " + (isRunning ? "Yes" : "No")); log.info(" Running: " + (isRunning ? "Yes" : "No"));
try { try {
var auctions = db.getAllAuctions(); var auctions = db.getAllAuctions();
var lots = db.getAllLots(); var lots = db.getAllLots();
int images = db.getImageCount(); int images = db.getImageCount();
Console.println(" Auctions: " + auctions.size()); log.info(" Auctions: " + auctions.size());
Console.println(" Lots: " + lots.size()); log.info(" Lots: " + lots.size());
Console.println(" Images: " + images); log.info(" Images: " + images);
// Count closing soon // Count closing soon
int closingSoon = 0; int closingSoon = 0;
@@ -411,10 +414,10 @@ public class WorkflowOrchestrator {
closingSoon++; closingSoon++;
} }
} }
Console.println(" Closing soon (< 30 min): " + closingSoon); log.info(" Closing soon (< 30 min): " + closingSoon);
} catch (Exception e) { } catch (Exception e) {
Console.println(" ⚠️ Could not retrieve status: " + e.getMessage()); log.info(" ⚠️ Could not retrieve status: " + e.getMessage());
} }
IO.println(); IO.println();
@@ -424,7 +427,7 @@ public class WorkflowOrchestrator {
* Gracefully shuts down all workflows * Gracefully shuts down all workflows
*/ */
public void shutdown() { public void shutdown() {
Console.println("\n🛑 Shutting down workflows..."); log.info("\n🛑 Shutting down workflows...");
isRunning = false; isRunning = false;
scheduler.shutdown(); scheduler.shutdown();
@@ -433,7 +436,7 @@ public class WorkflowOrchestrator {
if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) { if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
scheduler.shutdownNow(); scheduler.shutdownNow();
} }
Console.println("✓ Workflows shut down successfully\n"); log.info("✓ Workflows shut down successfully\n");
} catch (InterruptedException e) { } catch (InterruptedException e) {
scheduler.shutdownNow(); scheduler.shutdownNow();
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();

19
src/main/resources/application.properties
View File

@@ -1,10 +1,21 @@
# Application Configuration # Application Configuration
quarkus.application.name=auctiora # Values will be injected from pom.xml during build
quarkus.application.version=1.0-SNAPSHOT quarkus.application.name=${project.artifactId}
quarkus.application.version=${project.version}
# Custom properties for groupId if needed
application.groupId=${project.groupId}
application.artifactId=${project.artifactId}
application.version=${project.version}
# HTTP Configuration # HTTP Configuration
quarkus.http.port=8081 quarkus.http.port=8081
quarkus.http.host=0.0.0.0 # ========== DEVELOPMENT (quarkus:dev) ==========
%dev.quarkus.http.host=127.0.0.1
# ========== PRODUCTION (Docker/JAR) ==========
%prod.quarkus.http.host=0.0.0.0
# ========== TEST PROFILE ==========
%test.quarkus.http.host=localhost
# Enable CORS for frontend development # Enable CORS for frontend development
quarkus.http.cors=true quarkus.http.cors=true
@@ -26,7 +37,7 @@ quarkus.log.console.level=INFO
# Static resources # Static resources
quarkus.http.enable-compression=true quarkus.http.enable-compression=true
quarkus.rest.path=/api quarkus.rest.path=/
quarkus.http.root-path=/ quarkus.http.root-path=/
# Auction Monitor Configuration # Auction Monitor Configuration

7
src/main/resources/resources/beans.xml Normal file
View File

@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="https://jakarta.ee/xml/ns/jakartaee"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="https://jakarta.ee/xml/ns/jakartaee https://jakarta.ee/xml/ns/jakartaee/beans_3_0.xsd
https://jakarta.ee/xml/ns/jakartaee "
version="3.0" bean-discovery-mode="all">
</beans>

224
src/main/resources/resources/index.html Normal file
View File

@@ -0,0 +1,224 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Scrape-UI 1 - Enterprise</title>
<script src="https://cdn.tailwindcss.com"></script>
<style>
.gradient-bg {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
}
.card-hover {
transition: all 0.3s ease;
}
.card-hover:hover {
transform: translateY(-5px);
box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04);
}
</style>
</head>
<body class="bg-gray-50 min-h-screen">
<!-- Header -->
<header class="gradient-bg text-white py-8">
<div class="container mx-auto px-4">
<h1 class="text-4xl font-bold mb-2">Scrape-UI Enterprise</h1>
<p class="text-xl opacity-90">Powered by Quarkus + Modern Frontend</p>
</div>
</header>
<!-- Main Content -->
<main class="container mx-auto px-4 py-8">
<!-- API Status Card -->
<!-- API & Build Status Card -->
<div class="bg-white rounded-lg shadow-md p-6 mb-8 card-hover">
<h2 class="text-2xl font-bold mb-4 text-gray-800">Build & Runtime Status</h2>
<div id="api-status" class="space-y-4">
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
<!-- Build Information -->
<div class="bg-blue-50 p-4 rounded-lg">
<h3 class="font-semibold text-blue-800 mb-2">📦 Maven Build</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
<span class="text-gray-600">Group:</span>
<span class="font-mono font-medium" id="build-group">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Artifact:</span>
<span class="font-mono font-medium" id="build-artifact">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Version:</span>
<span class="font-mono font-medium px-2 py-1 bg-blue-100 rounded" id="build-version">-</span>
</div>
</div>
</div>
<!-- Runtime Information -->
<div class="bg-green-50 p-4 rounded-lg">
<h3 class="font-semibold text-green-800 mb-2">🚀 Runtime</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
<span class="text-gray-600">Status:</span>
<span class="px-2 py-1 rounded-full text-xs font-semibold bg-green-100 text-green-800" id="runtime-status">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Java:</span>
<span class="font-mono" id="java-version">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Platform:</span>
<span class="font-mono" id="runtime-os">-</span>
</div>
</div>
</div>
</div>
<!-- Timestamp & Additional Info -->
<div class="pt-4 border-t">
<div class="flex justify-between items-center">
<div>
<p class="text-sm text-gray-500">Last Updated</p>
<p class="font-medium" id="last-updated">-</p>
</div>
<button onclick="fetchStatus()" class="bg-gray-100 hover:bg-gray-200 text-gray-700 px-4 py-2 rounded-lg text-sm transition-colors">
🔄 Refresh
</button>
</div>
</div>
</div>
</div>
<!-- API Response Card -->
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h2 class="text-2xl font-bold mb-4 text-gray-800">API Test</h2>
<button id="test-api" class="bg-blue-600 text-white px-6 py-2 rounded-lg hover:bg-blue-700 transition-colors mb-4">
Test Greeting API
</button>
<div id="api-response" class="bg-gray-100 p-4 rounded-lg">
<pre class="text-sm text-gray-700">Click the button to test the API</pre>
</div>
</div>
<!-- Features Grid -->
<div class="grid grid-cols-1 md:grid-cols-3 gap-6 mt-8">
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">⚡ Quarkus Backend</h3>
<p class="text-gray-600">Fast startup, low memory footprint, optimized for containers</p>
</div>
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">🚀 REST API</h3>
<p class="text-gray-600">RESTful endpoints with JSON responses</p>
</div>
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">🎨 Modern UI</h3>
<p class="text-gray-600">Responsive design with Tailwind CSS</p>
</div>
</div>
</main>
<script>
// Fetch API status on load
async function fetchStatus() {
try {
const response = await fetch('/api/status')
if (!response.ok) {
throw new Error(`HTTP ${ response.status }: ${ response.statusText }`)
}
const data = await response.json()
// Update Build Information
document.getElementById('build-group').textContent = data.groupId || 'N/A'
document.getElementById('build-artifact').textContent = data.artifactId || data.name || 'N/A'
document.getElementById('build-version').textContent = data.version || 'N/A'
// Update Runtime Information
document.getElementById('runtime-status').textContent = data.status || 'unknown'
document.getElementById('java-version').textContent = data.javaVersion || System.getProperty?.('java.version') || 'N/A'
document.getElementById('runtime-os').textContent = data.os || 'N/A'
// Update Timestamp
const timestamp = data.timestamp ? new Date(data.timestamp).toLocaleString() : 'N/A'
document.getElementById('last-updated').textContent = timestamp
// Update status badge color based on status
const statusBadge = document.getElementById('runtime-status')
if (data.status?.toLowerCase() === 'running') {
statusBadge.className = 'px-2 py-1 rounded-full text-xs font-semibold bg-green-100 text-green-800'
} else {
statusBadge.className = 'px-2 py-1 rounded-full text-xs font-semibold bg-yellow-100 text-yellow-800'
}
} catch (error) {
console.error('Error fetching status:', error)
document.getElementById('api-status').innerHTML = `
<div class="bg-red-50 border-l-4 border-red-500 p-4">
<div class="flex">
<div class="flex-shrink-0">
<svg class="h-5 w-5 text-red-400" viewBox="0 0 20 20" fill="currentColor">
<path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clip-rule="evenodd" />
</svg>
</div>
<div class="ml-3">
<p class="text-sm text-red-700">Failed to load status: ${ error.message }</p>
<button onclick="fetchStatus()" class="mt-2 text-sm text-red-700 hover:text-red-600 font-medium">
Retry →
</button>
</div>
</div>
</div>
`
}
}
// Fetch API status on load
async function fetchStatus3() {
try {
const response = await fetch('/api/status')
const data = await response.json()
document.getElementById('api-status').innerHTML = `
<p><strong>Application:</strong> ${ data.application }</p>
<p><strong>Status:</strong> <span class="text-green-600 font-semibold">${ data.status }</span></p>
<p><strong>Version:</strong> ${ data.version }</p>
<p><strong>Timestamp:</strong> ${ data.timestamp }</p>
`
} catch (error) {
document.getElementById('api-status').innerHTML = `
<p class="text-red-600">Error loading status: ${ error.message }</p>
`
}
}
// Test greeting API
document.getElementById('test-api').addEventListener('click', async () => {
try {
const response = await fetch('/api/hello')
const data = await response.json()
document.getElementById('api-response').innerHTML = `
<pre class="text-sm text-gray-700">${ JSON.stringify(data, null, 2) }</pre>
`
} catch (error) {
document.getElementById('api-response').innerHTML = `
<pre class="text-sm text-red-600">Error: ${ error.message }</pre>
`
}
})
// Auto-refresh every 30 seconds
let refreshInterval = setInterval(fetchStatus, 30000);
// Stop auto-refresh when page loses focus (optional)
document.addEventListener('visibilitychange', function() {
if (document.hidden) {
clearInterval(refreshInterval);
} else {
refreshInterval = setInterval(fetchStatus, 30000);
fetchStatus(); // Refresh immediately when returning to tab
}
});
// Load status on page load
fetchStatus()
</script>
</body>
</html>

26
src/test/java/auctiora/AuctionParsingTest.java
View File

@@ -1,5 +1,6 @@
package auctiora; package auctiora;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@@ -19,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.*;
* Test auction parsing logic using saved HTML from test.html * Test auction parsing logic using saved HTML from test.html
* Tests the markup data extraction for each auction found * Tests the markup data extraction for each auction found
*/ */
@Slf4j
public class AuctionParsingTest { public class AuctionParsingTest {
private static String testHtml; private static String testHtml;
@@ -27,12 +29,12 @@ public class AuctionParsingTest {
public static void loadTestHtml() throws IOException { public static void loadTestHtml() throws IOException {
// Load the test HTML file // Load the test HTML file
testHtml = Files.readString(Paths.get("src/test/resources/test_auctions.html")); testHtml = Files.readString(Paths.get("src/test/resources/test_auctions.html"));
System.out.println("Loaded test HTML (" + testHtml.length() + " characters)"); log.info("Loaded test HTML ({} characters)", testHtml.length());
} }
@Test @Test
public void testLocationPatternMatching() { public void testLocationPatternMatching() {
System.out.println("\n=== Location Pattern Tests ==="); log.info("\n=== Location Pattern Tests ===");
// Test different location formats // Test different location formats
var testCases = new String[]{ var testCases = new String[]{
@@ -48,16 +50,16 @@ public class AuctionParsingTest {
if (elem != null) { if (elem != null) {
var text = elem.text(); var text = elem.text();
System.out.println("\nTest: " + testHtml); log.info("\nTest: {}", testHtml);
System.out.println("Text: " + text); log.info("Text: {}", text);
// Test regex pattern // Test regex pattern
if (text.matches(".*[A-Z]{2}$")) { if (text.matches(".*[A-Z]{2}$")) {
var countryCode = text.substring(text.length() - 2); var countryCode = text.substring(text.length() - 2);
var cityPart = text.substring(0, text.length() - 2).trim().replaceAll("[,\\s]+$", ""); var cityPart = text.substring(0, text.length() - 2).trim().replaceAll("[,\\s]+$", "");
System.out.println("→ Extracted: " + cityPart + ", " + countryCode); log.info("→ Extracted: {}, {}", cityPart, countryCode);
} else { } else {
System.out.println("→ No match"); log.info("→ No match");
} }
} }
} }
@@ -65,7 +67,7 @@ public class AuctionParsingTest {
@Test @Test
public void testFullTextPatternMatching() { public void testFullTextPatternMatching() {
System.out.println("\n=== Full Text Pattern Tests ==="); log.info("\n=== Full Text Pattern Tests ===");
// Test the complete auction text format // Test the complete auction text format
var testCases = new String[]{ var testCases = new String[]{
@@ -75,7 +77,7 @@ public class AuctionParsingTest {
}; };
for (var testText : testCases) { for (var testText : testCases) {
System.out.println("\nParsing: \"" + testText + "\""); log.info("\nParsing: \"{}\"", testText);
// Simulated extraction // Simulated extraction
var remaining = testText; var remaining = testText;
@@ -84,7 +86,7 @@ public class AuctionParsingTest {
var timePattern = java.util.regex.Pattern.compile("(\\w+)\\s+om\\s+(\\d{1,2}:\\d{2})"); var timePattern = java.util.regex.Pattern.compile("(\\w+)\\s+om\\s+(\\d{1,2}:\\d{2})");
var timeMatcher = timePattern.matcher(remaining); var timeMatcher = timePattern.matcher(remaining);
if (timeMatcher.find()) { if (timeMatcher.find()) {
System.out.println(" Time: " + timeMatcher.group(1) + " om " + timeMatcher.group(2)); log.info(" Time: {} om {}", timeMatcher.group(1), timeMatcher.group(2));
remaining = remaining.substring(timeMatcher.end()).trim(); remaining = remaining.substring(timeMatcher.end()).trim();
} }
@@ -94,7 +96,7 @@ public class AuctionParsingTest {
); );
var locMatcher = locPattern.matcher(remaining); var locMatcher = locPattern.matcher(remaining);
if (locMatcher.find()) { if (locMatcher.find()) {
System.out.println(" Location: " + locMatcher.group(1) + ", " + locMatcher.group(2)); log.info(" Location: {}, {}", locMatcher.group(1), locMatcher.group(2));
remaining = remaining.substring(0, locMatcher.start()).trim(); remaining = remaining.substring(0, locMatcher.start()).trim();
} }
@@ -102,12 +104,12 @@ public class AuctionParsingTest {
var lotPattern = java.util.regex.Pattern.compile("^(\\d+)\\s+"); var lotPattern = java.util.regex.Pattern.compile("^(\\d+)\\s+");
var lotMatcher = lotPattern.matcher(remaining); var lotMatcher = lotPattern.matcher(remaining);
if (lotMatcher.find()) { if (lotMatcher.find()) {
System.out.println(" Lot count: " + lotMatcher.group(1)); log.info(" Lot count: {}", lotMatcher.group(1));
remaining = remaining.substring(lotMatcher.end()).trim(); remaining = remaining.substring(lotMatcher.end()).trim();
} }
// What remains is title // What remains is title
System.out.println(" Title: " + remaining); log.info(" Title: {}", remaining);
} }
} }
} }

4
src/test/java/auctiora/ImageProcessingServiceTest.java
View File

@@ -19,14 +19,14 @@ class ImageProcessingServiceTest {
private DatabaseService mockDb; private DatabaseService mockDb;
private ObjectDetectionService mockDetector; private ObjectDetectionService mockDetector;
private RateLimitedHttpClient mockHttpClient; private RateLimitedHttpClient2 mockHttpClient;
private ImageProcessingService service; private ImageProcessingService service;
@BeforeEach @BeforeEach
void setUp() { void setUp() {
mockDb = mock(DatabaseService.class); mockDb = mock(DatabaseService.class);
mockDetector = mock(ObjectDetectionService.class); mockDetector = mock(ObjectDetectionService.class);
mockHttpClient = mock(RateLimitedHttpClient.class); mockHttpClient = mock(RateLimitedHttpClient2.class);
service = new ImageProcessingService(mockDb, mockDetector, mockHttpClient); service = new ImageProcessingService(mockDb, mockDetector, mockHttpClient);
} }

4
src/test/java/auctiora/IntegrationTest.java
View File

@@ -40,7 +40,7 @@ class IntegrationTest {
db = new DatabaseService(testDbPath); db = new DatabaseService(testDbPath);
db.ensureSchema(); db.ensureSchema();
notifier = new NotificationService("desktop", ""); notifier = new NotificationService("desktop");
detector = new ObjectDetectionService( detector = new ObjectDetectionService(
"non_existent.cfg", "non_existent.cfg",
@@ -48,7 +48,7 @@ class IntegrationTest {
"non_existent.txt" "non_existent.txt"
); );
RateLimitedHttpClient httpClient = new RateLimitedHttpClient(); RateLimitedHttpClient2 httpClient = new RateLimitedHttpClient2();
imageProcessor = new ImageProcessingService(db, detector, httpClient); imageProcessor = new ImageProcessingService(db, detector, httpClient);
monitor = new TroostwijkMonitor( monitor = new TroostwijkMonitor(

51
src/test/java/auctiora/NotificationServiceTest.java
View File

@@ -14,7 +14,7 @@ class NotificationServiceTest {
@Test @Test
@DisplayName("Should initialize with desktop-only configuration") @DisplayName("Should initialize with desktop-only configuration")
void testDesktopOnlyConfiguration() { void testDesktopOnlyConfiguration() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
assertNotNull(service); assertNotNull(service);
} }
@@ -22,8 +22,7 @@ class NotificationServiceTest {
@DisplayName("Should initialize with SMTP configuration") @DisplayName("Should initialize with SMTP configuration")
void testSMTPConfiguration() { void testSMTPConfiguration() {
NotificationService service = new NotificationService( NotificationService service = new NotificationService(
"smtp:test@gmail.com:app_password:recipient@example.com", "smtp:test@gmail.com:app_password:recipient@example.com"
""
); );
assertNotNull(service); assertNotNull(service);
} }
@@ -33,12 +32,12 @@ class NotificationServiceTest {
void testInvalidSMTPConfiguration() { void testInvalidSMTPConfiguration() {
// Missing parts // Missing parts
assertThrows(IllegalArgumentException.class, () -> assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:incomplete", "") new NotificationService("smtp:incomplete")
); );
// Wrong format // Wrong format
assertThrows(IllegalArgumentException.class, () -> assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:only:two:parts", "") new NotificationService("smtp:only:two:parts")
); );
} }
@@ -46,14 +45,14 @@ class NotificationServiceTest {
@DisplayName("Should reject unknown configuration type") @DisplayName("Should reject unknown configuration type")
void testUnknownConfiguration() { void testUnknownConfiguration() {
assertThrows(IllegalArgumentException.class, () -> assertThrows(IllegalArgumentException.class, () ->
new NotificationService("unknown_type", "") new NotificationService("unknown_type")
); );
} }
@Test @Test
@DisplayName("Should send desktop notification without error") @DisplayName("Should send desktop notification without error")
void testDesktopNotification() { void testDesktopNotification() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
// Should not throw exception even if system tray not available // Should not throw exception even if system tray not available
assertDoesNotThrow(() -> assertDoesNotThrow(() ->
@@ -64,7 +63,7 @@ class NotificationServiceTest {
@Test @Test
@DisplayName("Should send high priority notification") @DisplayName("Should send high priority notification")
void testHighPriorityNotification() { void testHighPriorityNotification() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() -> assertDoesNotThrow(() ->
service.sendNotification("Urgent message", "High Priority", 1) service.sendNotification("Urgent message", "High Priority", 1)
@@ -74,7 +73,7 @@ class NotificationServiceTest {
@Test @Test
@DisplayName("Should send normal priority notification") @DisplayName("Should send normal priority notification")
void testNormalPriorityNotification() { void testNormalPriorityNotification() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() -> assertDoesNotThrow(() ->
service.sendNotification("Regular message", "Normal Priority", 0) service.sendNotification("Regular message", "Normal Priority", 0)
@@ -84,7 +83,7 @@ class NotificationServiceTest {
@Test @Test
@DisplayName("Should handle notification when system tray not supported") @DisplayName("Should handle notification when system tray not supported")
void testNoSystemTraySupport() { void testNoSystemTraySupport() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
// Should gracefully handle missing system tray // Should gracefully handle missing system tray
assertDoesNotThrow(() -> assertDoesNotThrow(() ->
@@ -98,8 +97,7 @@ class NotificationServiceTest {
// Note: This won't actually send email without valid credentials // Note: This won't actually send email without valid credentials
// But it should initialize properly // But it should initialize properly
NotificationService service = new NotificationService( NotificationService service = new NotificationService(
"smtp:test@gmail.com:fake_password:test@example.com", "smtp:test@gmail.com:fake_password:test@example.com"
""
); );
// Should not throw during initialization // Should not throw during initialization
@@ -115,8 +113,7 @@ class NotificationServiceTest {
@DisplayName("Should include both desktop and email when SMTP configured") @DisplayName("Should include both desktop and email when SMTP configured")
void testBothNotificationChannels() { void testBothNotificationChannels() {
NotificationService service = new NotificationService( NotificationService service = new NotificationService(
"smtp:user@gmail.com:password:recipient@example.com", "smtp:user@gmail.com:password:recipient@example.com"
""
); );
// Both desktop and email should be attempted // Both desktop and email should be attempted
@@ -128,7 +125,7 @@ class NotificationServiceTest {
@Test @Test
@DisplayName("Should handle empty message gracefully") @DisplayName("Should handle empty message gracefully")
void testEmptyMessage() { void testEmptyMessage() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() -> assertDoesNotThrow(() ->
service.sendNotification("", "", 0) service.sendNotification("", "", 0)
@@ -138,7 +135,7 @@ class NotificationServiceTest {
@Test @Test
@DisplayName("Should handle very long message") @DisplayName("Should handle very long message")
void testLongMessage() { void testLongMessage() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
String longMessage = "A".repeat(1000); String longMessage = "A".repeat(1000);
assertDoesNotThrow(() -> assertDoesNotThrow(() ->
@@ -149,7 +146,7 @@ class NotificationServiceTest {
@Test @Test
@DisplayName("Should handle special characters in message") @DisplayName("Should handle special characters in message")
void testSpecialCharactersInMessage() { void testSpecialCharactersInMessage() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() -> assertDoesNotThrow(() ->
service.sendNotification( service.sendNotification(
@@ -164,9 +161,9 @@ class NotificationServiceTest {
@DisplayName("Should accept case-insensitive desktop config") @DisplayName("Should accept case-insensitive desktop config")
void testCaseInsensitiveDesktopConfig() { void testCaseInsensitiveDesktopConfig() {
assertDoesNotThrow(() -> { assertDoesNotThrow(() -> {
new NotificationService("DESKTOP", ""); new NotificationService("DESKTOP");
new NotificationService("Desktop", ""); new NotificationService("Desktop");
new NotificationService("desktop", ""); new NotificationService("desktop");
}); });
} }
@@ -175,19 +172,19 @@ class NotificationServiceTest {
void testSMTPConfigPartsValidation() { void testSMTPConfigPartsValidation() {
// Too few parts // Too few parts
assertThrows(IllegalArgumentException.class, () -> assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:user:pass", "") new NotificationService("smtp:user:pass")
); );
// Too many parts should work (extras ignored in split) // Too many parts should work (extras ignored in split)
assertDoesNotThrow(() -> assertDoesNotThrow(() ->
new NotificationService("smtp:user:pass:email:extra", "") new NotificationService("smtp:user:pass:email:extra")
); );
} }
@Test @Test
@DisplayName("Should handle multiple rapid notifications") @DisplayName("Should handle multiple rapid notifications")
void testRapidNotifications() { void testRapidNotifications() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() -> { assertDoesNotThrow(() -> {
for (int i = 0; i < 5; i++) { for (int i = 0; i < 5; i++) {
@@ -201,14 +198,14 @@ class NotificationServiceTest {
void testNullConfigParameter() { void testNullConfigParameter() {
// Second parameter can be empty string (kept for compatibility) // Second parameter can be empty string (kept for compatibility)
assertDoesNotThrow(() -> assertDoesNotThrow(() ->
new NotificationService("desktop", null) new NotificationService("desktop")
); );
} }
@Test @Test
@DisplayName("Should send bid change notification format") @DisplayName("Should send bid change notification format")
void testBidChangeNotificationFormat() { void testBidChangeNotificationFormat() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
String message = "Nieuw bod op kavel 12345: €150.00 (was €125.00)"; String message = "Nieuw bod op kavel 12345: €150.00 (was €125.00)";
String title = "Kavel bieding update"; String title = "Kavel bieding update";
@@ -221,7 +218,7 @@ class NotificationServiceTest {
@Test @Test
@DisplayName("Should send closing alert notification format") @DisplayName("Should send closing alert notification format")
void testClosingAlertNotificationFormat() { void testClosingAlertNotificationFormat() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
String message = "Kavel 12345 sluit binnen 5 min."; String message = "Kavel 12345 sluit binnen 5 min.";
String title = "Lot nearing closure"; String title = "Lot nearing closure";
@@ -234,7 +231,7 @@ class NotificationServiceTest {
@Test @Test
@DisplayName("Should send object detection notification format") @DisplayName("Should send object detection notification format")
void testObjectDetectionNotificationFormat() { void testObjectDetectionNotificationFormat() {
NotificationService service = new NotificationService("desktop", ""); NotificationService service = new NotificationService("desktop");
String message = "Lot contains: car, truck, machinery\nEstimated value: €5000"; String message = "Lot contains: car, truck, machinery\nEstimated value: €5000";
String title = "Object Detected"; String title = "Object Detected";

60
src/test/java/auctiora/Parser.java
View File

File diff suppressed because one or more lines are too long

62
src/test/java/auctiora/ParserTest.java Normal file
View File

File diff suppressed because one or more lines are too long

10
src/test/java/auctiora/ScraperDataAdapterTest.java
View File

@@ -55,9 +55,9 @@ class ScraperDataAdapterTest {
assertEquals("Cluj-Napoca", result.city()); assertEquals("Cluj-Napoca", result.city());
assertEquals("RO", result.country()); assertEquals("RO", result.country());
assertEquals("https://example.com/auction/A7-39813", result.url()); assertEquals("https://example.com/auction/A7-39813", result.url());
assertEquals("A7", result.type()); assertEquals("A7", result.typePrefix());
assertEquals(150, result.lotCount()); assertEquals(150, result.lotCount());
assertNotNull(result.closingTime()); assertNotNull(result.firstLotClosingTime());
} }
@Test @Test
@@ -75,7 +75,7 @@ class ScraperDataAdapterTest {
assertEquals("Amsterdam", result.city()); assertEquals("Amsterdam", result.city());
assertEquals("", result.country()); assertEquals("", result.country());
assertNull(result.closingTime()); assertNull(result.firstLotClosingTime());
} }
@Test @Test
@@ -196,7 +196,7 @@ class ScraperDataAdapterTest {
when(rs1.getString("first_lot_closing_time")).thenReturn(null); when(rs1.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo auction1 = ScraperDataAdapter.fromScraperAuction(rs1); AuctionInfo auction1 = ScraperDataAdapter.fromScraperAuction(rs1);
assertEquals("A7", auction1.type()); assertEquals("A7", auction1.typePrefix());
ResultSet rs2 = mock(ResultSet.class); ResultSet rs2 = mock(ResultSet.class);
when(rs2.getString("auction_id")).thenReturn("B1-12345"); when(rs2.getString("auction_id")).thenReturn("B1-12345");
@@ -207,7 +207,7 @@ class ScraperDataAdapterTest {
when(rs2.getString("first_lot_closing_time")).thenReturn(null); when(rs2.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo auction2 = ScraperDataAdapter.fromScraperAuction(rs2); AuctionInfo auction2 = ScraperDataAdapter.fromScraperAuction(rs2);
assertEquals("B1", auction2.type()); assertEquals("B1", auction2.typePrefix());
} }
@Test @Test

48
src/test/java/auctiora/TroostwijkMonitorTest.java
View File

@@ -43,7 +43,7 @@ class TroostwijkMonitorTest {
@DisplayName("Should initialize monitor successfully") @DisplayName("Should initialize monitor successfully")
void testMonitorInitialization() { void testMonitorInitialization() {
assertNotNull(monitor); assertNotNull(monitor);
assertNotNull(monitor.db); assertNotNull(monitor.getDb());
} }
@Test @Test
@@ -61,8 +61,8 @@ class TroostwijkMonitorTest {
@Test @Test
@DisplayName("Should handle empty database gracefully") @DisplayName("Should handle empty database gracefully")
void testEmptyDatabaseHandling() throws SQLException { void testEmptyDatabaseHandling() throws SQLException {
var auctions = monitor.db.getAllAuctions(); var auctions = monitor.getDb().getAllAuctions();
var lots = monitor.db.getAllLots(); var lots = monitor.getDb().getAllLots();
assertNotNull(auctions); assertNotNull(auctions);
assertNotNull(lots); assertNotNull(lots);
@@ -88,9 +88,9 @@ class TroostwijkMonitorTest {
false false
); );
monitor.db.upsertLot(lot); monitor.getDb().upsertLot(lot);
var lots = monitor.db.getAllLots(); var lots = monitor.getDb().getAllLots();
assertTrue(lots.stream().anyMatch(l -> l.lotId() == 22222)); assertTrue(lots.stream().anyMatch(l -> l.lotId() == 22222));
} }
@@ -113,9 +113,9 @@ class TroostwijkMonitorTest {
false false
); );
monitor.db.upsertLot(closingSoon); monitor.getDb().upsertLot(closingSoon);
var lots = monitor.db.getActiveLots(); var lots = monitor.getDb().getActiveLots();
var found = lots.stream() var found = lots.stream()
.filter(l -> l.lotId() == 44444) .filter(l -> l.lotId() == 44444)
.findFirst() .findFirst()
@@ -143,9 +143,9 @@ class TroostwijkMonitorTest {
false false
); );
monitor.db.upsertLot(futureLot); monitor.getDb().upsertLot(futureLot);
var lots = monitor.db.getActiveLots(); var lots = monitor.getDb().getActiveLots();
var found = lots.stream() var found = lots.stream()
.filter(l -> l.lotId() == 66666) .filter(l -> l.lotId() == 66666)
.findFirst() .findFirst()
@@ -173,9 +173,9 @@ class TroostwijkMonitorTest {
false false
); );
monitor.db.upsertLot(noClosing); monitor.getDb().upsertLot(noClosing);
var lots = monitor.db.getActiveLots(); var lots = monitor.getDb().getActiveLots();
var found = lots.stream() var found = lots.stream()
.filter(l -> l.lotId() == 88888) .filter(l -> l.lotId() == 88888)
.findFirst() .findFirst()
@@ -203,7 +203,7 @@ class TroostwijkMonitorTest {
false false
); );
monitor.db.upsertLot(lot); monitor.getDb().upsertLot(lot);
// Update notification flag // Update notification flag
var notified = new Lot( var notified = new Lot(
@@ -221,9 +221,9 @@ class TroostwijkMonitorTest {
true true
); );
monitor.db.updateLotNotificationFlags(notified); monitor.getDb().updateLotNotificationFlags(notified);
var lots = monitor.db.getActiveLots(); var lots = monitor.getDb().getActiveLots();
var found = lots.stream() var found = lots.stream()
.filter(l -> l.lotId() == 11110) .filter(l -> l.lotId() == 11110)
.findFirst() .findFirst()
@@ -251,7 +251,7 @@ class TroostwijkMonitorTest {
false false
); );
monitor.db.upsertLot(lot); monitor.getDb().upsertLot(lot);
// Simulate bid increase // Simulate bid increase
var higherBid = new Lot( var higherBid = new Lot(
@@ -269,9 +269,9 @@ class TroostwijkMonitorTest {
false false
); );
monitor.db.updateLotCurrentBid(higherBid); monitor.getDb().updateLotCurrentBid(higherBid);
var lots = monitor.db.getActiveLots(); var lots = monitor.getDb().getActiveLots();
var found = lots.stream() var found = lots.stream()
.filter(l -> l.lotId() == 13131) .filter(l -> l.lotId() == 13131)
.findFirst() .findFirst()
@@ -287,7 +287,7 @@ class TroostwijkMonitorTest {
Thread t1 = new Thread(() -> { Thread t1 = new Thread(() -> {
try { try {
for (int i = 0; i < 5; i++) { for (int i = 0; i < 5; i++) {
monitor.db.upsertLot(new Lot( monitor.getDb().upsertLot(new Lot(
20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat", 20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
100.0, "EUR", "https://example.com/" + i, null, false 100.0, "EUR", "https://example.com/" + i, null, false
)); ));
@@ -300,7 +300,7 @@ class TroostwijkMonitorTest {
Thread t2 = new Thread(() -> { Thread t2 = new Thread(() -> {
try { try {
for (int i = 5; i < 10; i++) { for (int i = 5; i < 10; i++) {
monitor.db.upsertLot(new Lot( monitor.getDb().upsertLot(new Lot(
20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat", 20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
200.0, "EUR", "https://example.com/" + i, null, false 200.0, "EUR", "https://example.com/" + i, null, false
)); ));
@@ -315,7 +315,7 @@ class TroostwijkMonitorTest {
t1.join(); t1.join();
t2.join(); t2.join();
var lots = monitor.db.getActiveLots(); var lots = monitor.getDb().getActiveLots();
long count = lots.stream() long count = lots.stream()
.filter(l -> l.lotId() >= 30000 && l.lotId() < 30010) .filter(l -> l.lotId() >= 30000 && l.lotId() < 30010)
.count(); .count();
@@ -351,7 +351,7 @@ class TroostwijkMonitorTest {
LocalDateTime.now().plusDays(2) LocalDateTime.now().plusDays(2)
); );
monitor.db.upsertAuction(auction); monitor.getDb().upsertAuction(auction);
// Insert related lot // Insert related lot
var lot = new Lot( var lot = new Lot(
@@ -369,11 +369,11 @@ class TroostwijkMonitorTest {
false false
); );
monitor.db.upsertLot(lot); monitor.getDb().upsertLot(lot);
// Verify // Verify
var auctions = monitor.db.getAllAuctions(); var auctions = monitor.getDb().getAllAuctions();
var lots = monitor.db.getAllLots(); var lots = monitor.getDb().getAllLots();
assertTrue(auctions.stream().anyMatch(a -> a.auctionId() == 40000)); assertTrue(auctions.stream().anyMatch(a -> a.auctionId() == 40000));
assertTrue(lots.stream().anyMatch(l -> l.lotId() == 50000)); assertTrue(lots.stream().anyMatch(l -> l.lotId() == 50000));

20
workflows/maven.yml Normal file
View File

@@ -0,0 +1,20 @@
name: Publish to Gitea Package Registry
on:
push:
tags:
- 'v*'
jobs:
publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up JDK 25
uses: actions/setup-java@v4
with:
java-version: '25'
distribution: 'temurin'
- name: Publish with Maven
run: mvn --batch-mode clean deploy
env:
GITEA_TOKEN: ${{ secrets.EA_PUBLISH_TOKEN }}