Compare commits

...

120 Commits

Author SHA1 Message Date
Tour
49a6c755ee fix-tests-cleanup
Former-commit-id: 3cc0d40fa3
2025-12-08 07:52:54 +01:00
Tour
df919abad5 fix-tests-cleanup
Former-commit-id: be65f4a5e6
2025-12-08 07:19:50 +01:00
Tour
aecf32eb19 fix-tests-cleanup
Former-commit-id: 3358a2693c
2025-12-08 05:37:35 +01:00
Tour
efc6b7ac21 fix-tests-cleanup
Former-commit-id: 62cda5c0cb
2025-12-08 05:37:29 +01:00
Tour
270df601d2 slash
Former-commit-id: 7600cebcbb
2025-12-07 18:06:17 +01:00
Tour
9cb19bbd8b Features
Former-commit-id: 394469923b
2025-12-07 16:25:29 +01:00
Tour
8a74b2bc71 test
Former-commit-id: 2da6049206
2025-12-07 15:51:11 +01:00
Tour
c7040f1ed7 redeploy
Former-commit-id: 3cf2d2ef7a
2025-12-07 15:27:29 +01:00
Tour
22c1e99c4a Enrich ALL lots on startup in background thread
Former-commit-id: afd7b311a9
2025-12-07 14:48:36 +01:00
Tour
f6f37ff9d1 Add startup enrichment trigger for lot intelligence data
- Added StartupEvent observer to QuarkusWorkflowScheduler
- Triggers enrichment of lots closing within 24 hours on startup
- Ensures bid intelligence data is populated immediately after deployment
- Fixes issue where server showed 0 lots with bids

This ensures the GraphQL enrichment service runs at startup to populate
bid_count, starting_bid, followers_count and other intelligence fields.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

Former-commit-id: 7ab21ae840
2025-12-07 14:43:10 +01:00
Tour
86b19db30b Fix database schema: Change auction_id and lot_id from BIGINT to TEXT
The scraper uses TEXT IDs like "A7-40063-2" but DatabaseService was creating
BIGINT columns, causing PRIMARY KEY constraint failures on the server.

Changes:
- auction_id: BIGINT -> TEXT PRIMARY KEY
- lot_id: BIGINT -> TEXT PRIMARY KEY
- sale_id: BIGINT -> TEXT
- Added UNIQUE constraints on URLs
- Added migration script (fix-schema.sql)

This fixes the "UNIQUE constraint failed: auctions.auction_id" errors
and allows bid data to populate correctly on the server.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

Former-commit-id: 12c3a732e4
2025-12-07 13:59:26 +01:00
Tour
4f0d5113f5 Fix GraphQL enrichment to use displayId instead of numeric lotId
- Added displayId (String) field to Lot record for full lot ID (e.g., "A1-34732-49")
- Updated ScraperDataAdapter to extract both numeric ID and displayId from database
- Fixed TroostwijkGraphQLClient to query by displayId using lotDetails() instead of lot()
- Matched Python scraper's query structure with LOT_BIDDING_QUERY pattern
- Updated GraphQL response parsing to handle lotDetails.location and biddingStatistics
- Added upsertLotWithIntelligence() method to DatabaseService for full intelligence updates
- Updated LotEnrichmentService to pass displayId to GraphQL client

This fixes the "No intelligence data returned" error on production server.
GraphQL API requires string displayId parameter, not numeric lot ID.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

Former-commit-id: 4d7da94315
2025-12-07 13:31:40 +01:00
Tour
2e8af3b1d2 Fix PRIMARY KEY constraint handling in auction import
- Handle both auction_id and URL constraint failures
- Add fallback UPDATE by URL when INSERT fails
- Properly catch and log constraint violations without propagating
- Fixes import errors on server with duplicate auctions


Former-commit-id: 80b9841aee
2025-12-07 13:10:39 +01:00
Tour
cadb703a88 goog
Former-commit-id: 00eb3f7aca
2025-12-07 12:59:22 +01:00
Tour
93d47436a8 goog
Former-commit-id: 825058f790
2025-12-07 12:56:53 +01:00
Tour
1ccb4558d7 goog
Former-commit-id: d5d245cfc1
2025-12-07 12:47:36 +01:00
Tour
d18d4f40f4 Features
Former-commit-id: ca19649b6a
2025-12-07 11:41:22 +01:00
Tour
b7e387da62 Features
Former-commit-id: 65bb5cd80a
2025-12-07 11:31:55 +01:00
Tour
fb31915b39 Fix mock tests
Former-commit-id: 43b5fc03fd
2025-12-07 11:08:59 +01:00
Tour
89969b8234 Fix mock tests
Former-commit-id: 11a76e0292
2025-12-07 09:59:08 +01:00
Tour
c63cc2dc3d Fix mock tests
Former-commit-id: a649b629e4
2025-12-07 06:51:18 +01:00
Tour
ca38327834 Fix mock tests
Former-commit-id: 3efa83bc44
2025-12-07 06:32:03 +01:00
Tour
03f94de020 Fix mock tests
Former-commit-id: ef804b3896
2025-12-07 06:28:37 +01:00
Tour
4764f072b5 Fix mock tests
Former-commit-id: f561a73b01
2025-12-07 02:36:00 +01:00
Tour
5d966da2c4 Fix mock tests
Former-commit-id: 432fcbc503
2025-12-06 21:37:08 +01:00
Tour
57524de4ad Fix mock tests
Former-commit-id: b4e0f8c13b
2025-12-06 21:36:55 +01:00
Tour
f750f211db Fix build: Update tests for image download refactor
- Remove RateLimitedHttpClient from ImageProcessingService constructor
- Rewrite ImageProcessingServiceTest to test new object detection workflow
- Fix IntegrationTest constructor call
- Add insertImage() back to DatabaseService for test compatibility
- Tests now focus on object detection rather than image downloading


Former-commit-id: e216a763ac
2025-12-06 21:34:29 +01:00
Tour
9baaca9013 go
Former-commit-id: 6091b7180f
2025-12-06 21:27:19 +01:00
Tour
49cb4f94aa test
Former-commit-id: 288ee6a2a6
2025-12-06 07:08:07 +01:00
Tour
4375021a65 %*
Former-commit-id: a25c0bdf5d
2025-12-06 07:04:53 +01:00
Tour
4ab55cb55c all
Former-commit-id: 4ecb6625c8
2025-12-06 06:23:34 +01:00
Tour
d055b52c43 all
Former-commit-id: 174d0b136e
2025-12-06 06:19:23 +01:00
Tour
f6246c5156 front-end-fix
Former-commit-id: d8f7464944
2025-12-06 06:14:26 +01:00
Tour
175fa3a4fd front-end-fix
Former-commit-id: 9f5003ecc5
2025-12-06 06:03:43 +01:00
Tour
f8cd3f21c7 front-end-fix
Former-commit-id: cda9b648ad
2025-12-06 05:59:22 +01:00
Tour
14389236a3 front-end-fix
Former-commit-id: 1af565ae1b
2025-12-06 05:51:55 +01:00
Tour
39c0161dc2 Fix mock tests
Former-commit-id: 36b03dea7b
2025-12-06 05:41:14 +01:00
Tour
745fbdadd5 front-end-fix
Former-commit-id: 528a217708
2025-12-06 05:39:59 +01:00
Tour
4223628c0e front-end-fix
Former-commit-id: d1a149e40d
2025-12-06 05:29:41 +01:00
Tour
08312daead Fix mock tests
Former-commit-id: 758e60ecb3
2025-12-05 21:10:17 +01:00
Tour
2b5762237d done
Former-commit-id: e06f5747ec
2025-12-05 20:58:32 +01:00
Tour
5e0c8ab531 done
Former-commit-id: 0b1be38681
2025-12-05 20:54:23 +01:00
Tour
7b07db83a3 init
Former-commit-id: e9b4298f58
2025-12-05 09:42:48 +01:00
Tour
ffa7b47d5d dns
Former-commit-id: 887295260f
2025-12-05 09:35:17 +01:00
Tour
30d2407b1e dns
Former-commit-id: a06434642c
2025-12-05 08:59:23 +01:00
Tour
3f984b9024 dns
Former-commit-id: 36a1edfecf
2025-12-05 08:57:29 +01:00
Tour
60e92ad234 all
Former-commit-id: 243573d4b2
2025-12-05 08:49:59 +01:00
Tour
ca1a1812d0 all
Former-commit-id: 41de6c1e8a
2025-12-05 08:35:19 +01:00
Tour
4629e7db39 all
Former-commit-id: 0ab9430f35
2025-12-05 08:27:43 +01:00
Tour
d9c900c7d7 update
Former-commit-id: 20c2129d06
2025-12-05 08:20:18 +01:00
Tour
64cc09ce7a update
Former-commit-id: 5430610b56
2025-12-05 08:14:15 +01:00
Tour
c431c551b8 update
Former-commit-id: c91b2d7f3a
2025-12-05 07:57:58 +01:00
Tour
040b52d182 update
Former-commit-id: 3357267581
2025-12-05 07:42:17 +01:00
Tour
a6e32573e2 update
Former-commit-id: c6263e78b2
2025-12-05 07:40:26 +01:00
Tour
f4578a5fc2 update
Former-commit-id: cf486796ac
2025-12-05 07:36:25 +01:00
Tour
1b80ca3fbc update
Former-commit-id: 79f63ba9a5
2025-12-05 07:35:20 +01:00
Tour
9dbd6f7e25 update
Former-commit-id: fb5fa1b0ff
2025-12-05 07:28:49 +01:00
Tour
96db0c9c0a monitor-page
Former-commit-id: 6325d07909
2025-12-05 07:06:34 +01:00
Tour
01743dce79 Fix mock tests
Former-commit-id: 04df491d64
2025-12-05 06:19:28 +01:00
Tour
b69571afee Fix mock tests
Former-commit-id: f05a8b73ec
2025-12-05 04:48:41 +01:00
Tour
0e9570c6fb Fix mock tests
Former-commit-id: 1292d09427
2025-12-05 03:44:31 +01:00
Tour
f309b77b06 Fix mock tests
Former-commit-id: 5083a68205
2025-12-05 03:44:28 +01:00
Tour
96cc09c387 Fix mock tests
Former-commit-id: 8ecd9fcbda
2025-12-05 03:42:36 +01:00
Tour
2ff7aa6027 Fix mock tests
Former-commit-id: ff8f5f2c1a
2025-12-04 20:14:28 +01:00
Tour
4ceb284ab7 Fix mock tests
Former-commit-id: 2ff6fcca17
2025-12-04 20:07:54 +01:00
Tour
7265a6dbaf Fix mock tests
Former-commit-id: d52bd8f94e
2025-12-04 20:00:33 +01:00
Tour
2bf5a3f51a Fix mock tests
Former-commit-id: ed74bb5e93
2025-12-04 19:38:38 +01:00
Tour
4642b31062 Fix mock tests
Former-commit-id: 9857f053a1
2025-12-04 04:30:44 +01:00
Tour
5b091cd2f0 USe ASM 9.8 with Java 25
Former-commit-id: e71d52be8a
2025-12-04 04:00:03 +01:00
Tour
59b640c696 USe ASM 9.8 with Java 25
Former-commit-id: cad27f1842
2025-12-04 03:31:27 +01:00
Tour
0904014b8c USe ASM 9.8 with Java 25
Former-commit-id: d2000a46bc
2025-12-04 03:21:03 +01:00
Tour
3ff8decf9a start
Former-commit-id: 8e06e20b70
2025-12-03 19:03:03 +01:00
Tour
1aa4d76559 start
Former-commit-id: 4c32043e5f
2025-12-03 17:30:09 +01:00
Tour
51f834fab8 start
Former-commit-id: 8fff75dcf2
2025-12-03 17:17:49 +01:00
Tour
6c1a6313c8 start
Former-commit-id: febd08821a
2025-12-03 15:40:19 +01:00
Tour
f3c450ad85 start
Former-commit-id: d3dc37576d
2025-12-03 15:32:41 +01:00
Tour
be2842f390 start
Former-commit-id: aef7a3aa30
2025-12-03 15:32:34 +01:00
Tour
6756c2f6c6 start
Former-commit-id: 815d6a9a4a
2025-12-03 15:18:21 +01:00
Tour
4bb36e23e9 start
Former-commit-id: 853c3cf53e
2025-12-03 15:09:39 +01:00
13cedfa84c start
Former-commit-id: 7fa3e4a545
2025-11-28 14:43:58 +01:00
0c73b53790 start
Former-commit-id: 836ce3527f
2025-11-28 06:37:04 +01:00
bbaab28fa3 start
Former-commit-id: b174f77f6c
2025-11-28 06:23:30 +01:00
9708a1bba7 start
Former-commit-id: 0f5800441a
2025-11-28 05:54:30 +01:00
92c5bd5d66 start
Former-commit-id: f5ee240283
2025-11-28 05:46:53 +01:00
cef5aae8fb start
Former-commit-id: bde45e0dc9
2025-11-28 05:39:23 +01:00
92cf72fb5e start
Former-commit-id: 5ab7d4f90d
2025-11-28 05:28:20 +01:00
1762a7f9f2 start
Former-commit-id: b560240c17
2025-11-28 05:16:51 +01:00
7e1e08c49f start
Former-commit-id: ec2efd4661
2025-11-28 05:05:33 +01:00
5a6a144bb3 Allow shell commands in dokku run
Former-commit-id: c26264b92a
2025-11-28 03:02:18 +01:00
518e4e8973 start
Former-commit-id: 9ff96cdf2f
2025-11-28 02:22:28 +01:00
9ba861dabb all
Former-commit-id: 58b07e9e84
2025-11-28 02:22:17 +01:00
b418ae9525 all
Former-commit-id: 90ce6bc907
2025-11-28 02:19:48 +01:00
d9f3fcfda6 all
Former-commit-id: b1295f4329
2025-11-28 02:16:49 +01:00
79a592576e all
Former-commit-id: 026eb05912
2025-11-28 02:15:38 +01:00
341249b577 all
Former-commit-id: 0a2ea083df
2025-11-28 02:12:53 +01:00
0991cafc42 start
Former-commit-id: a11aa41cb2
2025-11-28 02:08:56 +01:00
58b204e5bb all
Former-commit-id: 7fbc9a2c96
2025-11-27 13:51:59 +01:00
3d33b152b2 all
Former-commit-id: d570da4b0b
2025-11-27 13:44:14 +01:00
1653cf4f52 all
Former-commit-id: cd156de795
2025-11-27 13:42:18 +01:00
cda7e16350 all
Former-commit-id: b57d395fed
2025-11-27 13:39:29 +01:00
df5b4fa8e8 all
Former-commit-id: 05d4a4bc63
2025-11-27 13:35:58 +01:00
59806aa9ac all
Former-commit-id: 858ed6a0cf
2025-11-27 13:32:11 +01:00
62a59775ba all
Former-commit-id: 20cb782a63
2025-11-27 13:28:09 +01:00
3ba235cfd0 all
Former-commit-id: 94061891e3
2025-11-27 13:20:40 +01:00
4688d9d16e all
Former-commit-id: 075fda64f6
2025-11-27 13:14:45 +01:00
529ad1cf6c all
Former-commit-id: c540518723
2025-11-27 13:11:01 +01:00
618dd21dcc all
Former-commit-id: fad11d56b8
2025-11-27 13:08:30 +01:00
283a041484 all
Former-commit-id: 7ce6abd3e7
2025-11-27 12:54:54 +01:00
7dd4fb64f7 all
Former-commit-id: 97b0fb09f0
2025-11-27 12:53:36 +01:00
072763330d all
Former-commit-id: 15a4936310
2025-11-27 12:51:07 +01:00
886244e908 all
Former-commit-id: 913e0dfcc1
2025-11-27 12:45:01 +01:00
0c8064801f all
Former-commit-id: adb66611c1
2025-11-27 12:43:27 +01:00
8db8bf08a1 all
Former-commit-id: b0304034cb
2025-11-27 12:41:36 +01:00
044419e3b2 all
Former-commit-id: 6c42429214
2025-11-27 12:33:40 +01:00
b5a6c45a9d start
Former-commit-id: 4f1957fe95
2025-11-27 12:31:54 +01:00
c8dd26da2a start
Former-commit-id: 7cb9599eda
2025-11-27 12:24:59 +01:00
4d61b50de5 start
Former-commit-id: 48fc49db9c
2025-11-27 12:21:29 +01:00
a208ae36b4 start
Former-commit-id: 1aa76771c4
2025-11-27 12:21:21 +01:00
a4c8f14139 start
Former-commit-id: afa52cb11c
2025-11-27 08:15:26 +01:00
cd0c283491 start
Former-commit-id: 47854d8b39
2025-11-26 13:05:04 +01:00
85 changed files with 15350 additions and 1571 deletions

26
.dockerignore Normal file
View File

@@ -0,0 +1,26 @@
# Exclude large model files from Docker build
models/
*.weights
# Exclude build artifacts
target/
*.class
*.jar
# Exclude version control
.git/
.gitignore
# Exclude IDE files
.idea/
*.iml
.vscode/
# Exclude data files
images/
*.db
*.db-journal
# Exclude docs
README.md
*.md

42
.github/workflows/__deploy.old vendored Normal file
View File

@@ -0,0 +1,42 @@
name: Build and Deploy
on:
push:
branches: ["main"]
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Java
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '21'
- name: Build
run: mvn -B clean package
- name: Upload to JFrog
run: |
curl -u "${{ secrets.JFROG_USER }}:${{ secrets.JFROG_PASS }}" \
-T target/*.jar \
"http://JFROG-SERVER/artifactory/myrepo/app-latest.jar"
deploy:
needs: build
runs-on: ubuntu-latest
steps:
- name: Trigger remote deploy script
uses: appleboy/ssh-action@v0.1.7
with:
host: ${{ secrets.SERVER_IP }}
username: ${{ secrets.SERVER_USER }}
key: ${{ secrets.SERVER_SSH_KEY }}
script: |
/opt/myapp/update.sh

50
.github/workflows/_oldbuild.nothing vendored Normal file
View File

@@ -0,0 +1,50 @@
name: Build and Deploy Auction App
on:
push:
branches:
- main
jobs:
build_and_deploy:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Maven
run: |
apt update
apt install -y maven
- name: Set up Java 17
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: 17
- name: Build with Maven
run: mvn -B clean package
- name: Copy jar to server (no tar)
uses: appleboy/scp-action@v0.1.7
with:
host: ${{ secrets.SERVER_IP }}
username: ${{ secrets.SERVER_USER }}
key: ${{ secrets.SERVER_SSH_KEY }}
source: "target/*.jar"
target: "/opt/auction/"
overwrite: true
strip_components: 1 # Strips the 'target/' directory
timeout: 60s
- name: Restart service
uses: appleboy/ssh-action@v0.1.7
with:
host: ${{ secrets.SERVER_IP }}
username: ${{ secrets.SERVER_USER }}
key: ${{ secrets.SERVER_SSH_KEY }}
script: |
systemctl restart auction
echo "Deploy complete"

8
.gitignore vendored
View File

@@ -26,5 +26,9 @@ bin/
### VS Code ###
.vscode/
### Mac OS ###
.DS_Store
### MacOS ###
.DS_Store
NUL
target/
.idea/

8
.idea/.gitignore generated vendored
View File

@@ -1,8 +0,0 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

View File

@@ -1,5 +0,0 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default copy" />
</state>
</component>

13
.idea/compiler.xml generated
View File

@@ -1,13 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="troostwijk-scraper" />
</profile>
</annotationProcessing>
</component>
</project>

7
.idea/encodings.xml generated
View File

@@ -1,7 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$/src/main/java" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/src/main/resources" charset="UTF-8" />
</component>
</project>

View File

@@ -1,621 +0,0 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="AddOperatorModifier" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="AddVarianceModifier" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ArrayInDataClass" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="BintrayPublishingPlugin" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="BooleanLiteralArgument" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="CanBeParameter" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="CanBePrimaryConstructorProperty" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="CanBeVal" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="CanSealedSubClassBeObject" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="CascadeIf" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="CdiAlternativeInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CdiDecoratorInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CdiDisposerMethodInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CdiDomBeans" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="CdiInjectInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CdiInjectionPointsInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="CdiInterceptorInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CdiManagedBeanInconsistencyInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CdiNormalScopeInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="CdiObservesInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CdiScopeInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="CdiSpecializesInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CdiStereotypeInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CdiStereotypeRestrictionsInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CdiTypedAnnotationInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CdiUnknownProducersForDisposerMethodInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="CdiUnproxyableBeanTypesInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="ChangeToMethod" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ChangeToOperator" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ClashingTraitMethods" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="ClassCanBeRecord" enabled="false" level="WEAK WARNING" enabled_by_default="false">
<option name="myConversionStrategy" value="SILENTLY" />
</inspection_tool>
<inspection_tool class="ClassName" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ComplexRedundantLet" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="ComposeUnknownKeys" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="ComposeUnknownValues" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="ConflictingExtensionProperty" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="ConstPropertyName" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ConstantConditionIf" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ControlFlowWithEmptyBody" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="ConvertCallChainIntoSequence" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="ConvertLambdaToReference" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ConvertNaNEquality" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="ConvertPairConstructorToToFunction" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ConvertReferenceToLambda" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ConvertSecondaryConstructorToPrimary" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="ConvertToStringTemplate" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="ConvertTryFinallyToUseCall" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ConvertTwoComparisonsToRangeCheck" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="CopyWithoutNamedArguments" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="CssBrowserCompatibilityForProperties" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="CssConvertColorToHexInspection" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="CssConvertColorToRgbInspection" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="CssMissingSemicolon" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="CucumberExamplesColon" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CucumberMissedExamples" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="CucumberTableInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="CucumberUndefinedStep" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DataClassPrivateConstructor" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DeferredIsResult" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="DeferredResultUnused" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DelegatesTo" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DelegationToVarProperty" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DeprecatedCallableAddReplaceWith" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="DeprecatedGradleDependency" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DeprecatedMavenDependency" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DestructuringWrongName" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DifferentKotlinGradleVersion" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DifferentKotlinMavenVersion" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DifferentMavenStdlibVersion" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DifferentStdlibGradleVersion" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DockerFileAddOrCopySemantic" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="DockerFileArgumentCount" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="DockerFileAssignments" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="EmptyRange" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="EmptyWebServiceClass" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="EnumEntryName" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="EqualsOrHashCode" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="ExplicitThis" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="FakeJvmFieldConstant" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="FoldInitializerAndIfToElvis" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="ForEachParameterNotUsed" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ForgottenDebugOutputInspection" enabled="true" level="ERROR" enabled_by_default="true">
<option name="configuration">
<list>
<option value="\Codeception\Util\Debug::debug" />
<option value="\Codeception\Util\Debug::pause" />
<option value="\Doctrine\Common\Util\Debug::dump" />
<option value="\Doctrine\Common\Util\Debug::export" />
<option value="\Illuminate\Support\Debug\Dumper::dump" />
<option value="\Symfony\Component\Debug\Debug::enable" />
<option value="\Symfony\Component\Debug\DebugClassLoader::enable" />
<option value="\Symfony\Component\Debug\ErrorHandler::register" />
<option value="\Symfony\Component\Debug\ExceptionHandler::register" />
<option value="\TYPO3\CMS\Core\Utility\DebugUtility::debug" />
<option value="\Zend\Debug\Debug::dump" />
<option value="\Zend\Di\Display\Console::export" />
<option value="dd" />
<option value="debug_print_backtrace" />
<option value="debug_zval_dump" />
<option value="dpm" />
<option value="dpq" />
<option value="dsm" />
<option value="dump" />
<option value="dvm" />
<option value="error_log" />
<option value="kpr" />
<option value="phpinfo" />
<option value="print_r" />
<option value="var_dump" />
<option value="var_export" />
<option value="wp_die" />
<option value="xdebug_break" />
<option value="xdebug_call_class" />
<option value="xdebug_call_file" />
<option value="xdebug_call_function" />
<option value="xdebug_call_line" />
<option value="xdebug_code_coverage_started" />
<option value="xdebug_debug_zval" />
<option value="xdebug_debug_zval_stdout" />
<option value="xdebug_dump_superglobals" />
<option value="xdebug_enable" />
<option value="xdebug_get_code_coverage" />
<option value="xdebug_get_collected_errors" />
<option value="xdebug_get_declared_vars" />
<option value="xdebug_get_function_stack" />
<option value="xdebug_get_headers" />
<option value="xdebug_get_monitored_functions" />
<option value="xdebug_get_profiler_filename" />
<option value="xdebug_get_stack_depth" />
<option value="xdebug_get_tracefile_name" />
<option value="xdebug_is_enabled" />
<option value="xdebug_memory_usage" />
<option value="xdebug_peak_memory_usage" />
<option value="xdebug_print_function_stack" />
<option value="xdebug_start_code_coverage" />
<option value="xdebug_start_error_collection" />
<option value="xdebug_start_function_monitor" />
<option value="xdebug_start_trace" />
<option value="xdebug_stop_code_coverage" />
<option value="xdebug_stop_error_collection" />
<option value="xdebug_stop_function_monitor" />
<option value="xdebug_stop_trace" />
<option value="xdebug_time_index" />
<option value="xdebug_var_dump" />
</list>
</option>
<option name="migratedIntoUserSpace" value="true" />
</inspection_tool>
<inspection_tool class="FunctionName" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="FunctionWithLambdaExpressionBody" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="GherkinBrokenTableInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="GherkinMisplacedBackground" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="GherkinScenarioToScenarioOutline" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="GrAnnotationReferencingUnknownIdentifiers" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GrDeprecatedAPIUsage" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GrEqualsBetweenInconvertibleTypes" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GrFinalVariableAccess" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GrMethodMayBeStatic" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GrNamedVariantLabels" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GrPackage" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GrReassignedInClosureLocalVar" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GrUnnecessaryAlias" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="GrUnnecessaryDefModifier" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GrUnnecessaryPublicModifier" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GrUnnecessarySemicolon" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GrUnresolvedAccess" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyAccessToStaticFieldLockedOnInstance" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyAccessibility" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyAssignabilityCheck" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyConditionalWithIdenticalBranches" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyConstantConditional" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyConstantIfStatement" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyConstructorNamedArguments" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyDivideByZero" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyDocCheck" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="GroovyDoubleNegation" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyDuplicateSwitchBranch" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyEmptyStatementBody" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyFallthrough" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyGStringKey" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyIfStatementWithIdenticalBranches" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyImplicitNullArgumentCall" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyInArgumentCheck" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyInfiniteLoopStatement" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyInfiniteRecursion" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyLabeledStatement" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyMissingReturnStatement" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyPointlessBoolean" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyResultOfObjectAllocationIgnored" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovySillyAssignment" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovySynchronizationOnNonFinalField" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovySynchronizationOnVariableInitializedWithLiteral" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyTrivialConditional" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyTrivialIf" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyUncheckedAssignmentOfMemberOfRawType" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyUnnecessaryContinue" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyUnnecessaryReturn" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyUnreachableStatement" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyUnsynchronizedMethodOverridesSynchronizedMethod" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyUnusedAssignment" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyUnusedCatchParameter" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyUnusedDeclaration" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyUnusedIncOrDec" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="GroovyVariableNotAssigned" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="HasPlatformType" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="HtmlUnknownTag" enabled="true" level="WARNING" enabled_by_default="true">
<option name="myValues">
<value>
<list size="16">
<item index="0" class="java.lang.String" itemvalue="nobr" />
<item index="1" class="java.lang.String" itemvalue="noembed" />
<item index="2" class="java.lang.String" itemvalue="comment" />
<item index="3" class="java.lang.String" itemvalue="noscript" />
<item index="4" class="java.lang.String" itemvalue="embed" />
<item index="5" class="java.lang.String" itemvalue="script" />
<item index="6" class="java.lang.String" itemvalue="style" />
<item index="7" class="java.lang.String" itemvalue="nuxt-img" />
<item index="8" class="java.lang.String" itemvalue="nuxt-picture" />
<item index="9" class="java.lang.String" itemvalue="nuxt-content" />
<item index="10" class="java.lang.String" itemvalue="contentdoc" />
<item index="11" class="java.lang.String" itemvalue="contentquery" />
<item index="12" class="java.lang.String" itemvalue="nuxt" />
<item index="13" class="java.lang.String" itemvalue="ogimagescreenshot" />
<item index="14" class="java.lang.String" itemvalue="seokit" />
<item index="15" class="java.lang.String" itemvalue="nuxtlink" />
</list>
</value>
</option>
<option name="myCustomValuesEnabled" value="true" />
</inspection_tool>
<inspection_tool class="HttpRequestContentLengthIsIgnored" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="HttpRequestPlaceholder" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="IfThenToElvis" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="IfThenToSafeAccess" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="ImplicitNullableNothingType" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ImplicitThis" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ImplicitlyExposedWebServiceMethods" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="IncorrectHttpHeaderInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="IntroduceWhenSubject" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="JBoss" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JCenterRepository" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JSConstructorReturnsPrimitive" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="JSObsoletePrivateAccessSyntax" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JSUnresolvedReactComponent" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="JavaCollectionsStaticMethod" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="JavaCollectionsStaticMethodOnImmutableList" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JavaFxColorRgb" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JavaFxDefaultTag" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JavaFxEventHandler" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JavaFxRedundantPropertyValue" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JavaFxResourcePropertyValue" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JavaFxUnresolvedFxIdReference" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JavaFxUnresolvedStyleClassReference" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JavaFxUnusedImports" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JavaMapForEach" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="JavaStylePropertiesInvocation" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="JoinDeclarationAndAssignment" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="JpaAttributeMemberSignatureInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JpaAttributeTypeInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JpaConfigDomFacetInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JpaDataSourceORMDomInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JpaDataSourceORMInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JpaDomInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JpaEntityListenerInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JpaEntityListenerWarningsInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="JpaMissingIdInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JpaModelReferenceInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JpaORMDomInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JpaObjectClassSignatureInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JpaQlInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="JpaQueryApiInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="KDocUnresolvedReference" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="KotlinCovariantEquals" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="KotlinDeprecation" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="KotlinDoubleNegation" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="KotlinEqualsBetweenInconvertibleTypes" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="KotlinInternalInJava" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="KotlinInvalidBundleOrProperty" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="KotlinMavenPluginPhase" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="KotlinRedundantOverride" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="KotlinSealedInheritorsInJava" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="KotlinTestJUnit" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="KotlinThrowableNotThrown" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="KotlinUnusedImport" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="LSPLocalInspectionTool" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="LateinitVarOverridesLateinitVar" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="LeakingThis" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="LessResolvedByNameOnly" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="LessUnresolvedMixin" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="LessUnresolvedVariable" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="LiftReturnOrAssignment" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="LocalVariableName" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="LoopToCallChain" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="MainFunctionReturnUnit" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MapGetWithNotNullAssertionOperator" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="MayBeConstant" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="MemberVisibilityCanBePrivate" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="MicronautDataMethodInconsistencyInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="MicronautDataRepositoryMethodParametersInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MicronautDataRepositoryMethodReturnTypeInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MigrateDiagnosticSuppression" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MissingOverrideAnnotation" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoreObjectMethods" value="true" />
<option name="ignoreAnonymousClassMethods" value="false" />
</inspection_tool>
<inspection_tool class="MnProperties" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MnUnresolvedPathVariable" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MnYaml" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MongoJSDeprecationInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MongoJSResolveInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MongoJSSideEffectsInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MoveLambdaOutsideParentheses" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="MoveSuspiciousCallableReferenceIntoParentheses" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="MoveVariableDeclarationIntoWhen" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="MsOrderByInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="MultipleMethodDesignatorsInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="MultipleRepositoryUrls" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MysqlLoadDataPathInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="MysqlParsingInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="NestedLambdaShadowedImplicitParameter" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="NewInstanceOfSingleton" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="NonJaxWsWebServices" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="NullChecksToSafeCall" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="NullableBooleanElvis" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="ObjectLiteralToLambda" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="ObjectPropertyName" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ObsoleteExperimentalCoroutines" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="ObsoleteKotlinJsPackages" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="OneWayWebMethod" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="OptionalExpectation" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="OraMissingBodyInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="OraOverloadInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="OraUnmatchedForwardDeclarationInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="OverridingDeprecatedMember" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="PackageDirectoryMismatch" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="PackageName" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="PathAnnotation" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="PgSelectFromProcedureInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="PlatformExtensionReceiverOfInline" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="PrivatePropertyName" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="PropertyName" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ProtectedInFinal" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="PyArgumentEqualDefaultInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true" />
<inspection_tool class="PyAugmentAssignmentInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true" />
<inspection_tool class="PyBehaveInspection" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="PyClassicStyleClassInspection" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="PyCompatibilityInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ourVersions">
<value>
<list size="2">
<item index="0" class="java.lang.String" itemvalue="2.7" />
<item index="1" class="java.lang.String" itemvalue="3.11" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyMandatoryEncodingInspection" enabled="true" level="WARNING" enabled_by_default="true" />
<inspection_tool class="PyMissingOrEmptyDocstringInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true" />
<inspection_tool class="PyMissingTypeHintsInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true" />
<inspection_tool class="RecursiveEqualsCall" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RecursivePropertyAccessor" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantAsSequence" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantAsync" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantCompanionReference" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantElseInIf" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="RedundantElvisReturnNull" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantEmptyInitializerBlock" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="RedundantEnumConstructorInvocation" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantExplicitType" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantGetter" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantIf" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantLambdaArrow" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantLambdaOrAnonymousFunction" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantModalityModifier" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantNullableReturnType" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantObjectTypeCheck" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="RedundantRequireNotNullCall" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantReturnLabel" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantRunCatching" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantSamConstructor" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantSemicolon" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantSetter" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantSuspendModifier" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantUnitExpression" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantUnitReturnType" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantVisibilityModifier" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RedundantWith" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveCurlyBracesFromTemplate" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveEmptyClassBody" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="RemoveEmptyParenthesesFromAnnotationEntry" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveEmptyParenthesesFromLambdaCall" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="RemoveEmptyPrimaryConstructor" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveEmptySecondaryConstructorBody" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveExplicitSuperQualifier" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveExplicitTypeArguments" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveForLoopIndices" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveRedundantBackticks" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveRedundantCallsOfConversionMethods" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveRedundantQualifierName" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveRedundantSpreadOperator" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveSetterParameterType" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveSingleExpressionStringTemplate" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="RemoveToStringInStringTemplate" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceArrayEqualityOpWithArraysEquals" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceArrayOfWithLiteral" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceAssertBooleanWithAssertEquality" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceAssociateFunction" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceCallWithBinaryOperator" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceGetOrSet" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="ReplaceGuardClauseWithFunctionCall" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ReplaceIsEmptyWithIfEmpty" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceJavaStaticMethodWithKotlinAnalog" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceManualRangeWithIndicesCalls" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="ReplaceNegatedIsEmptyWithIsNotEmpty" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplacePutWithAssignment" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceRangeStartEndInclusiveWithFirstLast" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceRangeToWithUntil" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceSizeCheckWithIsNotEmpty" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceSizeZeroCheckWithIsEmpty" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceStringFormatWithLiteral" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ReplaceSubstringWithDropLast" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ReplaceSubstringWithIndexingOperation" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ReplaceSubstringWithSubstringAfter" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ReplaceSubstringWithSubstringBefore" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ReplaceSubstringWithTake" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ReplaceToStringWithStringTemplate" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="ReplaceToWithInfixForm" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceWithEnumMap" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceWithIgnoreCaseEquals" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="ReplaceWithOperatorAssignment" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="RestParamTypeInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="RestResourceMethodInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="RestWrongDefaultValueInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="SafeCastWithReturn" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SassScssResolvedByNameOnly" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SassScssUnresolvedMixin" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SassScssUnresolvedPlaceholderSelector" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SassScssUnresolvedVariable" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="ScopeFunctionConversion" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="SecondUnsafeCall" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SecurityAdvisoriesInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="optionConfiguration">
<list>
<option value="barryvdh/laravel-debugbar" />
<option value="behat/behat" />
<option value="brianium/paratest" />
<option value="codeception/codeception" />
<option value="codedungeon/phpunit-result-printer" />
<option value="composer/composer" />
<option value="doctrine/coding-standard" />
<option value="filp/whoops" />
<option value="friendsofphp/php-cs-fixer" />
<option value="humbug/humbug" />
<option value="infection/infection" />
<option value="jakub-onderka/php-parallel-lint" />
<option value="johnkary/phpunit-speedtrap" />
<option value="kalessil/production-dependencies-guard" />
<option value="mikey179/vfsStream" />
<option value="mockery/mockery" />
<option value="mybuilder/phpunit-accelerator" />
<option value="orchestra/testbench" />
<option value="pdepend/pdepend" />
<option value="phan/phan" />
<option value="phing/phing" />
<option value="phpcompatibility/php-compatibility" />
<option value="phpmd/phpmd" />
<option value="phpro/grumphp" />
<option value="phpspec/phpspec" />
<option value="phpspec/prophecy" />
<option value="phpstan/phpstan" />
<option value="phpunit/phpunit" />
<option value="povils/phpmnd" />
<option value="roave/security-advisories" />
<option value="satooshi/php-coveralls" />
<option value="sebastian/phpcpd" />
<option value="slevomat/coding-standard" />
<option value="spatie/phpunit-watcher" />
<option value="squizlabs/php_codesniffer" />
<option value="sstalle/php7cc" />
<option value="symfony/debug" />
<option value="symfony/maker-bundle" />
<option value="symfony/phpunit-bridge" />
<option value="symfony/var-dumper" />
<option value="vimeo/psalm" />
<option value="wimg/php-compatibility" />
<option value="wp-coding-standards/wpcs" />
<option value="yiisoft/yii2-coding-standards" />
<option value="yiisoft/yii2-debug" />
<option value="yiisoft/yii2-gii" />
<option value="zendframework/zend-coding-standard" />
<option value="zendframework/zend-debug" />
<option value="zendframework/zend-test" />
</list>
</option>
</inspection_tool>
<inspection_tool class="SelfAssignment" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SelfReferenceConstructorParameter" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SetterBackingFieldAssignment" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SimpleRedundantLet" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SimplifiableCall" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SimplifiableCallChain" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SimplifyAssertNotNull" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="SimplifyBooleanWithConstants" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SimplifyNegatedBinaryExpression" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SimplifyNestedEachInScopeFunction" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SimplifyWhenWithBooleanConstantCondition" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SingletonConstructor" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="SortModifiers" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SqlAddNotNullColumnInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlAggregatesInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlAmbiguousColumnInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlAutoIncrementDuplicateInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlCallNotationInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="SqlCaseVsCoalesceInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlCaseVsIfInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlCheckUsingColumnsInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlConstantConditionInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlCurrentSchemaInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlDeprecateTypeInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlDerivedTableAliasInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlDialectInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlDropIndexedColumnInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlDtInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlDuplicateColumnInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlErrorHandlingInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="SqlIdentifierInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlIdentifierLengthInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="SqlIllegalCursorStateInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlInsertIntoGeneratedColumnInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlInsertNullIntoNotNullInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlInsertValuesInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlJoinWithoutOnInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlMisleadingReferenceInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlMissingReturnInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="SqlMultipleLimitClausesInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlNoDataSourceInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlNullComparisonInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlRedundantAliasInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlRedundantCodeInCoalesceInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlRedundantElseNullInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlRedundantLimitInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlRedundantOrderingDirectionInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlResolveInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="SqlShadowingAliasInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlShouldBeInGroupByInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlSideEffectsInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlSignatureInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlStorageInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlStringLengthExceededInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlTransactionStatementInTriggerInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlTriggerTransitionInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlTypeInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlUnicodeStringLiteralInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlUnreachableCodeInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlUnusedCteInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlUnusedSubqueryItemInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlUnusedVariableInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SqlWithoutWhereInspection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="Stylelint" enabled="true" level="ERROR" enabled_by_default="true" />
<inspection_tool class="SuspendFunctionOnCoroutineScope" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SuspiciousAsDynamic" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SuspiciousCollectionReassignment" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="SuspiciousEqualsCombination" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SuspiciousVarProperty" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="TestFunctionName" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="TrailingComma" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="TypeCustomizer" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="TypeScriptUnresolvedFunction" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="TypeScriptUnresolvedReactComponent" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="TypeScriptUnresolvedVariable" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="TypeScriptValidateJSTypes" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="UnlabeledReturnInsideLambda" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="UnnecessaryQualifiedReference" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="UnnecessaryVariable" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="UnresolvedReference" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="UnresolvedRestParam" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="UnsafeCastFromDynamic" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="UnstableApiUsage" enabled="true" level="WARNING" enabled_by_default="true">
<option name="unstableApiAnnotations">
<set>
<option value="io.reactivex.annotations.Beta" />
<option value="io.reactivex.annotations.Experimental" />
<option value="org.apache.http.annotation.Beta" />
<option value="org.gradle.api.Incubating" />
<option value="org.jetbrains.annotations.ApiStatus.Experimental" />
<option value="org.jetbrains.annotations.ApiStatus.Internal" />
<option value="org.jetbrains.annotations.ApiStatus.ScheduledForRemoval" />
<option value="rx.annotations.Beta" />
<option value="rx.annotations.Experimental" />
</set>
</option>
</inspection_tool>
<inspection_tool class="UnusedDataClassCopyResult" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="UnusedEquals" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="UnusedLambdaExpressionBody" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="UnusedReceiverParameter" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="UnusedSymbol" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="UnusedUnaryOperator" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="UseExpressionBody" enabled="false" level="INFORMATION" enabled_by_default="false" />
<inspection_tool class="UsePropertyAccessSyntax" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="UseWithIndex" enabled="false" level="INFO" enabled_by_default="false" />
<inspection_tool class="UselessCallOnCollection" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="UselessCallOnNotNull" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="ValidExternallyBoundObject" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="VoidMethodAnnotatedWithGET" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="W3CssValidation" enabled="true" level="WARNING" enabled_by_default="true">
<option name="myCssVersion" value="css3svg" />
<option name="myIgnoreVendorSpecificProperties" value="false" />
</inspection_tool>
<inspection_tool class="WSReferenceInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="WadlDomInspection" enabled="false" level="ERROR" enabled_by_default="false" />
<inspection_tool class="WebpackConfigHighlighting" enabled="false" level="WARNING" enabled_by_default="false" />
<inspection_tool class="WrapUnaryOperator" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="WsdlHighlightingInspection" enabled="false" level="ERROR" enabled_by_default="false" />
</profile>
</component>

View File

@@ -1,30 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="RemoteRepositoriesConfiguration">
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Apache Central Repository" />
<option name="url" value="https://repo.maven.apache.org/maven2" />
</remote-repository>
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Maven Central repository" />
<option name="url" value="https://repo1.maven.org/maven2" />
</remote-repository>
<remote-repository>
<option name="id" value="jboss.community" />
<option name="name" value="JBoss Community repository" />
<option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
</remote-repository>
<remote-repository>
<option name="id" value="maven-central" />
<option name="name" value="Maven Central Repository" />
<option name="url" value="https://repo1.maven.org/maven2/" />
</remote-repository>
<remote-repository>
<option name="id" value="artifactory" />
<option name="name" value="libs-release-local" />
<option name="url" value="https://repo.appmodel.nl/artifactory/libs-release-local" />
</remote-repository>
</component>
</project>

View File

@@ -1,10 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="MaterialThemeProjectNewConfig">
<option name="metadata">
<MTProjectMetadataState>
<option name="userId" value="-3d6055cc:19abfe94c99:-7ff7" />
</MTProjectMetadataState>
</option>
</component>
</project>

14
.idea/misc.xml generated
View File

@@ -1,14 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_25" default="true" project-jdk-name="openjdk-ea-25" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>

6
.idea/vcs.xml generated
View File

@@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

6
.mvn/jvm.config Normal file
View File

@@ -0,0 +1,6 @@
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--enable-native-access=ALL-UNNAMED

117
.mvn/wrapper/MavenWrapperDownloader.java vendored Normal file
View File

@@ -0,0 +1,117 @@
/*
* Copyright 2007-present the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.net.*;
import java.io.*;
import java.nio.channels.*;
import java.util.Properties;
public class MavenWrapperDownloader {
private static final String WRAPPER_VERSION = "0.5.6";
/**
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
*/
private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+ WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
/**
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
* use instead of the default one.
*/
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
".mvn/wrapper/maven-wrapper.properties";
/**
* Path where the maven-wrapper.jar will be saved to.
*/
private static final String MAVEN_WRAPPER_JAR_PATH =
".mvn/wrapper/maven-wrapper.jar";
/**
* Name of the property which should be used to override the default download url for the wrapper.
*/
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
public static void main(String args[]) {
System.out.println("- Downloader started");
File baseDirectory = new File(args[0]);
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
// If the maven-wrapper.properties exists, read it and check if it contains a custom
// wrapperUrl parameter.
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
String url = DEFAULT_DOWNLOAD_URL;
if(mavenWrapperPropertyFile.exists()) {
FileInputStream mavenWrapperPropertyFileInputStream = null;
try {
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
Properties mavenWrapperProperties = new Properties();
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
} catch (IOException e) {
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
} finally {
try {
if(mavenWrapperPropertyFileInputStream != null) {
mavenWrapperPropertyFileInputStream.close();
}
} catch (IOException e) {
// Ignore ...
}
}
}
System.out.println("- Downloading from: " + url);
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
if(!outputFile.getParentFile().exists()) {
if(!outputFile.getParentFile().mkdirs()) {
System.out.println(
"- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
}
}
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
try {
downloadFileFromURL(url, outputFile);
System.out.println("Done");
System.exit(0);
} catch (Throwable e) {
System.out.println("- Error downloading");
e.printStackTrace();
System.exit(1);
}
}
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
String username = System.getenv("MVNW_USERNAME");
char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
Authenticator.setDefault(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(username, password);
}
});
}
URL website = new URL(urlString);
ReadableByteChannel rbc;
rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(destination);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
rbc.close();
}
}

1
.mvn/wrapper/maven-wrapper.config vendored Normal file
View File

@@ -0,0 +1 @@
jvmArguments=-Djava.util.logging.manager=org.jboss.logmanager.LogManager --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED

BIN
.mvn/wrapper/maven-wrapper.jar vendored Normal file

Binary file not shown.

2
.mvn/wrapper/maven-wrapper.properties vendored Normal file
View File

@@ -0,0 +1,2 @@
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip
wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar

BIN
.mvn/wrapper/maven-wrapper_old.jar vendored Normal file

Binary file not shown.

41
Dockerfile Normal file
View File

@@ -0,0 +1,41 @@
# Stage 1: Build
FROM maven:3.9-eclipse-temurin-25-alpine AS builder
WORKDIR /app
# Copy POM first (allows for cached dependency layer)
COPY pom.xml .
RUN mvn dependency:resolve -B
COPY src ./src
# Updated with both properties to avoid the warning
RUN mvn package -DskipTests -Dquarkus.package.jar.type=uber-jar -Dquarkus.package.jar.enabled=true
# Stage 2: Runtime (DEBIAN-based for OpenCV native libs)
FROM eclipse-temurin:25-jre
WORKDIR /app
# Install dependencies + wget for health checks
RUN apt-get update && \
apt-get install -y --no-install-recommends \
libstdc++6 \
libgomp1 \
wget \
&& rm -rf /var/lib/apt/lists/*
# Create user (Debian syntax)
RUN groupadd -r quarkus && useradd -r -g quarkus quarkus
# Create user with explicit UID 1000 (Debian syntax)
# RUN groupadd -r -g 1000 quarkus && useradd -r -u 1000 -g quarkus quarkus
# Create the data directory and set ownership BEFORE switching user
# RUN mkdir -p /mnt/okcomputer/output && chown -R quarkus:quarkus /mnt/okcomputer/output
# Copy the built jar with correct pattern
COPY --from=builder --chown=quarkus:quarkus /app/target/auctiora-*.jar app.jar
USER quarkus
EXPOSE 8081
ENTRYPOINT ["java", \
"-Dio.netty.tryReflectionSetAccessible=true", \
"--enable-native-access=ALL-UNNAMED", \
"--add-opens", "java.base/java.nio=ALL-UNNAMED", \
"-jar", "app.jar"]

563
README.md Normal file
View File

@@ -0,0 +1,563 @@
# Troostwijk Auction Scraper
A Java-based web scraper for Dutch auctions on Troostwijk Auctions with **100% free** desktop/email notifications, SQLite persistence, and AI-powered object detection.
## Features
- **Auction Discovery**: Automatically discovers active Dutch auctions
- **Data Scraping**: Fetches detailed lot information via Troostwijk's JSON API
- **SQLite Storage**: Persists auction data, lots, images, and detected objects
- **Image Processing**: Downloads and analyzes lot images using OpenCV YOLO object detection
- **Free Notifications**: Real-time notifications when:
- Bids change on monitored lots
- Auctions are closing soon (within 5 minutes)
- Via desktop notifications (Windows/macOS/Linux system tray) ✅
- Optionally via email (Gmail SMTP - free) ✅
## Dependencies
All dependencies are managed via Maven (see `pom.xml`):
- **jsoup 1.17.2** - HTML parsing and HTTP client
- **Jackson 2.17.0** - JSON processing
- **SQLite JDBC 3.45.1.0** - Database operations
- **JavaMail 1.6.2** - Email notifications (free)
- **OpenCV 4.9.0** - Image processing and object detection
## Quick Start
### Development: Sync Production Data
To work with real production data locally:
```powershell
# Linux/Mac (Bash)
./scripts/sync-production-data.sh --db-only
```
See [scripts/README.md](scripts/README.md) for full documentation.
## Setup
### 1. Notification Options (Choose One)
#### Option A: Desktop Notifications Only ⭐ (Recommended - Zero Setup)
Desktop notifications work out of the box on:
- **Windows**: System tray notifications
- **macOS**: Notification Center
- **Linux**: Desktop environment notifications (GNOME, KDE, etc.)
**No configuration required!** Just run with default settings:
```bash
export NOTIFICATION_CONFIG="desktop"
# Or simply don't set it - desktop is the default
```
#### Option B: Desktop + Email Notifications 📧 (Free Gmail)
1. Enable 2-Factor Authentication in your Google Account
2. Go to: **Google Account → Security → 2-Step Verification → App passwords**
3. Generate an app password for "Mail"
4. Set environment variable:
```bash
export NOTIFICATION_CONFIG="smtp:your.email@gmail.com:your_app_password:recipient@example.com"
```
**Format**: `smtp:username:app_password:recipient_email`
**Example**:
```bash
export NOTIFICATION_CONFIG="smtp:john.doe@gmail.com:abcd1234efgh5678:john.doe@gmail.com"
```
**Note**: This is completely free using Gmail's SMTP server. No paid services required!
### 2. OpenCV Native Libraries
Download and install OpenCV native libraries for your platform:
**Windows:**
```bash
# Download from https://opencv.org/releases/
# Extract and add to PATH or use:
java -Djava.library.path="C:\opencv\build\java\x64" -jar scraper.jar
```
**Linux:**
```bash
sudo apt-get install libopencv-dev
```
**macOS:**
```bash
brew install opencv
```
### 3. YOLO Model Files
Download YOLO model files for object detection:
```bash
mkdir /mnt/okcomputer/output/models
cd /mnt/okcomputer/output/models
# Download YOLOv4 config
wget https://raw.githubusercontent.com/AlexeyAB/darknet/master/cfg/yolov4.cfg
# Download YOLOv4 weights (245 MB)
wget https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v3_optimal/yolov4.weights
# Download COCO class names
wget https://raw.githubusercontent.com/AlexeyAB/darknet/master/data/coco.names
```
## Building
```bash
mvn clean package
```
This creates:
- `target/troostwijk-scraper-1.0-SNAPSHOT.jar` - Regular JAR
- `target/troostwijk-scraper-1.0-SNAPSHOT-jar-with-dependencies.jar` - Executable JAR with all dependencies
## Running
### Quick Start (Desktop Notifications Only)
```bash
java -Djava.library.path="/path/to/opencv/lib" \
-jar target/troostwijk-scraper-1.0-SNAPSHOT-jar-with-dependencies.jar
```
### With Email Notifications
```bash
export NOTIFICATION_CONFIG="smtp:your@gmail.com:app_password:your@gmail.com"
java -Djava.library.path="/path/to/opencv/lib" \
-jar target/troostwijk-scraper-1.0-SNAPSHOT-jar-with-dependencies.jar
```
### Using Maven
```bash
mvn exec:java -Dexec.mainClass="com.auction.scraper.TroostwijkScraper"
```
## System Architecture & Integration Flow
> **📊 Complete Integration Flowchart**: See [docs/INTEGRATION_FLOWCHART.md](docs/INTEGRATION_FLOWCHART.md) for the detailed intelligence integration diagram with GraphQL API fields, analytics, and dashboard features.
### Quick Overview
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ COMPLETE SYSTEM INTEGRATION DIAGRAM │
└─────────────────────────────────────────────────────────────────────────────┘
┌──────────────────────────────────────────────────────────────────────────────┐
│ PHASE 1: EXTERNAL SCRAPER (Python/Playwright) - ARCHITECTURE-TROOSTWIJK │
└──────────────────────────────────────────────────────────────────────────────┘
┌─────────────────────────────┼─────────────────────────────┐
▼ ▼ ▼
[Listing Pages] [Auction Pages] [Lot Pages]
/auctions?page=N /a/auction-id /l/lot-id
│ │ │
│ Extract URLs │ Parse __NEXT_DATA__ │ Parse __NEXT_DATA__
├────────────────────────────▶│ JSON │ JSON
│ │ │
│ ▼ ▼
│ ┌────────────────┐ ┌────────────────┐
│ │ INSERT auctions│ │ INSERT lots │
│ │ to SQLite │ │ INSERT images │
│ └────────────────┘ │ (URLs only) │
│ │ └────────────────┘
│ │ │
└─────────────────────────────┴────────────────────────────┘
┌──────────────────┐
│ SQLITE DATABASE │
│ output/cache.db │
└──────────────────┘
┌─────────────────┼─────────────────┐
▼ ▼ ▼
[auctions table] [lots table] [images table]
- auction_id - lot_id - id
- title - auction_id - lot_id
- location - title - url
- lots_count - current_bid - local_path
- closing_time - bid_count - downloaded=0
- closing_time
┌─────────────────────────────────────┴─────────────────────────────────────┐
│ PHASE 2: MONITORING & PROCESSING (Java) - THIS PROJECT │
└────────────────────────────────────────────────────────────────────────────┘
┌─────────────────┼─────────────────┐
▼ ▼ ▼
[TroostwijkMonitor] [DatabaseService] [ScraperDataAdapter]
│ │ │
│ Read lots │ Query lots │ Transform data
│ every hour │ Import images │ TEXT → INTEGER
│ │ │ "€123" → 123.0
└─────────────────┴─────────────────┘
┌─────────────────────────┼─────────────────────────┐
▼ ▼ ▼
[Bid Monitoring] [Image Processing] [Closing Alerts]
Check API every 1h Download images Check < 5 min
│ │ │
│ New bid? │ Process via │ Time critical?
├─[YES]──────────┐ │ ObjectDetection ├─[YES]────┐
│ │ │ │ │
▼ │ ▼ │ │
[Update current_bid] │ ┌──────────────────┐ │ │
in database │ │ YOLO Detection │ │ │
│ │ OpenCV DNN │ │ │
│ └──────────────────┘ │ │
│ │ │ │
│ │ Detect objects │ │
│ ├─[vehicle] │ │
│ ├─[furniture] │ │
│ ├─[machinery] │ │
│ │ │ │
│ ▼ │ │
│ [Save labels to DB] │ │
│ [Estimate value] │ │
│ │ │ │
│ │ │ │
└─────────┴───────────────────────┴──────────┘
┌───────────────────────────────────────────────┴────────────────────────────┐
│ PHASE 3: NOTIFICATION SYSTEM - USER INTERACTION TRIGGERS │
└────────────────────────────────────────────────────────────────────────────┘
┌─────────────────┴─────────────────┐
▼ ▼
[NotificationService] [User Decision Points]
│ │
┌───────────────────┼───────────────────┐ │
▼ ▼ ▼ │
[Desktop Notify] [Email Notify] [Priority Level] │
Windows/macOS/ Gmail SMTP 0=Normal │
Linux system (FREE) 1=High │
tray │
│ │ │ │
└───────────────────┴───────────────────┘ │
│ │
▼ ▼
┌──────────────────┐ ┌──────────────────┐
│ USER INTERACTION │ │ TRIGGER EVENTS: │
│ NOTIFICATIONS │ │ │
└──────────────────┘ └──────────────────┘
│ │
┌───────────────────┼───────────────────┐ │
▼ ▼ ▼ │
┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ │
│ 1. BID CHANGE │ │ 2. OBJECT │ │ 3. CLOSING │ │
│ │ │ DETECTED │ │ ALERT │ │
│ "Nieuw bod op │ │ │ │ │ │
│ kavel 12345: │ │ "Lot contains: │ │ "Kavel 12345 │ │
│ €150 (was €125)"│ │ - Vehicle │ │ sluit binnen │ │
│ │ │ - Machinery │ │ 5 min." │ │
│ Priority: NORMAL │ │ Est: €5000" │ │ Priority: HIGH │ │
│ │ │ │ │ │ │
│ Action needed: │ │ Action needed: │ │ Action needed: │ │
│ ▸ Place bid? │ │ ▸ Review item? │ │ ▸ Place final │ │
│ ▸ Monitor? │ │ ▸ Confirm value? │ │ bid? │ │
│ ▸ Ignore? │ │ ▸ Add to watch? │ │ ▸ Let expire? │ │
└──────────────────┘ └──────────────────┘ └──────────────────┘ │
│ │ │ │
└───────────────────┴───────────────────┴─────────────┘
┌─────────────────────────────────────────────────────────────────────────────┐
│ USER ACTIONS & EXCEPTIONS │
├─────────────────────────────────────────────────────────────────────────────┤
│ Additional interaction points: │
│ │
│ 4. VIEWING DAY QUESTIONS │
│ "Bezichtiging op [date] - kunt u aanwezig zijn?" │
│ Action: ▸ Confirm attendance ▸ Request alternative ▸ Decline │
│ │
│ 5. ITEM RECOGNITION CONFIRMATION │
│ "Detected: [object] - Is deze correcte identificatie?" │
│ Action: ▸ Confirm ▸ Correct label ▸ Add notes │
│ │
│ 6. VALUE ESTIMATE APPROVAL │
│ "Geschatte waarde: €X - Akkoord?" │
│ Action: ▸ Accept ▸ Adjust ▸ Request manual review │
│ │
│ 7. EXCEPTION HANDLING │
│ "Afwijkende sluitingstijd / locatiewijziging / special terms" │
│ Action: ▸ Acknowledge ▸ Update preferences ▸ Withdraw interest │
└─────────────────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────────────────┐
│ OBJECT DETECTION & VALUE ESTIMATION PIPELINE │
└─────────────────────────────────────────────────────────────────────────────┘
[Downloaded Image] → [ImageProcessingService]
│ │
│ ▼
│ [ObjectDetectionService]
│ │
│ ├─ Load YOLO model
│ ├─ Run inference (416x416)
│ ├─ Post-process detections
│ │ (confidence > 0.5)
│ │
│ ▼
│ ┌──────────────────────┐
│ │ Detected Objects: │
│ │ - person │
│ │ - car │
│ │ - truck │
│ │ - furniture │
│ │ - machinery │
│ │ - electronics │
│ │ (80 COCO classes) │
│ └──────────────────────┘
│ │
│ ▼
│ [Value Estimation Logic]
│ (Future enhancement)
│ │
│ ├─ Match objects to auction categories
│ ├─ Historical price analysis
│ ├─ Condition assessment
│ ├─ Market trends
│ │
│ ▼
│ ┌──────────────────────┐
│ │ Estimated Value: │
│ │ €X - €Y range │
│ │ Confidence: 75% │
│ └──────────────────────┘
│ │
└──────────────────────┴─ [Save to DB]
[Trigger notification if
value > threshold]
```
```mermaid
flowchart TD
subgraph P1["PHASE 1: EXTERNAL SCRAPER (Python/Playwright)"]
direction LR
A1[Listing Pages<br/>/auctions?page=N] --> A2[Extract URLs]
B1[Auction Pages<br/>/a/auction-id] --> B2[Parse __NEXT_DATA__ JSON]
C1[Lot Pages<br/>/l/lot-id] --> C2[Parse __NEXT_DATA__ JSON]
A2 --> D1[INSERT auctions to SQLite]
B2 --> D1
C2 --> D2[INSERT lots & image URLs]
D1 --> DB[(SQLite Database<br/>output/cache.db)]
D2 --> DB
end
DB --> P2_Entry
subgraph P2["PHASE 2: MONITORING & PROCESSING (Java)"]
direction TB
P2_Entry[Data Ready] --> Monitor[TroostwijkMonitor<br/>Read lots every hour]
P2_Entry --> DBService[DatabaseService<br/>Query & Import]
P2_Entry --> Adapter[ScraperDataAdapter<br/>Transform TEXT → INTEGER]
Monitor --> BM[Bid Monitoring<br/>Check API every 1h]
DBService --> IP[Image Processing<br/>Download & Analyze]
Adapter --> DataForNotify[Formatted Data]
BM --> BidUpdate{New bid?}
BidUpdate -->|Yes| UpdateDB[Update current_bid in DB]
UpdateDB --> NotifyTrigger1
IP --> Detection[Object Detection<br/>YOLO/OpenCV DNN]
Detection --> ObjectCheck{Detect objects?}
ObjectCheck -->|Vehicle| Save1[Save labels & estimate value]
ObjectCheck -->|Furniture| Save2[Save labels & estimate value]
ObjectCheck -->|Machinery| Save3[Save labels & estimate value]
Save1 --> NotifyTrigger2
Save2 --> NotifyTrigger2
Save3 --> NotifyTrigger2
CA[Closing Alerts<br/>Check < 5 min] --> TimeCheck{Time critical?}
TimeCheck -->|Yes| NotifyTrigger3
end
NotifyTrigger1 --> NS
NotifyTrigger2 --> NS
NotifyTrigger3 --> NS
subgraph P3["PHASE 3: NOTIFICATION SYSTEM"]
NS[NotificationService] --> DN[Desktop Notify<br/>Windows/macOS/Linux]
NS --> EN[Email Notify<br/>Gmail SMTP]
NS --> PL[Set Priority Level<br/>0=Normal, 1=High]
end
DN --> UI[User Interaction & Decisions]
EN --> UI
PL --> UI
subgraph UI_Details[User Decision Points / Trigger Events]
direction LR
E1["1. BID CHANGE<br/>'Nieuw bod op kavel 12345...'<br/>Actions: Place bid? Monitor? Ignore?"]
E2["2. OBJECT DETECTED<br/>'Lot contains: Vehicle...'<br/>Actions: Review? Confirm value?"]
E3["3. CLOSING ALERT<br/>'Kavel 12345 sluit binnen 5 min.'<br/>Actions: Place final bid? Let expire?"]
E4["4. VIEWING DAY QUESTIONS<br/>'Bezichtiging op [date]...'"]
E5["5. ITEM RECOGNITION CONFIRMATION<br/>'Detected: [object]...'"]
E6["6. VALUE ESTIMATE APPROVAL<br/>'Geschatte waarde: €X...'"]
E7["7. EXCEPTION HANDLING<br/>'Afwijkende sluitingstijd...'"]
end
UI --> UI_Details
%% Object Detection Sub-Flow Detail
subgraph P2_Detail["Object Detection & Value Estimation Pipeline"]
direction LR
DI[Downloaded Image] --> IPS[ImageProcessingService]
IPS --> ODS[ObjectDetectionService]
ODS --> Load[Load YOLO model]
ODS --> Run[Run inference]
ODS --> Post[Post-process detections<br/>confidence > 0.5]
Post --> ObjList["Detected Objects List<br/>(80 COCO classes)"]
ObjList --> VEL[Value Estimation Logic<br/>Future enhancement]
VEL --> Match[Match to categories]
VEL --> History[Historical price analysis]
VEL --> Condition[Condition assessment]
VEL --> Market[Market trends]
Market --> ValueEst["Estimated Value Range<br/>Confidence: 75%"]
ValueEst --> SaveToDB[Save to Database]
SaveToDB --> TriggerNotify{Value > threshold?}
end
IP -.-> P2_Detail
TriggerNotify -.-> NotifyTrigger2
```
## Integration Hooks & Timing
| Event | Frequency | Trigger | Notification Type | User Action Required |
|--------------------------------|-------------------|----------------------------|----------------------------|------------------------|
| **New auction discovered** | On scrape | Scraper finds new auction | Desktop + Email (optional) | Review auction |
| **Bid change detected** | Every 1 hour | Monitor detects higher bid | Desktop + Email | Place counter-bid? |
| **Closing soon (< 30 min)** | When detected | Time-based check | Desktop + Email | Review lot |
| **Closing imminent (< 5 min)** | When detected | Time-based check | Desktop + Email (HIGH) | Final bid decision |
| **Object detected** | On image process | YOLO finds objects | Desktop + Email | Confirm identification |
| **Value estimated** | After detection | Estimation complete | Desktop + Email | Approve estimate |
| **Viewing day scheduled** | From lot metadata | Scraper extracts date | Desktop + Email | Confirm attendance |
| **Exception/Change** | On update | Scraper detects change | Desktop + Email (HIGH) | Acknowledge |
## Project Structure
```
src/main/java/com/auction/
├── Main.java # Entry point
├── TroostwijkMonitor.java # Monitoring & orchestration
├── DatabaseService.java # SQLite operations
├── ScraperDataAdapter.java # Schema translation (TEXT→INT, €→float)
├── ImageProcessingService.java # Downloads & processes images
├── ObjectDetectionService.java # OpenCV YOLO detection
├── NotificationService.java # Desktop + Email notifications (FREE)
├── Lot.java # Domain model for auction lots
├── AuctionInfo.java # Domain model for auctions
└── Console.java # Logging utility
```
## Configuration
Edit `TroostwijkScraper.main()` to customize:
- **Database file**: `troostwijk.db` (SQLite database location)
- **YOLO paths**: Model configuration and weights files
- **Monitoring frequency**: Default is every 1 hour
- **Closing alerts**: Default is 5 minutes before closing
## Database Schema
The scraper creates three tables:
**sales**
- `sale_id` (PRIMARY KEY)
- `title`, `location`, `closing_time`
**lots**
- `lot_id` (PRIMARY KEY)
- `sale_id`, `title`, `description`, `manufacturer`, `type`, `year`
- `category`, `current_bid`, `currency`, `url`
- `closing_time`, `closing_notified`
**images**
- `id` (PRIMARY KEY)
- `lot_id`, `url`, `local_path`, `labels` (detected objects)
## Notification Examples
### Desktop Notification
![System Tray Notification]
```
🔔 Kavel bieding update
Nieuw bod op kavel 12345: €150.00 (was €125.00)
```
### Email Notification
```
From: your.email@gmail.com
To: your.email@gmail.com
Subject: [Troostwijk] Kavel bieding update
Nieuw bod op kavel 12345: €150.00 (was €125.00)
```
**High Priority Alerts** (closing soon):
```
⚠️ Lot nearing closure
Kavel 12345 sluit binnen 5 min.
```
## Why This Approach?
✅ **100% Free** - No paid services (Twilio, Pushover, etc.)
✅ **No External Dependencies** - Desktop notifications built into Java
✅ **Works Offline** - Desktop notifications don't need internet
✅ **Privacy First** - Your data stays on your machine
✅ **Cross-Platform** - Windows, macOS, Linux supported
✅ **Optional Email** - Add Gmail notifications if you want
## Troubleshooting
### Desktop Notifications Not Showing
- **Windows**: Check if Java has notification permissions
- **Linux**: Ensure you have a desktop environment running (not headless)
- **macOS**: Check System Preferences → Notifications
### Email Not Sending
1. Verify 2FA is enabled in Google Account
2. Confirm you're using an **App Password** (not your regular Gmail password)
3. Check that "Less secure app access" is NOT needed (app passwords work with 2FA)
4. Verify the SMTP format: `smtp:username:app_password:recipient`
## Notes
- Desktop notifications require a graphical environment (not headless servers)
- For headless servers, use email-only notifications
- Gmail SMTP is free and has generous limits (500 emails/day)
- OpenCV native libraries must match your platform architecture
- YOLO weights file is ~245 MB
```shell
ssh tour@athena.lan "docker run --rm -v shared-auction-data:/data -v /tmp:/tmp alpine cp /data/cache.db /tmp/cache.db" && scp tour@athena.lan:/tmp/cache.db c:/mnt/okcomputer/cache.db
```
## License
This is example code for educational purposes.

View File

@@ -1,11 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

56
docker-compose.yml Normal file
View File

@@ -0,0 +1,56 @@
services:
auctiora:
user: "1000:1000"
build:
context: /opt/apps/auctiora
dockerfile: Dockerfile
container_name: auctiora
restart: unless-stopped
networks:
- traefik_net
environment:
# Database configuration
- AUCTION_DATABASE_PATH=/mnt/okcomputer/output/cache.db
- AUCTION_IMAGES_PATH=/mnt/okcomputer/output/images
# Notification configuration
# - AUCTION_NOTIFICATION_CONFIG=desktop
- AUCTION_NOTIFICATION_CONFIG=smtp:michael.bakker1986@gmail.com:agrepolhlnvhipkv:michael.bakker1986@gmail.com
# Quarkus configuration
- QUARKUS_HTTP_PORT=8081
- QUARKUS_HTTP_HOST=0.0.0.0
- QUARKUS_LOG_CONSOLE_LEVEL=INFO
# Scheduler configuration (cron expressions)
- AUCTION_WORKFLOW_SCRAPER_IMPORT_CRON=0 */30 * * * ?
- AUCTION_WORKFLOW_IMAGE_PROCESSING_CRON=0 0 * * * ?
- AUCTION_WORKFLOW_BID_MONITORING_CRON=0 */15 * * * ?
- AUCTION_WORKFLOW_CLOSING_ALERTS_CRON=0 */5 * * * ?
volumes:
# Mount database and images directory1
- shared-auction-data:/mnt/okcomputer/output
labels:
- "traefik.enable=true"
- "traefik.http.routers.auctiora.rule=Host(`auctiora.appmodel.nl`)"
- "traefik.http.routers.auctiora.entrypoints=websecure"
- "traefik.http.routers.auctiora.tls=true"
- "traefik.http.routers.auctiora.tls.certresolver=letsencrypt"
- "traefik.http.services.auctiora.loadbalancer.server.port=8081"
#healthcheck:
# test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8081/q/health/live"]
# interval: 30s
# timeout: 3s
# retries: 3
# start_period: 10s
networks:
traefik_net:
external: true
name: traefik_net
volumes:
shared-auction-data:
external: true

View File

@@ -0,0 +1,326 @@
# Troostwijk Scraper - Architecture & Data Flow
## System Overview
The scraper follows a **3-phase hierarchical crawling pattern** to extract auction and lot data from Troostwijk Auctions website.
## Architecture Diagram
```
┌─────────────────────────────────────────────────────────────────┐
│ TROOSTWIJK SCRAPER │
└─────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 1: COLLECT AUCTION URLs │
│ ┌──────────────┐ ┌──────────────┐ │
│ │ Listing Page │────────▶│ Extract /a/ │ │
│ │ /auctions? │ │ auction URLs │ │
│ │ page=1..N │ └──────────────┘ │
│ └──────────────┘ │ │
│ ▼ │
│ [ List of Auction URLs ] │
└─────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 2: EXTRACT LOT URLs FROM AUCTIONS │
│ ┌──────────────┐ ┌──────────────┐ │
│ │ Auction Page │────────▶│ Parse │ │
│ │ /a/... │ │ __NEXT_DATA__│ │
│ └──────────────┘ │ JSON │ │
│ │ └──────────────┘ │
│ │ │ │
│ ▼ ▼ │
│ ┌──────────────┐ ┌──────────────┐ │
│ │ Save Auction │ │ Extract /l/ │ │
│ │ Metadata │ │ lot URLs │ │
│ │ to DB │ └──────────────┘ │
│ └──────────────┘ │ │
│ ▼ │
│ [ List of Lot URLs ] │
└─────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────┐
│ PHASE 3: SCRAPE LOT DETAILS │
│ ┌──────────────┐ ┌──────────────┐ │
│ │ Lot Page │────────▶│ Parse │ │
│ │ /l/... │ │ __NEXT_DATA__│ │
│ └──────────────┘ │ JSON │ │
│ └──────────────┘ │
│ │ │
│ ┌─────────────────────────┴─────────────────┐ │
│ ▼ ▼ │
│ ┌──────────────┐ ┌──────────────┐ │
│ │ Save Lot │ │ Save Images │ │
│ │ Details │ │ URLs to DB │ │
│ │ to DB │ └──────────────┘ │
│ └──────────────┘ │ │
│ ▼ │
│ [Optional Download] │
└─────────────────────────────────────────────────────────────────┘
```
## Database Schema
```sql
CACHE TABLE (HTML Storage with Compression)
cache
url (TEXT, PRIMARY KEY)
content (BLOB) -- Compressed HTML (zlib) │
timestamp (REAL)
status_code (INTEGER)
compressed (INTEGER) -- 1=compressed, 0=plain │
AUCTIONS TABLE
auctions
auction_id (TEXT, PRIMARY KEY) -- e.g. "A7-39813" │
url (TEXT, UNIQUE)
title (TEXT)
location (TEXT) -- e.g. "Cluj-Napoca, RO" │
lots_count (INTEGER)
first_lot_closing_time (TEXT)
scraped_at (TEXT)
LOTS TABLE
lots
lot_id (TEXT, PRIMARY KEY) -- e.g. "A1-28505-5" │
auction_id (TEXT) -- FK to auctions │
url (TEXT, UNIQUE)
title (TEXT)
current_bid (TEXT) -- "€123.45" or "No bids" │
bid_count (INTEGER)
closing_time (TEXT)
viewing_time (TEXT)
pickup_date (TEXT)
location (TEXT) -- e.g. "Dongen, NL" │
description (TEXT)
category (TEXT)
scraped_at (TEXT)
FOREIGN KEY (auction_id) auctions(auction_id)
IMAGES TABLE (Image URLs & Download Status)
images THIS TABLE HOLDS IMAGE LINKS
id (INTEGER, PRIMARY KEY AUTOINCREMENT)
lot_id (TEXT) -- FK to lots │
url (TEXT) -- Image URL │
local_path (TEXT) -- Path after download │
downloaded (INTEGER) -- 0=pending, 1=downloaded │
FOREIGN KEY (lot_id) lots(lot_id)
```
## Sequence Diagram
```
User Scraper Playwright Cache DB Data Tables
│ │ │ │ │
│ Run │ │ │ │
├──────────────▶│ │ │ │
│ │ │ │ │
│ │ Phase 1: Listing Pages │ │
│ ├───────────────▶│ │ │
│ │ goto() │ │ │
│ │◀───────────────┤ │ │
│ │ HTML │ │ │
│ ├───────────────────────────────▶│ │
│ │ compress & cache │ │
│ │ │ │ │
│ │ Phase 2: Auction Pages │ │
│ ├───────────────▶│ │ │
│ │◀───────────────┤ │ │
│ │ HTML │ │ │
│ │ │ │ │
│ │ Parse __NEXT_DATA__ JSON │ │
│ │────────────────────────────────────────────────▶│
│ │ │ │ INSERT auctions
│ │ │ │ │
│ │ Phase 3: Lot Pages │ │
│ ├───────────────▶│ │ │
│ │◀───────────────┤ │ │
│ │ HTML │ │ │
│ │ │ │ │
│ │ Parse __NEXT_DATA__ JSON │ │
│ │────────────────────────────────────────────────▶│
│ │ │ │ INSERT lots │
│ │────────────────────────────────────────────────▶│
│ │ │ │ INSERT images│
│ │ │ │ │
│ │ Export to CSV/JSON │ │
│ │◀────────────────────────────────────────────────┤
│ │ Query all data │ │
│◀──────────────┤ │ │ │
│ Results │ │ │ │
```
## Data Flow Details
### 1. **Page Retrieval & Caching**
```
Request URL
├──▶ Check cache DB (with timestamp validation)
│ │
│ ├─[HIT]──▶ Decompress (if compressed=1)
│ │ └──▶ Return HTML
│ │
│ └─[MISS]─▶ Fetch via Playwright
│ │
│ ├──▶ Compress HTML (zlib level 9)
│ │ ~70-90% size reduction
│ │
│ └──▶ Store in cache DB (compressed=1)
└──▶ Return HTML for parsing
```
### 2. **JSON Parsing Strategy**
```
HTML Content
└──▶ Extract <script id="__NEXT_DATA__">
├──▶ Parse JSON
│ │
│ ├─[has pageProps.lot]──▶ Individual LOT
│ │ └──▶ Extract: title, bid, location, images, etc.
│ │
│ └─[has pageProps.auction]──▶ AUCTION
│ │
│ ├─[has lots[] array]──▶ Auction with lots
│ │ └──▶ Extract: title, location, lots_count
│ │
│ └─[no lots[] array]──▶ Old format lot
│ └──▶ Parse as lot
└──▶ Fallback to HTML regex parsing (if JSON fails)
```
### 3. **Image Handling**
```
Lot Page Parsed
├──▶ Extract images[] from JSON
│ │
│ └──▶ INSERT INTO images (lot_id, url, downloaded=0)
└──▶ [If DOWNLOAD_IMAGES=True]
├──▶ Download each image
│ │
│ ├──▶ Save to: /images/{lot_id}/001.jpg
│ │
│ └──▶ UPDATE images SET local_path=?, downloaded=1
└──▶ Rate limit between downloads (0.5s)
```
## Key Configuration
| Setting | Value | Purpose |
|---------|-------|---------|
| `CACHE_DB` | `/mnt/okcomputer/output/cache.db` | SQLite database path |
| `IMAGES_DIR` | `/mnt/okcomputer/output/images` | Downloaded images storage |
| `RATE_LIMIT_SECONDS` | `0.5` | Delay between requests |
| `DOWNLOAD_IMAGES` | `False` | Toggle image downloading |
| `MAX_PAGES` | `50` | Number of listing pages to crawl |
## Output Files
```
/mnt/okcomputer/output/
├── cache.db # SQLite database (compressed HTML + data)
├── auctions_{timestamp}.json # Exported auctions
├── auctions_{timestamp}.csv # Exported auctions
├── lots_{timestamp}.json # Exported lots
├── lots_{timestamp}.csv # Exported lots
└── images/ # Downloaded images (if enabled)
├── A1-28505-5/
│ ├── 001.jpg
│ └── 002.jpg
└── A1-28505-6/
└── 001.jpg
```
## Extension Points for Integration
### 1. **Downstream Processing Pipeline**
```python
# Query lots without downloaded images
SELECT lot_id, url FROM images WHERE downloaded = 0
# Process images: OCR, classification, etc.
# Update status when complete
UPDATE images SET downloaded = 1, local_path = ? WHERE id = ?
```
### 2. **Real-time Monitoring**
```python
# Check for new lots every N minutes
SELECT COUNT(*) FROM lots WHERE scraped_at > datetime('now', '-1 hour')
# Monitor bid changes
SELECT lot_id, current_bid, bid_count FROM lots WHERE bid_count > 0
```
### 3. **Analytics & Reporting**
```python
# Top locations
SELECT location, COUNT(*) as lot_count FROM lots GROUP BY location
# Auction statistics
SELECT
a.auction_id,
a.title,
COUNT(l.lot_id) as actual_lots,
SUM(CASE WHEN l.bid_count > 0 THEN 1 ELSE 0 END) as lots_with_bids
FROM auctions a
LEFT JOIN lots l ON a.auction_id = l.auction_id
GROUP BY a.auction_id
```
### 4. **Image Processing Integration**
```python
# Get all images for a lot
SELECT url, local_path FROM images WHERE lot_id = 'A1-28505-5'
# Batch process unprocessed images
SELECT i.id, i.lot_id, i.local_path, l.title, l.category
FROM images i
JOIN lots l ON i.lot_id = l.lot_id
WHERE i.downloaded = 1 AND i.local_path IS NOT NULL
```
## Performance Characteristics
- **Compression**: ~70-90% HTML size reduction (1GB → ~100-300MB)
- **Rate Limiting**: Exactly 0.5s between requests (respectful scraping)
- **Caching**: 24-hour default cache validity (configurable)
- **Throughput**: ~7,200 pages/hour (with 0.5s rate limit)
- **Scalability**: SQLite handles millions of rows efficiently
## Error Handling
- **Network failures**: Cached as status_code=500, retry after cache expiry
- **Parse failures**: Falls back to HTML regex patterns
- **Compression errors**: Auto-detects and handles uncompressed legacy data
- **Missing fields**: Defaults to "No bids", empty string, or 0
## Rate Limiting & Ethics
- **REQUIRED**: 0.5 second delay between ALL requests
- **Respects cache**: Avoids unnecessary re-fetching
- **User-Agent**: Identifies as standard browser
- **No parallelization**: Single-threaded sequential crawling

View File

@@ -0,0 +1,258 @@
# Database Architecture
## Overview
The Auctiora auction monitoring system uses **SQLite** as its database engine, shared between the scraper process and the monitor application for simplicity and performance.
## Current State (Dec 2025)
- **Database**: `C:\mnt\okcomputer\output\cache.db`
- **Size**: 1.6 GB
- **Records**: 16,006 lots, 536,502 images
- **Concurrent Processes**: 2 (scraper + monitor)
- **Access Pattern**: Scraper writes, Monitor reads + occasional updates
## Why SQLite?
### ✅ Advantages for This Use Case
1. **Embedded Architecture**
- No separate database server to manage
- Zero network latency (local file access)
- Perfect for single-machine scraping + monitoring
2. **Excellent Read Performance**
- Monitor performs mostly SELECT queries
- Well-indexed access by `lot_id`, `url`, `auction_id`
- Sub-millisecond query times for simple lookups
3. **Simplicity**
- Single file database
- Automatic backup via file copy
- No connection pooling or authentication overhead
4. **Proven Scalability**
- Tested up to 281 TB database size
- 1.6 GB is only 0.0006% of capacity
- Handles billions of rows efficiently
5. **WAL Mode for Concurrency**
- Multiple readers don't block each other
- Readers don't block writers
- Writers don't block readers
- Perfect for scraper + monitor workload
## Configuration
### Connection String (DatabaseService.java:28)
```java
jdbc:sqlite:C:\mnt\okcomputer\output\cache.db?journal_mode=WAL&busy_timeout=10000
```
### Key PRAGMAs (DatabaseService.java:38-40)
```sql
PRAGMA journal_mode=WAL; -- Write-Ahead Logging for concurrency
PRAGMA busy_timeout=10000; -- 10s retry on lock contention
PRAGMA synchronous=NORMAL; -- Balance safety and performance
```
### What These Settings Do
| Setting | Purpose | Impact |
|---------|---------|--------|
| `journal_mode=WAL` | Write-Ahead Logging | Enables concurrent read/write access |
| `busy_timeout=10000` | Wait 10s on lock | Prevents immediate `SQLITE_BUSY` errors |
| `synchronous=NORMAL` | Balanced sync mode | Faster writes, still crash-safe |
## Schema Integration
### Scraper Schema (Read-Only for Monitor)
```sql
CREATE TABLE lots (
lot_id TEXT PRIMARY KEY,
auction_id TEXT,
url TEXT UNIQUE, -- ⚠️ Enforced by scraper
title TEXT,
current_bid TEXT,
closing_time TEXT,
manufacturer TEXT,
type TEXT,
year INTEGER,
currency TEXT DEFAULT 'EUR',
closing_notified INTEGER DEFAULT 0,
...
)
```
### Monitor Schema (Tables Created by Monitor)
```sql
CREATE TABLE images (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id INTEGER,
url TEXT,
local_path TEXT,
labels TEXT, -- Object detection results
processed_at INTEGER,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)
```
### Handling Schema Conflicts
**Problem**: Scraper has `UNIQUE` constraint on `lots.url`
**Solution** (DatabaseService.java:361-424):
```java
// Try UPDATE first
UPDATE lots SET ... WHERE lot_id = ?
// If no rows updated, INSERT OR IGNORE
INSERT OR IGNORE INTO lots (...) VALUES (...)
```
This approach:
- ✅ Updates existing lots by `lot_id`
- ✅ Skips inserts that violate UNIQUE constraints
- ✅ No crashes on re-imports or duplicate URLs
## Performance Characteristics
### Current Performance
- Simple SELECT by ID: <1ms
- Full table scan (16K lots): ~50ms
- Image INSERT: <5ms
- Concurrent operations: No blocking observed
### Scalability Projections
| Metric | Current | 1 Year | 3 Years | SQLite Limit |
|--------|---------|--------|---------|--------------|
| Lots | 16K | 365K | 1M | 1B+ rows |
| Images | 536K | 19M | 54M | 1B+ rows |
| DB Size | 1.6GB | 36GB | 100GB | 281TB |
| Queries | <1ms | <5ms | <20ms | Depends on indexes |
## When to Migrate to PostgreSQL/MySQL
### 🚨 Migration Triggers
Consider migrating if you encounter **any** of these:
1. **Concurrency Limits**
- >5 concurrent writers needed
- Frequent `SQLITE_BUSY` errors despite WAL mode
- Need for distributed access across multiple servers
2. **Performance Degradation**
- Database >50GB AND queries >1s for simple SELECTs
- Complex JOIN queries become bottleneck
- Index sizes exceed available RAM
3. **Operational Requirements**
- Need for replication (master/slave)
- Geographic distribution required
- High availability / failover needed
- Remote access from multiple locations
4. **Advanced Features**
- Full-text search on large text fields
- Complex analytical queries (window functions, CTEs)
- User management and fine-grained permissions
- Connection pooling for web applications
### Migration Path (If Needed)
1. **Choose Database**: PostgreSQL (recommended) or MySQL
2. **Schema Export**: Use SQLite `.schema` command
3. **Data Migration**: Use `sqlite3-to-postgres` or custom scripts
4. **Update Connection**: Change JDBC URL in `application.properties`
5. **Update Queries**: Fix SQL dialect differences
6. **Performance Tuning**: Create appropriate indexes
Example PostgreSQL configuration:
```properties
# application.properties
auction.database.url=jdbc:postgresql://localhost:5432/auctiora
auction.database.username=monitor
auction.database.password=${DB_PASSWORD}
```
## Current Recommendation: ✅ **Stick with SQLite**
### Rationale
1. **Sufficient Capacity**: 1.6GB is 0.0006% of SQLite's limit
2. **Excellent Performance**: Sub-millisecond queries
3. **Simple Operations**: No complex transactions or analytics
4. **Low Concurrency**: Only 2 processes (scraper + monitor)
5. **Local Architecture**: No need for network DB access
6. **Zero Maintenance**: No DB server to manage or monitor
### Monitoring Dashboard Metrics
Track these to know when to reconsider:
```sql
-- Add to praetium.html dashboard
SELECT
(SELECT COUNT(*) FROM lots) as lot_count,
(SELECT COUNT(*) FROM images) as image_count,
(SELECT page_count * page_size FROM pragma_page_count(), pragma_page_size()) as db_size_bytes,
(SELECT (page_count - freelist_count) * 100.0 / page_count FROM pragma_page_count(), pragma_freelist_count()) as db_utilization
```
**Review decision when**:
- Database >20GB
- Query times >500ms for simple lookups
- More than 3 concurrent processes needed
## Backup Strategy
### Recommended Approach
```bash
# Nightly backup via Windows Task Scheduler
sqlite3 C:\mnt\okcomputer\output\cache.db ".backup C:\backups\cache_$(date +%Y%m%d).db"
# Keep last 30 days
forfiles /P C:\backups /M cache_*.db /D -30 /C "cmd /c del @path"
```
### WAL File Management
SQLite creates additional files in WAL mode:
- `cache.db` - Main database
- `cache.db-wal` - Write-Ahead Log
- `cache.db-shm` - Shared memory
**Important**: Backup all three files together for consistency.
## Integration Points
### Scraper Process
- **Writes**: INSERT new lots, auctions, images
- **Schema Owner**: Creates tables, enforces constraints
- **Frequency**: Continuous (every 30 minutes)
### Monitor Process (Auctiora)
- **Reads**: SELECT lots, auctions for monitoring
- **Writes**: UPDATE bid amounts, notification flags; INSERT image processing results
- **Schema**: Adds `images` table for object detection
- **Frequency**: Every 15 seconds (dashboard refresh)
### Conflict Resolution
| Conflict | Strategy | Implementation |
|----------|----------|----------------|
| Duplicate lot_id | UPDATE instead of INSERT | DatabaseService.upsertLot() |
| Duplicate URL | INSERT OR IGNORE | Silent skip |
| Oversized IDs (>Long.MAX_VALUE) | Return 0L, skip import | ScraperDataAdapter.extractNumericId() |
| Invalid timestamps | Try-catch, log, continue | DatabaseService.getAllAuctions() |
| Database locked | 10s busy_timeout + WAL | Connection string |
## References
- [SQLite Documentation](https://www.sqlite.org/docs.html)
- [WAL Mode](https://www.sqlite.org/wal.html)
- [SQLite Limits](https://www.sqlite.org/limits.html)
- [When to Use SQLite](https://www.sqlite.org/whentouse.html)

109
docs/DATA_SYNC_SETUP.md Normal file
View File

@@ -0,0 +1,109 @@
# Production Data Sync Setup
Quick reference for syncing production data from `athena.lan` to your local development environment.
## 🚀 One-Command Setup
### Linux/Mac
```bash
./scripts/sync-production-data.sh
```
## 📋 Complete Usage
### Bash (Linux/Mac/Git Bash)
```bash
# Database only
./scripts/sync-production-data.sh --db-only
# Everything
./scripts/sync-production-data.sh --all
# Images only
./scripts/sync-production-data.sh --images-only
```
## 🔧 What It Does
1. **Connects to athena.lan** via SSH
2. **Copies database** from Docker volume to /tmp
3. **Downloads to local** machine (c:\mnt\okcomputer\cache.db)
4. **Backs up** existing local database automatically
5. **Shows statistics** (auction count, lot count, etc.)
6. **Cleans up** temporary files on remote server
### With Images
- Also syncs the `/data/images/` directory
- Uses rsync for incremental sync (if available)
- Can be large (several GB)
## 📊 What You Get
### Database (`cache.db`)
- **~8.9 GB** of production data
- 16,000+ lots
- 536,000+ images metadata
- Full auction history
- HTTP cache from scraper
### Images (`images/`)
- Downloaded lot images
- Organized by lot ID
- Variable size (can be large)
## ⚡ Quick Workflow
### Daily Development
```powershell
# Morning: Get fresh data
.\scripts\Sync-ProductionData.sh -Force
# Develop & test
mvn quarkus:dev
# View dashboard
start http://localhost:8080
```
## 🔒 Safety Features
-**Automatic backups** before overwriting
-**Confirmation prompts** (unless `-Force`)
-**Error handling** with clear messages
-**Cleanup** of temporary files
-**Non-destructive** - production data is never modified
## 🐛 Troubleshooting
### "Permission denied" or SSH errors
```bash
# Test SSH connection
ssh tour@athena.lan "echo OK"
# If fails, check your SSH key
ssh-add -l
```
### Database already exists
- Script automatically backs up existing database
- Backup format: `cache.db.backup-YYYYMMDD-HHMMSS`
### Slow image transfer
- Install rsync for 10x faster incremental sync
- Or sync database only: `.\scripts\Sync-ProductionData.sh` (default)
## 📚 Full Documentation
See [scripts/README.md](../scripts/README.md) for:
- Prerequisites
- Performance tips
- Automation setup
- Detailed troubleshooting
## 🎯 Common Use Cases
**Quick Links**:
- [Main README](../README.md)
- [Scripts Documentation](../scripts/README.md)
- [Integration Flowchart](INTEGRATION_FLOWCHART.md)
- [Intelligence Features](INTELLIGENCE_FEATURES_SUMMARY.md)

153
docs/EXPERT_ANALITICS.sql Normal file
View File

@@ -0,0 +1,153 @@
-- Extend 'lots' table
ALTER TABLE lots
ADD COLUMN starting_bid DECIMAL(12, 2);
ALTER TABLE lots
ADD COLUMN estimated_min DECIMAL(12, 2);
ALTER TABLE lots
ADD COLUMN estimated_max DECIMAL(12, 2);
ALTER TABLE lots
ADD COLUMN reserve_price DECIMAL(12, 2);
ALTER TABLE lots
ADD COLUMN reserve_met BOOLEAN DEFAULT FALSE;
ALTER TABLE lots
ADD COLUMN bid_increment DECIMAL(12, 2);
ALTER TABLE lots
ADD COLUMN watch_count INTEGER DEFAULT 0;
ALTER TABLE lots
ADD COLUMN view_count INTEGER DEFAULT 0;
ALTER TABLE lots
ADD COLUMN first_bid_time TEXT;
ALTER TABLE lots
ADD COLUMN last_bid_time TEXT;
ALTER TABLE lots
ADD COLUMN bid_velocity DECIMAL(5, 2);
-- bids per hour
-- New table: bid history (CRITICAL)
CREATE TABLE bid_history
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT REFERENCES lots (lot_id),
bid_amount DECIMAL(12, 2) NOT NULL,
bid_time TEXT NOT NULL,
is_winning BOOLEAN DEFAULT FALSE,
is_autobid BOOLEAN DEFAULT FALSE,
bidder_id TEXT, -- anonymized
created_at TEXT DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX idx_bid_history_lot_time ON bid_history (lot_id, bid_time);
-- Extend 'lots' table
ALTER TABLE lots
ADD COLUMN condition_score DECIMAL(3, 2); -- 0.00-10.00
ALTER TABLE lots
ADD COLUMN condition_description TEXT;
ALTER TABLE lots
ADD COLUMN year_manufactured INTEGER;
ALTER TABLE lots
ADD COLUMN serial_number TEXT;
ALTER TABLE lots
ADD COLUMN originality_score DECIMAL(3, 2); -- % original parts
ALTER TABLE lots
ADD COLUMN provenance TEXT;
ALTER TABLE lots
ADD COLUMN comparable_lot_ids TEXT;
-- JSON array
-- New table: comparable sales
CREATE TABLE comparable_sales
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT REFERENCES lots (lot_id),
comparable_lot_id TEXT,
similarity_score DECIMAL(3, 2), -- 0.00-1.00
price_difference_percent DECIMAL(5, 2),
created_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- New table: market indices
CREATE TABLE market_indices
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
category TEXT NOT NULL,
manufacturer TEXT,
avg_price DECIMAL(12, 2),
median_price DECIMAL(12, 2),
price_change_30d DECIMAL(5, 2),
volume_change_30d DECIMAL(5, 2),
calculated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- Extend 'auctions' table
ALTER TABLE auctions
ADD COLUMN auction_house TEXT;
ALTER TABLE auctions
ADD COLUMN auction_house_rating DECIMAL(3, 2);
ALTER TABLE auctions
ADD COLUMN buyers_premium_percent DECIMAL(5, 2);
ALTER TABLE auctions
ADD COLUMN payment_methods TEXT; -- JSON
ALTER TABLE auctions
ADD COLUMN shipping_cost_min DECIMAL(12, 2);
ALTER TABLE auctions
ADD COLUMN shipping_cost_max DECIMAL(12, 2);
ALTER TABLE auctions
ADD COLUMN seller_verified BOOLEAN DEFAULT FALSE;
-- New table: auction performance metrics
CREATE TABLE auction_metrics
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
auction_id TEXT REFERENCES auctions (auction_id),
sell_through_rate DECIMAL(5, 2),
avg_hammer_vs_estimate DECIMAL(5, 2),
total_hammer_price DECIMAL(15, 2),
total_starting_price DECIMAL(15, 2),
calculated_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- New table: seasonal trends
CREATE TABLE seasonal_trends
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
category TEXT NOT NULL,
month INTEGER NOT NULL,
avg_price_multiplier DECIMAL(4, 2), -- vs annual avg
volume_multiplier DECIMAL(4, 2),
PRIMARY KEY (category, month)
);
-- New table: external market data
CREATE TABLE external_market_data
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
category TEXT NOT NULL,
manufacturer TEXT,
model TEXT,
dealer_avg_price DECIMAL(12, 2),
retail_avg_price DECIMAL(12, 2),
wholesale_avg_price DECIMAL(12, 2),
source TEXT,
fetched_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- New table: image analysis results
CREATE TABLE image_analysis
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
image_id INTEGER REFERENCES images (id),
damage_detected BOOLEAN,
damage_severity DECIMAL(3, 2),
wear_level TEXT CHECK (wear_level IN ('EXCELLENT', 'GOOD', 'FAIR', 'POOR')),
estimated_hours_used INTEGER,
ai_confidence DECIMAL(3, 2)
);
-- New table: economic indicators
CREATE TABLE economic_indicators
(
id INTEGER PRIMARY KEY AUTOINCREMENT,
indicator_date TEXT NOT NULL,
currency TEXT NOT NULL,
exchange_rate DECIMAL(10, 4),
inflation_rate DECIMAL(5, 2),
market_volatility DECIMAL(5, 2)
);

View File

@@ -0,0 +1,38 @@
```mermaid
graph TD
A[Add bid_history table] --> B[Add watch_count + estimates]
B --> C[Create market_indices]
C --> D[Add condition + year fields]
D --> E[Build comparable matching]
E --> F[Enrich with auction house data]
F --> G[Add AI image analysis]
```
| Current Practice | New Requirement | Why |
|-----------------------|---------------------------------|---------------------------|
| Scrape once per hour | **Scrape every bid update** | Capture velocity & timing |
| Save only current bid | **Save full bid history** | Detect patterns & sniping |
| Ignore watchers | **Track watch\_count** | Predict competition |
| Skip auction metadata | **Capture house estimates** | Anchor valuations |
| No historical data | **Store sold prices** | Train prediction models |
| Basic text scraping | **Parse condition/serial/year** | Enable comparables |
```bazaar
Week 1-2: Foundation
Implement bid_history scraping (most critical)
Add watch_count, starting_bid, estimated_min/max fields
Calculate basic bid_velocity
Week 3-4: Valuation
Extract year_manufactured, manufacturer, condition_description
Create market_indices (manually or via external API)
Build comparable lot matching logic
Week 5-6: Intelligence Layer
Add auction house performance tracking
Implement undervaluation detection algorithm
Create price alert system
Week 7-8: Automation
Integrate image analysis API
Add economic indicator tracking
Refine ML-based price predictions
```

126
docs/GraphQL.md Normal file
View File

@@ -0,0 +1,126 @@
# GraphQL Auction Schema Explorer
A Python script for exploring and testing GraphQL queries against the TBAuctions storefront API. This tool helps understand the auction schema by testing different query structures and viewing the responses.
## Features
- Three pre-configured GraphQL queries with varying levels of detail
- Asynchronous HTTP requests using aiohttp for efficient testing
- Error handling and formatted JSON output
- Configurable auction ID, locale, and platform parameters
## Prerequisites
- Python 3.7 or higher
- Required packages: `aiohttp`
## Installation
1. Clone or download this script
2. Install dependencies:
```bash
pip install aiohttp
```
## Usage
Run the script directly:
```bash
python auction_explorer.py
```
Or make it executable and run:
```bash
chmod +x auction_explorer.py
./auction_explorer.py
```
## Queries Included
The script tests three different query structures:
### 1. `viewingDays_simple`
Basic query that retrieves city and country code for viewing days.
### 2. `viewingDays_with_times`
Extended query that includes date ranges (`from` and `to`) along with city information.
### 3. `full_auction`
Comprehensive query that fetches:
- Auction ID and display ID
- Bidding status
- Buyer's premium
- Viewing days with location and timing
- Collection days with location and timing
## Configuration
Modify these variables in the script as needed:
```python
GRAPHQL_ENDPOINT = "https://storefront.tbauctions.com/storefront/graphql"
auction_id = "9d5d9d6b-94de-4147-b523-dfa512d85dfa" # Replace with your auction ID
variables = {
"auctionId": auction_id,
"locale": "nl", # Change locale as needed
"platform": "TWK" # Change platform as needed
}
```
## Output Format
The script outputs:
- Query name and separator
- Success status with formatted JSON response
- Or error messages if the query fails
Example output:
```
============================================================
QUERY: viewingDays_simple
============================================================
SUCCESS:
{
"data": {
"auction": {
"viewingDays": [
{
"city": "Amsterdam",
"countryCode": "NL"
}
]
}
}
}
```
## Customization
To add new queries, extend the `QUERIES` dictionary:
```python
QUERIES = {
"your_query_name": """
query YourQuery($auctionId: TbaUuid!, $locale: String!, $platform: Platform!) {
auction(id: $auctionId, locale: $locale, platform: $platform) {
# Your fields here
}
}
""",
# ... existing queries
}
```
## Notes
- The script includes a 500ms delay between queries to avoid rate limiting
- Timeout is set to 30 seconds per request
- All queries use the same GraphQL endpoint and variables
- Error responses are displayed in a readable format
## License
This script is provided for educational and exploratory purposes.

View File

@@ -0,0 +1,393 @@
# Auctiora Intelligence Integration Flowchart
## Complete System Integration Diagram
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ COMPLETE SYSTEM INTEGRATION DIAGRAM │
└─────────────────────────────────────────────────────────────────────────────┘
┌──────────────────────────────────────────────────────────────────────────────┐
│ PHASE 1: EXTERNAL SCRAPER (Python/Playwright) - ARCHITECTURE-TROOSTWIJK │
└──────────────────────────────────────────────────────────────────────────────┘
┌─────────────────────────────┼─────────────────────────────┐
▼ ▼ ▼
[Listing Pages] [Auction Pages] [Lot Pages]
/auctions?page=N /a/auction-id /l/lot-id
│ │ │
│ Extract URLs │ Parse __NEXT_DATA__ │ Parse __NEXT_DATA__
├────────────────────────────▶│ JSON (GraphQL) │ JSON (GraphQL)
│ │ │
│ ▼ ▼
│ ┌────────────────┐ ┌────────────────┐
│ │ INSERT auctions│ │ INSERT lots │
│ │ to SQLite │ │ INSERT images │
│ └────────────────┘ │ (URLs only) │
│ │ └────────────────┘
│ │ │
└─────────────────────────────┴────────────────────────────┘
┌──────────────────┐
│ SQLITE DATABASE │
│ output/cache.db │
└──────────────────┘
┌─────────────────┼─────────────────┐
▼ ▼ ▼
[auctions table] [lots table] [images table]
- auction_id - lot_id - id
- title - auction_id - lot_id
- location - title - url
- lots_count - current_bid - local_path
- closing_time - bid_count - downloaded=0
- closing_time
- followersCount ⭐ NEW
- estimatedMin ⭐ NEW
- estimatedMax ⭐ NEW
- nextBidStepInCents ⭐ NEW
- condition ⭐ NEW
- vat ⭐ NEW
- buyerPremiumPercentage ⭐ NEW
- quantity ⭐ NEW
- biddingStatus ⭐ NEW
- remarks ⭐ NEW
┌─────────────────────────────────────┴─────────────────────────────────────┐
│ PHASE 2: MONITORING & PROCESSING (Java) - THIS PROJECT │
└────────────────────────────────────────────────────────────────────────────┘
┌─────────────────┼─────────────────┐
▼ ▼ ▼
[TroostwijkMonitor] [DatabaseService] [ScraperDataAdapter]
│ │ │
│ Read lots │ Query lots │ Transform data
│ every hour │ Import images │ TEXT → INTEGER
│ │ │ "€123" → 123.0
└─────────────────┴─────────────────┘
┌─────────────────────────┼─────────────────────────┐
▼ ▼ ▼
[Bid Monitoring] [Image Processing] [Closing Alerts]
Check API every 1h Download images Check < 5 min
│ │ │
│ New bid? │ Process via │ Time critical?
├─[YES]──────────┐ │ ObjectDetection ├─[YES]────┐
│ │ │ │ │
▼ │ ▼ │ │
[Update current_bid] │ ┌──────────────────┐ │ │
in database │ │ YOLO Detection │ │ │
│ │ OpenCV DNN │ │ │
│ └──────────────────┘ │ │
│ │ │ │
│ │ Detect objects │ │
│ ├─[vehicle] │ │
│ ├─[furniture] │ │
│ ├─[machinery] │ │
│ │ │ │
│ ▼ │ │
│ [Save labels to DB] │ │
│ [Estimate value] │ │
│ │ │ │
│ │ │ │
└─────────┴───────────────────────┴──────────┘
┌───────────────────────────────────────────────┴────────────────────────────┐
│ PHASE 3: INTELLIGENCE LAYER ⭐ NEW - PREDICTIVE ANALYTICS │
└────────────────────────────────────────────────────────────────────────────┘
┌─────────────────┴─────────────────┐
▼ ▼
[Intelligence Engine] [Analytics Calculations]
│ │
┌───────────────────┼──────────────┐ │
▼ ▼ ▼ │
[Sleeper Detection] [Bargain Finder] [Popularity Tracker] │
High followers Price < estimate Watch count analysis │
Low current bid Opportunity Competition level │
│ │ │ │
│ │ │ │
└───────────────────┴──────────────┴───────────────────┘
┌─────────────────┴─────────────────┐
▼ ▼
[Total Cost Calculator] [Next Bid Calculator]
Current bid × (1 + VAT/100) Current bid + increment
× (1 + premium/100) (from API or calculated)
│ │
└─────────────────┬─────────────────┘
┌───────────────────────────────────────────────┴────────────────────────────┐
│ PHASE 4: NOTIFICATION SYSTEM - USER INTERACTION TRIGGERS │
└────────────────────────────────────────────────────────────────────────────┘
┌─────────────────┴─────────────────┐
▼ ▼
[NotificationService] [User Decision Points]
│ │
┌───────────────────┼───────────────────┐ │
▼ ▼ ▼ │
[Desktop Notify] [Email Notify] [Priority Level] │
Windows/macOS/ Gmail SMTP 0=Normal │
Linux system (FREE) 1=High │
tray │
│ │ │ │
└───────────────────┴───────────────────┘ │
│ │
▼ ▼
┌──────────────────┐ ┌──────────────────┐
│ USER INTERACTION │ │ TRIGGER EVENTS: │
│ NOTIFICATIONS │ │ │
└──────────────────┘ └──────────────────┘
│ │
┌───────────────────┼───────────────────┐ │
▼ ▼ ▼ │
┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ │
│ 1. BID CHANGE │ │ 2. OBJECT │ │ 3. CLOSING │ │
│ │ │ DETECTED │ │ ALERT │ │
│ "Nieuw bod op │ │ │ │ │ │
│ kavel 12345: │ │ "Lot contains: │ │ "Kavel 12345 │ │
│ €150 (was €125)"│ │ - Vehicle │ │ sluit binnen │ │
│ │ │ - Machinery │ │ 5 min." │ │
│ Priority: NORMAL │ │ Est: €5000" │ │ Priority: HIGH │ │
│ │ │ │ │ │ │
│ Action needed: │ │ Action needed: │ │ Action needed: │ │
│ ▸ Place bid? │ │ ▸ Review item? │ │ ▸ Place final │ │
│ ▸ Monitor? │ │ ▸ Confirm value? │ │ bid? │ │
│ ▸ Ignore? │ │ ▸ Add to watch? │ │ ▸ Let expire? │ │
└──────────────────┘ └──────────────────┘ └──────────────────┘ │
│ │ │ │
└───────────────────┴───────────────────┴─────────────┘
┌─────────────────────────────────────────────────────────────────────────────┐
│ INTELLIGENCE NOTIFICATIONS ⭐ NEW │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ 4. SLEEPER LOT ALERT │
│ "Lot 12345: 25 watchers, only €50 bid - Opportunity!" │
│ Action: ▸ Place strategic bid ▸ Monitor competition ▸ Set alert │
│ │
│ 5. BARGAIN DETECTED │
│ "Lot 67890: Current €200, Estimate €400-€600 - Below estimate!" │
│ Action: ▸ Bid now ▸ Research comparable ▸ Add to watchlist │
│ │
│ 6. HIGH COMPETITION WARNING │
│ "Lot 11111: 75 watchers, bid velocity 5/hr - Strong competition" │
│ Action: ▸ Review strategy ▸ Set max bid ▸ Find alternatives │
│ │
│ 7. TOTAL COST NOTIFICATION │
│ "True cost: €500 bid + €105 VAT (21%) + €50 premium (10%) = €655" │
│ Action: ▸ Confirm budget ▸ Adjust bid ▸ Calculate logistics │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
## Intelligence Dashboard Flow
```mermaid
flowchart TD
subgraph P1["PHASE 1: DATA COLLECTION"]
A1[GraphQL API] --> A2[Scraper Extracts 15+ New Fields]
A2 --> A3[followersCount]
A2 --> A4[estimatedMin/Max]
A2 --> A5[nextBidStepInCents]
A2 --> A6[vat + buyerPremiumPercentage]
A2 --> A7[condition + biddingStatus]
A3 & A4 & A5 & A6 & A7 --> DB[(SQLite Database)]
end
DB --> P2_Entry
subgraph P2["PHASE 2: INTELLIGENCE PROCESSING"]
P2_Entry[Lot.java Model] --> Intelligence[Intelligence Methods]
Intelligence --> Sleeper[isSleeperLot<br/>High followers, low bid]
Intelligence --> Bargain[isBelowEstimate<br/>Price < estimate]
Intelligence --> Popular[getPopularityLevel<br/>Watch count tiers]
Intelligence --> Cost[calculateTotalCost<br/>Bid + VAT + Premium]
Intelligence --> NextBid[calculateNextBid<br/>API increment]
end
P2_Entry --> API_Layer
subgraph API["PHASE 3: REST API ENDPOINTS"]
API_Layer[AuctionMonitorResource] --> E1[/intelligence/sleepers]
API_Layer --> E2[/intelligence/bargains]
API_Layer --> E3[/intelligence/popular]
API_Layer --> E4[/intelligence/price-analysis]
API_Layer --> E5[/lots/:id/intelligence]
API_Layer --> E6[/charts/watch-distribution]
end
E1 & E2 & E3 & E4 & E5 & E6 --> Dashboard
subgraph UI["PHASE 4: INTELLIGENCE DASHBOARD"]
Dashboard[index.html] --> Widget1[Sleeper Lots Widget<br/>Opportunities]
Dashboard --> Widget2[Bargain Lots Widget<br/>Below Estimate]
Dashboard --> Widget3[Popular Lots Widget<br/>High Competition]
Dashboard --> Table[Enhanced Table<br/>Watchers | Est. Range | Total Cost]
Table --> Badges[Smart Badges:<br/>DEAL | Watch Count | Time Left]
end
Widget1 --> UserAction
Widget2 --> UserAction
Widget3 --> UserAction
Table --> UserAction
subgraph Actions["PHASE 5: USER ACTIONS"]
UserAction[User Decision] --> Bid[Place Strategic Bid]
UserAction --> Monitor[Add to Watchlist]
UserAction --> Research[Research Comparables]
UserAction --> Calculate[Budget Calculator]
end
```
## Key Intelligence Features
### 1. Follower/Watch Count Analytics
- **Data Source**: `followersCount` from GraphQL API
- **Intelligence Value**:
- Predict lot popularity before bidding wars
- Calculate interest-to-bid conversion rates
- Identify "sleeper" lots (high followers, low bids)
- Alert on sudden interest spikes
### 2. Price vs Estimate Analysis
- **Data Source**: `estimatedMin`, `estimatedMax` from GraphQL API
- **Intelligence Value**:
- Identify bargains: `currentBid < estimatedMin`
- Identify overvalued: `currentBid > estimatedMax`
- Build pricing models per category
- Track auction house estimate accuracy
### 3. True Cost Calculator
- **Data Source**: `vat`, `buyerPremiumPercentage` from GraphQL API
- **Intelligence Value**:
- Calculate total cost: `bid × (1 + VAT/100) × (1 + premium/100)`
- Budget planning with accurate all-in costs
- Compare true costs across lots
- Prevent bidding surprises
### 4. Exact Bid Increment
- **Data Source**: `nextBidStepInCents` from GraphQL API
- **Intelligence Value**:
- Show exact next bid amount
- No calculation errors
- Better UX for bidding recommendations
- Strategic bid placement
### 5. Structured Location & Category
- **Data Source**: `cityLocation`, `countryCode`, `categoryPath` from GraphQL API
- **Intelligence Value**:
- Filter by distance from user
- Calculate pickup logistics costs
- Category-based analytics
- Regional pricing trends
## Integration Hooks & Timing
| Event | Frequency | Trigger | Notification Type | User Action Required |
|--------------------------------|-------------------|----------------------------|----------------------------|------------------------|
| **Sleeper lot detected** | On data refresh | followers > 10, bid < €100 | Desktop + Email | Review opportunity |
| **Bargain detected** | On data refresh | bid < estimatedMin | Desktop + Email | Consider bidding |
| **High competition** | On data refresh | followers > 50 | Desktop | Review strategy |
| **Bid change detected** | Every 1 hour | Monitor detects higher bid | Desktop + Email | Place counter-bid? |
| **Closing soon (< 30 min)** | When detected | Time-based check | Desktop + Email | Review lot |
| **Closing imminent (< 5 min)** | When detected | Time-based check | Desktop + Email (HIGH) | Final bid decision |
| **Object detected** | On image process | YOLO finds objects | Desktop + Email | Confirm identification |
| **True cost calculated** | On page load | User views lot | Dashboard display | Budget confirmation |
## API Endpoints Reference
### Intelligence Endpoints
- `GET /api/monitor/intelligence/sleepers` - Returns high-interest, low-bid lots
- `GET /api/monitor/intelligence/bargains` - Returns lots priced below estimate
- `GET /api/monitor/intelligence/popular?level={HIGH|MEDIUM|LOW}` - Returns lots by popularity
- `GET /api/monitor/intelligence/price-analysis` - Returns price vs estimate statistics
- `GET /api/monitor/lots/{lotId}/intelligence` - Returns detailed intelligence for specific lot
### Chart Endpoints
- `GET /api/monitor/charts/watch-distribution` - Returns follower count distribution
- `GET /api/monitor/charts/country-distribution` - Returns geographic distribution
- `GET /api/monitor/charts/category-distribution` - Returns category distribution
- `GET /api/monitor/charts/bidding-trend?hours=24` - Returns time series data
## Dashboard Intelligence Widgets
### Sleeper Lots Widget
- **Color**: Purple gradient
- **Icon**: Eye (fa-eye)
- **Metric**: Count of lots with followers > 10 and bid < €100
- **Action**: Click to filter table to sleeper lots only
### Bargain Lots Widget
- **Color**: Green gradient
- **Icon**: Tag (fa-tag)
- **Metric**: Count of lots where current bid < estimated minimum
- **Action**: Click to filter table to bargain lots only
### Popular/Hot Lots Widget
- **Color**: Orange gradient
- **Icon**: Fire (fa-fire)
- **Metric**: Count of lots with followers > 20
- **Action**: Click to filter table to popular lots only
## Enhanced Table Features
### New Columns
1. **Watchers** - Shows follower count with color-coded badges:
- 50+ followers: Red (high competition)
- 21-50 followers: Orange (medium competition)
- 6-20 followers: Blue (some interest)
- 0-5 followers: Gray (minimal interest)
2. **Est. Range** - Shows auction house estimate: `€min-€max`
- Displays "DEAL" badge if current bid < estimate
3. **Total Cost** - Shows true cost including VAT and buyer premium:
- Hover tooltip shows breakdown: `Including VAT (21%) + Premium (10%)`
### Smart Indicators
- **DEAL Badge**: Green badge when `currentBid < estimatedMin`
- **Watch Count Badge**: Color-coded by competition level
- **Urgency Badge**: Time-based coloring (< 10 min = red)
## Technical Implementation
### Backend (Java)
- **File**: `src/main/java/auctiora/Lot.java`
- Added 24 new fields from GraphQL API
- Added 9 intelligence calculation methods
- Immutable record with Lombok `@With` annotation
- **File**: `src/main/java/auctiora/AuctionMonitorResource.java`
- Added 6 new REST API endpoints
- Enhanced insights with sleeper/bargain/popular detection
- Added watch distribution chart endpoint
### Frontend (HTML/JavaScript)
- **File**: `src/main/resources/META-INF/resources/index.html`
- Added 3 intelligence widgets with click handlers
- Enhanced closing soon table with 3 new columns
- Added `fetchIntelligenceData()` function
- Added smart badges and color coding
- Added total cost calculator display
## Future Enhancements
1. **Bid History Table** - Track bid changes over time
2. **Comparative Analytics** - Compare similar lots across auctions
3. **Machine Learning** - Predict final hammer price based on patterns
4. **Geographic Filtering** - Distance-based sorting and filtering
5. **Email Alerts** - Custom alerts for sleepers, bargains, etc.
6. **Mobile App** - Push notifications for time-critical events
7. **Bid Automation** - Auto-bid up to maximum with increment logic
---
**Last Updated**: December 2025
**Version**: 2.1
**Author**: Auctiora Intelligence Team

650
docs/QUARKUS_GUIDE.md Normal file
View File

@@ -0,0 +1,650 @@
# Quarkus Auction Monitor - Complete Guide
## 🚀 Overview
The Troostwijk Auction Monitor now runs on **Quarkus**, a Kubernetes-native Java framework optimized for fast startup and low memory footprint.
### Key Features
**Quarkus Scheduler** - Built-in cron-based scheduling
**REST API** - Control and monitor via HTTP endpoints
**Health Checks** - Kubernetes-ready liveness/readiness probes
**CDI/Dependency Injection** - Type-safe service management
**Fast Startup** - 0.5s startup time
**Low Memory** - ~50MB RSS memory footprint
**Hot Reload** - Development mode with live coding
---
## 📦 Quick Start
### Option 1: Run with Maven (Development)
```bash
# Start in dev mode with live reload
mvn quarkus:dev
# Access application
# API: http://localhost:8081/api/monitor/status
# Health: http://localhost:8081/health
```
### Option 2: Build and Run JAR
```bash
# Build
mvn clean package
# Run
java -jar target/quarkus-app/quarkus-run.jar
# Or use fast-jar (recommended for production)
mvn clean package -Dquarkus.package.jar.type=fast-jar
java -jar target/quarkus-app/quarkus-run.jar
```
### Option 3: Docker
```bash
# Build image
docker build -t auction-monitor:latest .
# Run container
docker run -p 8081:8081 \
-v $(pwd)/data:/mnt/okcomputer/output \
auction-monitor:latest
```
### Option 4: Docker Compose (Recommended)
```bash
# Start services
docker-compose up -d
# View logs
docker-compose logs -f
# Stop services
docker-compose down
```
---
## 🔧 Configuration
### application.properties
All configuration is in `src/main/resources/application.properties`:
```properties
# Database
auction.database.path=C:\\mnt\\okcomputer\\output\\cache.db
auction.images.path=C:\\mnt\\okcomputer\\output\\images
# Notifications
auction.notification.config=desktop
# Or for email: smtp:your@gmail.com:app_password:recipient@example.com
# YOLO Models (optional)
auction.yolo.config=models/yolov4.cfg
auction.yolo.weights=models/yolov4.weights
auction.yolo.classes=models/coco.names
# Workflow Schedules (cron expressions)
auction.workflow.scraper-import.cron=0 */30 * * * ? # Every 30 min
auction.workflow.image-processing.cron=0 0 * * * ? # Every 1 hour
auction.workflow.bid-monitoring.cron=0 */15 * * * ? # Every 15 min
auction.workflow.closing-alerts.cron=0 */5 * * * ? # Every 5 min
# HTTP Server
quarkus.http.port=8081
quarkus.http.host=0.0.0.0
```
### Environment Variables
Override configuration with environment variables:
```bash
export AUCTION_DATABASE_PATH=/path/to/cache.db
export AUCTION_NOTIFICATION_CONFIG=desktop
export QUARKUS_HTTP_PORT=8081
```
---
## 📅 Scheduled Workflows
Quarkus automatically runs these workflows based on cron expressions:
| Workflow | Schedule | Cron Expression | Description |
|----------|----------|-----------------|-------------|
| **Scraper Import** | Every 30 min | `0 */30 * * * ?` | Import auctions/lots from external scraper |
| **Image Processing** | Every 1 hour | `0 0 * * * ?` | Download images & run object detection |
| **Bid Monitoring** | Every 15 min | `0 */15 * * * ?` | Check for bid changes |
| **Closing Alerts** | Every 5 min | `0 */5 * * * ?` | Send alerts for lots closing soon |
### Cron Expression Format
```
┌───────────── second (0-59)
│ ┌───────────── minute (0-59)
│ │ ┌───────────── hour (0-23)
│ │ │ ┌───────────── day of month (1-31)
│ │ │ │ ┌───────────── month (1-12)
│ │ │ │ │ ┌───────────── day of week (0-6, Sunday=0)
│ │ │ │ │ │
0 */30 * * * ? = Every 30 minutes
0 0 * * * ? = Every hour at minute 0
0 0 0 * * ? = Every day at midnight
```
---
## 🌐 REST API
### Base URL
```
http://localhost:8081/api/monitor
```
### Endpoints
#### 1. Get Status
```bash
GET /api/monitor/status
# Example
curl http://localhost:8081/api/monitor/status
# Response
{
"running": true,
"auctions": 25,
"lots": 150,
"images": 300,
"closingSoon": 5
}
```
#### 2. Get Statistics
```bash
GET /api/monitor/statistics
# Example
curl http://localhost:8081/api/monitor/statistics
# Response
{
"totalAuctions": 25,
"totalLots": 150,
"totalImages": 300,
"activeLots": 120,
"lotsWithBids": 80,
"totalBidValue": "€125,450.00",
"averageBid": "€1,568.13"
}
```
#### 3. Trigger Workflows Manually
```bash
# Scraper Import
POST /api/monitor/trigger/scraper-import
curl -X POST http://localhost:8081/api/monitor/trigger/scraper-import
# Image Processing
POST /api/monitor/trigger/image-processing
curl -X POST http://localhost:8081/api/monitor/trigger/image-processing
# Bid Monitoring
POST /api/monitor/trigger/bid-monitoring
curl -X POST http://localhost:8081/api/monitor/trigger/bid-monitoring
# Closing Alerts
POST /api/monitor/trigger/closing-alerts
curl -X POST http://localhost:8081/api/monitor/trigger/closing-alerts
```
#### 4. Get Auctions
```bash
# All auctions
GET /api/monitor/auctions
curl http://localhost:8081/api/monitor/auctions
# Filter by country
GET /api/monitor/auctions?country=NL
curl http://localhost:8081/api/monitor/auctions?country=NL
```
#### 5. Get Lots
```bash
# Active lots
GET /api/monitor/lots
curl http://localhost:8081/api/monitor/lots
# Lots closing soon (within 30 minutes by default)
GET /api/monitor/lots/closing-soon
curl http://localhost:8081/api/monitor/lots/closing-soon
# Custom minutes threshold
GET /api/monitor/lots/closing-soon?minutes=60
curl http://localhost:8081/api/monitor/lots/closing-soon?minutes=60
```
#### 6. Get Lot Images
```bash
GET /api/monitor/lots/{lotId}/images
# Example
curl http://localhost:8081/api/monitor/lots/12345/images
```
#### 7. Test Notification
```bash
POST /api/monitor/test-notification
Content-Type: application/json
{
"message": "Test message",
"title": "Test Title",
"priority": "0"
}
# Example
curl -X POST http://localhost:8081/api/monitor/test-notification \
-H "Content-Type: application/json" \
-d '{"message":"Test notification","title":"Test","priority":"0"}'
```
---
## 🏥 Health Checks
Quarkus provides built-in health checks for Kubernetes/Docker:
### Liveness Probe
```bash
GET /health/live
# Example
curl http://localhost:8081/health/live
# Response
{
"status": "UP",
"checks": [
{
"name": "Auction Monitor is alive",
"status": "UP"
}
]
}
```
### Readiness Probe
```bash
GET /health/ready
# Example
curl http://localhost:8081/health/ready
# Response
{
"status": "UP",
"checks": [
{
"name": "database",
"status": "UP",
"data": {
"auctions": 25
}
}
]
}
```
### Startup Probe
```bash
GET /health/started
# Example
curl http://localhost:8081/health/started
```
### Combined Health
```bash
GET /health
# Returns all health checks
curl http://localhost:8081/health
```
---
## 🐳 Docker Deployment
### Build Image
```bash
docker build -t auction-monitor:1.0 .
```
### Run Container
```bash
docker run -d \
--name auction-monitor \
-p 8081:8081 \
-v $(pwd)/data:/mnt/okcomputer/output \
-e AUCTION_NOTIFICATION_CONFIG=desktop \
auction-monitor:1.0
```
### Docker Compose
```yaml
version: '3.8'
services:
auction-monitor:
image: auction-monitor:1.0
ports:
- "8081:8081"
volumes:
- ./data:/mnt/okcomputer/output
environment:
- AUCTION_DATABASE_PATH=/mnt/okcomputer/output/cache.db
- AUCTION_NOTIFICATION_CONFIG=desktop
healthcheck:
test: ["CMD", "wget", "--spider", "http://localhost:8081/health/live"]
interval: 30s
timeout: 3s
retries: 3
```
---
## ☸️ Kubernetes Deployment
### deployment.yaml
```yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: auction-monitor
spec:
replicas: 1
selector:
matchLabels:
app: auction-monitor
template:
metadata:
labels:
app: auction-monitor
spec:
containers:
- name: auction-monitor
image: auction-monitor:1.0
ports:
- containerPort: 8081
env:
- name: AUCTION_DATABASE_PATH
value: /data/cache.db
- name: QUARKUS_HTTP_PORT
value: "8081"
volumeMounts:
- name: data
mountPath: /mnt/okcomputer/output
livenessProbe:
httpGet:
path: /health/live
port: 8081
initialDelaySeconds: 10
periodSeconds: 30
readinessProbe:
httpGet:
path: /health/ready
port: 8081
initialDelaySeconds: 5
periodSeconds: 10
startupProbe:
httpGet:
path: /health/started
port: 8081
failureThreshold: 30
periodSeconds: 10
volumes:
- name: data
persistentVolumeClaim:
claimName: auction-data-pvc
---
apiVersion: v1
kind: Service
metadata:
name: auction-monitor
spec:
selector:
app: auction-monitor
ports:
- port: 8081
targetPort: 8081
type: LoadBalancer
```
---
## 🔄 Development Mode
Quarkus dev mode provides live reload for rapid development:
```bash
# Start dev mode
mvn quarkus:dev
# Features available:
# - Live reload (no restart needed)
# - Dev UI: http://localhost:8081/q/dev/
# - Continuous testing
# - Debug on port 5005
```
### Dev UI
Access at: `http://localhost:8081/q/dev/`
Features:
- Configuration editor
- Scheduler dashboard
- Health checks
- REST endpoints explorer
- Continuous testing
---
## 🧪 Testing
### Run All Tests
```bash
mvn test
```
### Run Quarkus Tests
```bash
mvn test -Dtest=*QuarkusTest
```
### Integration Test with Running Application
```bash
# Terminal 1: Start application
mvn quarkus:dev
# Terminal 2: Run integration tests
curl http://localhost:8081/api/monitor/status
curl http://localhost:8081/health/live
curl -X POST http://localhost:8081/api/monitor/trigger/scraper-import
```
---
## 📊 Monitoring & Logging
### View Logs
```bash
# Docker
docker logs -f auction-monitor
# Docker Compose
docker-compose logs -f
# Kubernetes
kubectl logs -f deployment/auction-monitor
```
### Log Levels
Configure in `application.properties`:
```properties
# Production
quarkus.log.console.level=INFO
# Development
%dev.quarkus.log.console.level=DEBUG
# Specific logger
quarkus.log.category."com.auction".level=DEBUG
```
### Scheduled Job Logs
```
14:30:00 INFO [com.auc.Qua] (executor-thread-1) 📥 [WORKFLOW 1] Importing scraper data...
14:30:00 INFO [com.auc.Qua] (executor-thread-1) → Imported 5 auctions
14:30:00 INFO [com.auc.Qua] (executor-thread-1) → Imported 25 lots
14:30:00 INFO [com.auc.Qua] (executor-thread-1) ✓ Scraper import completed in 1250ms
```
---
## ⚙️ Performance
### Startup Time
- **JVM Mode**: ~0.5 seconds
- **Native Image**: ~0.014 seconds
### Memory Footprint
- **JVM Mode**: ~50MB RSS
- **Native Image**: ~15MB RSS
### Build Native Image (Optional)
```bash
# Requires GraalVM
mvn package -Pnative
# Run native executable
./target/troostwijk-scraper-1.0-SNAPSHOT-runner
```
---
## 🔐 Security
### Environment Variables for Secrets
```bash
# Don't commit credentials!
export AUCTION_NOTIFICATION_CONFIG=smtp:user@gmail.com:SECRET_PASSWORD:recipient@example.com
# Or use Kubernetes secrets
kubectl create secret generic auction-secrets \
--from-literal=notification-config='smtp:user@gmail.com:password:recipient@example.com'
```
### Kubernetes Secret
```yaml
apiVersion: v1
kind: Secret
metadata:
name: auction-secrets
type: Opaque
stringData:
notification-config: smtp:user@gmail.com:app_password:recipient@example.com
```
---
## 🛠️ Troubleshooting
### Issue: Schedulers not running
**Check scheduler status:**
```bash
curl http://localhost:8081/health/ready
```
**Enable debug logging:**
```properties
quarkus.log.category."io.quarkus.scheduler".level=DEBUG
```
### Issue: Database not found
**Check file permissions:**
```bash
ls -la C:/mnt/okcomputer/output/cache.db
```
**Create directory:**
```bash
mkdir -p C:/mnt/okcomputer/output
```
### Issue: Port 8081 already in use
**Change port:**
```bash
mvn quarkus:dev -Dquarkus.http.port=8082
# Or
export QUARKUS_HTTP_PORT=8082
```
### Issue: Health check failing
**Check application logs:**
```bash
docker logs auction-monitor
```
**Verify database connection:**
```bash
curl http://localhost:8081/health/ready
```
---
## 📚 Additional Resources
- [Quarkus Official Guide](https://quarkus.io/guides/)
- [Quarkus Scheduler](https://quarkus.io/guides/scheduler)
- [Quarkus REST](https://quarkus.io/guides/rest)
- [Quarkus Health](https://quarkus.io/guides/smallrye-health)
- [Quarkus Docker](https://quarkus.io/guides/container-image)
---
## Summary
**Quarkus Framework** integrated for modern Java development
**CDI/Dependency Injection** for clean architecture
**@Scheduled** annotations for cron-based workflows
**REST API** for control and monitoring
**Health Checks** for Kubernetes/Docker
**Fast Startup** and low memory footprint
**Docker/Kubernetes** ready
**Production** optimized
**Run and enjoy! 🎉**

209
docs/RATE_LIMITING.md Normal file
View File

@@ -0,0 +1,209 @@
# HTTP Rate Limiting
## Overview
The Troostwijk Scraper implements **per-host HTTP rate limiting** to prevent overloading external services (especially Troostwijk APIs) and avoid getting blocked.
## Features
-**Per-host rate limiting** - Different limits for different hosts
-**Token bucket algorithm** - Allows burst traffic while maintaining steady rate
-**Automatic host detection** - Extracts host from URL automatically
-**Request statistics** - Tracks success/failure/rate-limited requests
-**Thread-safe** - Uses semaphores for concurrent request handling
-**Configurable** - Via `application.properties`
## Configuration
Edit `src/main/resources/application.properties`:
```properties
# Default rate limit for all hosts (requests per second)
auction.http.rate-limit.default-max-rps=2
# Troostwijk-specific rate limit (requests per second)
auction.http.rate-limit.troostwijk-max-rps=1
# HTTP request timeout (seconds)
auction.http.timeout-seconds=30
```
### Recommended Settings
| Service | Max RPS | Reason |
|---------|---------|--------|
| `troostwijkauctions.com` | **1 req/s** | Prevent blocking by Troostwijk |
| Other image hosts | **2 req/s** | Balance speed and politeness |
## Usage
The `RateLimitedHttpClient` is automatically injected into services that make HTTP requests:
```java
@Inject
RateLimitedHttpClient httpClient;
// GET request for text
HttpResponse<String> response = httpClient.sendGet(url);
// GET request for binary data (images)
HttpResponse<byte[]> response = httpClient.sendGetBytes(imageUrl);
```
### Integrated Services
1. **TroostwijkMonitor** - API calls for bid monitoring
2. **ImageProcessingService** - Image downloads
3. **QuarkusWorkflowScheduler** - Scheduled workflows
## Monitoring
### REST API Endpoints
#### Get All Rate Limit Statistics
```bash
GET http://localhost:8081/api/monitor/rate-limit/stats
```
Response:
```json
{
"hosts": 2,
"statistics": {
"api.troostwijkauctions.com": {
"totalRequests": 150,
"successfulRequests": 148,
"failedRequests": 1,
"rateLimitedRequests": 0,
"averageDurationMs": 245
},
"images.troostwijkauctions.com": {
"totalRequests": 320,
"successfulRequests": 315,
"failedRequests": 5,
"rateLimitedRequests": 2,
"averageDurationMs": 892
}
}
}
```
#### Get Statistics for Specific Host
```bash
GET http://localhost:8081/api/monitor/rate-limit/stats/api.troostwijkauctions.com
```
Response:
```json
{
"host": "api.troostwijkauctions.com",
"totalRequests": 150,
"successfulRequests": 148,
"failedRequests": 1,
"rateLimitedRequests": 0,
"averageDurationMs": 245
}
```
## How It Works
### Token Bucket Algorithm
1. **Bucket initialization** - Starts with `maxRequestsPerSecond` tokens
2. **Request consumption** - Each request consumes 1 token
3. **Token refill** - Bucket refills every second
4. **Blocking** - If no tokens available, request waits
### Per-Host Rate Limiting
The client automatically:
1. Extracts hostname from URL (e.g., `api.troostwijkauctions.com`)
2. Creates/retrieves rate limiter for that host
3. Applies configured limit (Troostwijk-specific or default)
4. Tracks statistics per host
### Request Flow
```
Request → Extract Host → Get Rate Limiter → Acquire Token → Send Request → Record Stats
troostwijkauctions.com?
Yes: 1 req/s | No: 2 req/s
```
## Warning Signs
Monitor for these indicators of rate limiting issues:
| Metric | Warning Threshold | Action |
|--------|------------------|--------|
| `rateLimitedRequests` | > 0 | Server is rate limiting you - reduce `max-rps` |
| `failedRequests` | > 5% | Investigate connection issues or increase timeout |
| `averageDurationMs` | > 3000ms | Server may be slow - reduce load |
## Testing
### Manual Test via cURL
```bash
# Test Troostwijk API rate limiting
for i in {1..10}; do
echo "Request $i at $(date +%T)"
curl -s http://localhost:8081/api/monitor/status > /dev/null
sleep 0.5
done
# Check statistics
curl http://localhost:8081/api/monitor/rate-limit/stats | jq
```
### Check Logs
Rate limiting is logged at DEBUG level:
```
03:15:23 DEBUG [RateLimitedHttpClient] HTTP 200 GET api.troostwijkauctions.com (245ms)
03:15:24 DEBUG [RateLimitedHttpClient] HTTP 200 GET api.troostwijkauctions.com (251ms)
03:15:25 WARN [RateLimitedHttpClient] ⚠️ Rate limited by api.troostwijkauctions.com (HTTP 429)
```
## Troubleshooting
### Problem: Getting HTTP 429 (Too Many Requests)
**Solution:** Decrease `max-rps` for that host:
```properties
auction.http.rate-limit.troostwijk-max-rps=0.5
```
### Problem: Requests too slow
**Solution:** Increase `max-rps` (be careful not to get blocked):
```properties
auction.http.rate-limit.default-max-rps=3
```
### Problem: Requests timing out
**Solution:** Increase timeout:
```properties
auction.http.timeout-seconds=60
```
## Best Practices
1. **Start conservative** - Begin with low limits (1 req/s)
2. **Monitor statistics** - Watch `rateLimitedRequests` metric
3. **Respect robots.txt** - Check host's crawling policy
4. **Use off-peak hours** - Run heavy scraping during low-traffic times
5. **Implement exponential backoff** - If receiving 429s, wait longer between retries
## Future Enhancements
Potential improvements:
- [ ] Dynamic rate adjustment based on 429 responses
- [ ] Exponential backoff on failures
- [ ] Per-endpoint rate limiting (not just per-host)
- [ ] Request queue visualization
- [ ] Integration with external rate limit APIs (e.g., Redis)

304
docs/VALUATION.md Normal file
View File

@@ -0,0 +1,304 @@
# Auction Valuation Mathematics - Technical Reference
## 1. Fair Market Value (FMV) - Core Valuation Formula
The baseline valuation is calculated using a **weighted comparable sales approach**:
$$
FMV = \frac{\sum_{i=1}^{n} \left( P_i \cdot \omega_c \cdot \omega_t \cdot \omega_p \cdot \omega_h \right)}{\sum_{i=1}^{n} \left( \omega_c \cdot \omega_t \cdot \omega_p \cdot \omega_h \right)}
$$
**Variables:**
- $P_i$ = Final hammer price of comparable lot *i* (€)
- $\omega_c$ = **Condition weight**: $\exp(-\lambda_c \cdot |C_{target} - C_i|)$
- $\omega_t$ = **Time weight**: $\exp(-\lambda_t \cdot |T_{target} - T_i|)$
- $\omega_p$ = **Provenance weight**: $1 + \delta_p \cdot (P_{target} - P_i)$
- $\omega_h$ = **Historical weight**: $\left( \frac{1}{1 + e^{-kh \cdot (D_i - D_{median})}} \right)$
**Parameter Definitions:**
- $C \in [0, 10]$ = Condition score (10 = perfect)
- $T$ = Manufacturing year
- $P \in \{0,1\}$ = Provenance flag (1 = documented history)
- $D_i$ = Days since comparable sale
- $\lambda_c = 0.693$ = Condition decay constant (50% weight at 1-point difference)
- $\lambda_t = 0.048$ = Time decay constant (50% weight at 15-year difference)
- $\delta_p = 0.15$ = Provenance premium coefficient
- $kh = 0.01$ = Historical relevance coefficient
---
## 2. Condition Adjustment Multiplier
Normalizes prices across condition states:
$$
M_{cond} = \exp\left( \alpha_c \cdot \sqrt{C_{target}} - \beta_c \right)
$$
**Variables:**
- $\alpha_c = 0.15$ = Condition sensitivity parameter
- $\beta_c = 0.40$ = Baseline condition offset
- $C_{target}$ = Target lot condition score
**Interpretation:**
- $C = 10$ (mint): $M_{cond} = 1.48$ (48% premium over poor condition)
- $C = 5$ (average): $M_{cond} = 0.91$
---
## 3. Time-Based Depreciation Model
For equipment/machinery with measurable lifespan:
$$
V_{age} = V_{new} \cdot \left( 1 - \gamma \cdot \ln\left( 1 + \frac{Y_{current} - Y_{manu}}{Y_{expected}} \right) \right)
$$
**Variables:**
- $V_{new}$ = Original market value (€)
- $\gamma = 0.25$ = Depreciation aggressivity factor
- $Y_{current}$ = Current year
- $Y_{manu}$ = Manufacturing year
- $Y_{expected}$ = Expected useful life span (years)
**Example:** 10-year-old machinery with 25-year expected life retains 85% of value.
---
## 4. Provenance Premium Calculation
$$
\Delta_{prov} = V_{base} \cdot \left( \eta_0 + \eta_1 \cdot \ln(1 + N_{docs}) \right)
$$
**Variables:**
- $V_{base}$ = Base valuation without provenance (€)
- $N_{docs}$ = Number of verifiable provenance documents
- $\eta_0 = 0.08$ = Base provenance premium (8%)
- $\eta_1 = 0.035$ = Marginal document premium coefficient
---
## 5. Undervaluation Detection Score
Critical for identifying mispriced opportunities:
$$
U_{score} = \frac{FMV - P_{current}}{FMV} \cdot \sigma_{market} \cdot \left( 1 + \frac{B_{velocity}}{B_{threshold}} \right) \cdot \ln\left( 1 + \frac{W_{watch}}{W_{bid}} \right)
$$
**Variables:**
- $P_{current}$ = Current bid price (€)
- $\sigma_{market} \in [0,1]$ = Market volatility factor (from indices)
- $B_{velocity}$ = Bids per hour (bph)
- $B_{threshold} = 10$ bph = High-velocity threshold
- $W_{watch}$ = Watch count
- $W_{bid}$ = Bid count
**Trigger condition:** $U_{score} &gt; 0.25$ (25% undervaluation) with confidence &gt; 0.70
---
## 6. Bid Velocity Indicator (Competition Heat)
Measures real-time competitive intensity:
$$
\Lambda_b(t) = \frac{dB}{dt} \cdot \exp\left( -\lambda_{cool} \cdot (t - t_{last}) \right)
$$
**Variables:**
- $\frac{dB}{dt}$ = Bid frequency derivative (bids/minute)
- $\lambda_{cool} = 0.1$ = Cool-down decay constant
- $t_{last}$ = Timestamp of last bid (minutes)
**Interpretation:**
- $\Lambda_b &gt; 5$ = **Hot lot** (bidding war likely)
- $\Lambda_b &lt; 0.5$ = **Cold lot** (potential sleeper)
---
## 7. Final Price Prediction Model
Composite machine learning-style formula:
$$
\hat{P}_{final} = FMV \cdot \left( 1 + \epsilon_{bid} + \epsilon_{time} + \epsilon_{comp} \right)
$$
**Error Components:**
- **Bid momentum error**:
$$\epsilon_{bid} = \tanh\left( \phi_1 \cdot \Lambda_b - \phi_2 \cdot \frac{P_{current}}{FMV} \right)$$
- **Time-to-close error**:
$$\epsilon_{time} = \psi \cdot \exp\left( -\frac{t_{close}}{30} \right)$$
- **Competition error**:
$$\epsilon_{comp} = \rho \cdot \ln\left( 1 + \frac{W_{watch}}{50} \right)$$
**Parameters:**
- $\phi_1 = 0.15$, $\phi_2 = 0.10$ = Bid momentum coefficients
- $\psi = 0.20$ = Time pressure coefficient
- $\rho = 0.08$ = Competition coefficient
- $t_{close}$ = Minutes until close
**Confidence interval**:
$$
CI_{95\%} = \hat{P}_{final} \pm 1.96 \cdot \sigma_{residual}
$$
---
## 8. Bidding Strategy Recommendation Engine
Optimal max bid and timing:
$$
S_{max} =
\begin{cases}
FMV \cdot (1 - \theta_{agg}) & \text{if } U_{score} &gt; 0.20 \\
FMV \cdot (1 + \theta_{cons}) & \text{if } \Lambda_b &gt; 3 \\
\hat{P}_{final} - \delta_{margin} & \text{otherwise}
\end{cases}
$$
**Variables:**
- $\theta_{agg} = 0.10$ = Aggressive buyer discount target (10% below FMV)
- $\theta_{cons} = 0.05$ = Conservative buyer overbid tolerance
- $\delta_{margin} = €50$ = Minimum margin below predicted final
**Timing function**:
$$
t_{optimal} = t_{close} - \begin{cases}
5 \text{ min} & \text{if } \Lambda_b &lt; 1 \\
30 \text{ sec} & \text{if } \Lambda_b &gt; 5 \\
10 \text{ min} & \text{otherwise}
\end{cases}
$$
---
## Variable Reference Table
| Symbol | Variable | Unit | Data Source |
|--------|----------|------|-------------|
| $P_i$ | Comparable sale price | € | `bid_history.final` |
| $C$ | Condition score | [0,10] | Image analysis + text parsing |
| $T$ | Manufacturing year | Year | Lot description extraction |
| $W_{watch}$ | Number of watchers | Count | Page metadata |
| $\Lambda_b$ | Bid velocity | bids/min | `bid_history.timestamp` diff |
| $t_{close}$ | Time until close | Minutes | `lots.closing_time` - NOW() |
| $\sigma_{market}$ | Market volatility | [0,1] | `market_indices.price_change_30d` |
| $N_{docs}$ | Provenance documents | Count | PDF link analysis |
| $B_{velocity}$ | Bid acceleration | bph² | Second derivative of $\Lambda_b$ |
---
## Backend Implementation (Quarkus Pseudo-Code)
```java
@Inject
MLModelService mlModel;
public Valuation calculateFairMarketValue(Lot lot) {
List&lt;Comparable&gt; comparables = db.findComparables(lot, minSimilarity=0.75, limit=20);
double weightedSum = 0.0;
double weightSum = 0.0;
for (Comparable comp : comparables) {
double wc = Math.exp(-0.693 * Math.abs(lot.getConditionScore() - comp.getConditionScore()));
double wt = Math.exp(-0.048 * Math.abs(lot.getYear() - comp.getYear()));
double wp = 1 + 0.15 * (lot.hasProvenance() ? 1 : 0 - comp.hasProvenance() ? 1 : 0);
double weight = wc * wt * wp;
weightedSum += comp.getFinalPrice() * weight;
weightSum += weight;
}
double fm v = weightSum &gt; 0 ? weightedSum / weightSum : lot.getEstimatedMin();
// Apply condition multiplier
fm v *= Math.exp(0.15 * Math.sqrt(lot.getConditionScore()) - 0.40);
return new Valuation(fm v, calculateConfidence(comparables.size()));
}
public BiddingStrategy getBiddingStrategy(String lotId) {
var lot = db.getLot(lotId);
var bidHistory = db.getBidHistory(lotId);
var watchers = lot.getWatchCount();
// Analyze patterns
boolean isSnipeTarget = watchers &gt; 50 && bidHistory.size() &lt; 5;
boolean hasReserve = lot.getReservePrice() &gt; 0;
double bidVelocity = calculateBidVelocity(bidHistory);
// Strategy recommendation
String strategy = isSnipeTarget ? "SNIPING_DETECTED" :
(hasReserve && lot.getCurrentBid() &lt; lot.getReservePrice() * 0.9) ? "RESERVE_AVOID" :
bidVelocity &gt; 5.0 ? "AGGRESSIVE_COMPETITION" : "STANDARD";
return new BiddingStrategy(
strategy,
calculateRecommendedMax(lot),
isSnipeTarget ? "FINAL_30_SECONDS" : "FINAL_10_MINUTES",
getCompetitionLevel(watchers, bidHistory.size())
);
}
```
```sqlite
-- Core bidding intelligence
ALTER TABLE lots ADD COLUMN starting_bid DECIMAL(12,2);
ALTER TABLE lots ADD COLUMN estimated_min DECIMAL(12,2);
ALTER TABLE lots ADD COLUMN estimated_max DECIMAL(12,2);
ALTER TABLE lots ADD COLUMN reserve_price DECIMAL(12,2);
ALTER TABLE lots ADD COLUMN watch_count INTEGER DEFAULT 0;
ALTER TABLE lots ADD COLUMN first_bid_time TEXT;
ALTER TABLE lots ADD COLUMN last_bid_time TEXT;
ALTER TABLE lots ADD COLUMN bid_velocity DECIMAL(5,2);
-- Bid history (critical)
CREATE TABLE bid_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT REFERENCES lots(lot_id),
bid_amount DECIMAL(12,2) NOT NULL,
bid_time TEXT NOT NULL,
is_winning BOOLEAN DEFAULT FALSE,
is_autobid BOOLEAN DEFAULT FALSE,
bidder_id TEXT,
created_at TEXT DEFAULT CURRENT_TIMESTAMP
);
-- Valuation support
ALTER TABLE lots ADD COLUMN condition_score DECIMAL(3,2);
ALTER TABLE lots ADD COLUMN year_manufactured INTEGER;
ALTER TABLE lots ADD COLUMN provenance TEXT;
CREATE TABLE comparable_sales (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT REFERENCES lots(lot_id),
comparable_lot_id TEXT,
similarity_score DECIMAL(3,2),
price_difference_percent DECIMAL(5,2)
);
CREATE TABLE market_indices (
category TEXT NOT NULL,
manufacturer TEXT,
avg_price DECIMAL(12,2),
price_change_30d DECIMAL(5,2),
PRIMARY KEY (category, manufacturer)
);
-- Alert system
CREATE TABLE price_alerts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT REFERENCES lots(lot_id),
alert_type TEXT CHECK(alert_type IN ('UNDervalued', 'ACCELERATING', 'RESERVE_IN_SIGHT')),
trigger_price DECIMAL(12,2),
is_triggered BOOLEAN DEFAULT FALSE
);
```

310
mvnw vendored Normal file
View File

@@ -0,0 +1,310 @@
#!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Maven Start Up Batch script
#
# Required ENV vars:
# ------------------
# JAVA_HOME - location of a JDK home dir
#
# Optional ENV vars
# -----------------
# M2_HOME - location of maven2's installed home dir
# MAVEN_OPTS - parameters passed to the Java VM when running Maven
# e.g. to debug Maven itself, use
# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
# ----------------------------------------------------------------------------
if [ -z "$MAVEN_SKIP_RC" ] ; then
if [ -f /etc/mavenrc ] ; then
. /etc/mavenrc
fi
if [ -f "$HOME/.mavenrc" ] ; then
. "$HOME/.mavenrc"
fi
fi
# OS specific support. $var _must_ be set to either true or false.
cygwin=false;
darwin=false;
mingw=false
case "`uname`" in
CYGWIN*) cygwin=true ;;
MINGW*) mingw=true;;
Darwin*) darwin=true
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html
if [ -z "$JAVA_HOME" ]; then
if [ -x "/usr/libexec/java_home" ]; then
export JAVA_HOME="`/usr/libexec/java_home`"
else
export JAVA_HOME="/Library/Java/Home"
fi
fi
;;
esac
if [ -z "$JAVA_HOME" ] ; then
if [ -r /etc/gentoo-release ] ; then
JAVA_HOME=`java-config --jre-home`
fi
fi
if [ -z "$M2_HOME" ] ; then
## resolve links - $0 may be a link to maven's home
PRG="$0"
# need this for relative symlinks
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG="`dirname "$PRG"`/$link"
fi
done
saveddir=`pwd`
M2_HOME=`dirname "$PRG"`/..
# make it fully qualified
M2_HOME=`cd "$M2_HOME" && pwd`
cd "$saveddir"
# echo Using m2 at $M2_HOME
fi
# For Cygwin, ensure paths are in UNIX format before anything is touched
if $cygwin ; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --unix "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
fi
# For Mingw, ensure paths are in UNIX format before anything is touched
if $mingw ; then
[ -n "$M2_HOME" ] &&
M2_HOME="`(cd "$M2_HOME"; pwd)`"
[ -n "$JAVA_HOME" ] &&
JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
fi
if [ -z "$JAVA_HOME" ]; then
javaExecutable="`which javac`"
if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
# readlink(1) is not available as standard on Solaris 10.
readLink=`which readlink`
if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
if $darwin ; then
javaHome="`dirname \"$javaExecutable\"`"
javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
else
javaExecutable="`readlink -f \"$javaExecutable\"`"
fi
javaHome="`dirname \"$javaExecutable\"`"
javaHome=`expr "$javaHome" : '\(.*\)/bin'`
JAVA_HOME="$javaHome"
export JAVA_HOME
fi
fi
fi
if [ -z "$JAVACMD" ] ; then
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
else
JAVACMD="`which java`"
fi
fi
if [ ! -x "$JAVACMD" ] ; then
echo "Error: JAVA_HOME is not defined correctly." >&2
echo " We cannot execute $JAVACMD" >&2
exit 1
fi
if [ -z "$JAVA_HOME" ] ; then
echo "Warning: JAVA_HOME environment variable is not set."
fi
CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
# traverses directory structure from process work directory to filesystem root
# first directory with .mvn subdirectory is considered project base directory
find_maven_basedir() {
if [ -z "$1" ]
then
echo "Path not specified to find_maven_basedir"
return 1
fi
basedir="$1"
wdir="$1"
while [ "$wdir" != '/' ] ; do
if [ -d "$wdir"/.mvn ] ; then
basedir=$wdir
break
fi
# workaround for JBEAP-8937 (on Solaris 10/Sparc)
if [ -d "${wdir}" ]; then
wdir=`cd "$wdir/.."; pwd`
fi
# end of workaround
done
echo "${basedir}"
}
# concatenates all lines of a file
concat_lines() {
if [ -f "$1" ]; then
echo "$(tr -s '\n' ' ' < "$1")"
fi
}
BASE_DIR=`find_maven_basedir "$(pwd)"`
if [ -z "$BASE_DIR" ]; then
exit 1;
fi
##########################################################################################
# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
# This allows using the maven wrapper in projects that prohibit checking in binary data.
##########################################################################################
if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found .mvn/wrapper/maven-wrapper.jar"
fi
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
fi
if [ -n "$MVNW_REPOURL" ]; then
jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
else
jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar"
fi
while IFS="=" read key value; do
case "$key" in (wrapperUrl) jarUrl="$value"; break ;;
esac
done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
if [ "$MVNW_VERBOSE" = true ]; then
echo "Downloading from: $jarUrl"
fi
wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
if $cygwin; then
wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"`
fi
if command -v wget > /dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found wget ... using wget"
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
wget "$jarUrl" -O "$wrapperJarPath"
else
wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath"
fi
elif command -v curl > /dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found curl ... using curl"
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
curl -o "$wrapperJarPath" "$jarUrl" -f
else
curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
fi
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Falling back to using Java to download"
fi
javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
# For Cygwin, switch paths to Windows format before running javac
if $cygwin; then
javaClass=`cygpath --path --windows "$javaClass"`
fi
if [ -e "$javaClass" ]; then
if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Compiling MavenWrapperDownloader.java ..."
fi
# Compiling the Java class
("$JAVA_HOME/bin/javac" "$javaClass")
fi
if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
# Running the downloader
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Running MavenWrapperDownloader.java ..."
fi
("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
fi
fi
fi
fi
##########################################################################################
# End of extension
##########################################################################################
export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
if [ "$MVNW_VERBOSE" = true ]; then
echo $MAVEN_PROJECTBASEDIR
fi
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
# For Cygwin, switch paths to Windows format before running java
if $cygwin; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --path --windows "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
[ -n "$MAVEN_PROJECTBASEDIR" ] &&
MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
fi
# Provide a "standardized" way to retrieve the CLI args that will
# work with both Windows and non-Windows executions.
MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
export MAVEN_CMD_LINE_ARGS
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
exec "$JAVACMD" \
$MAVEN_OPTS \
-classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
"-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"

187
mvnw.cmd vendored Normal file
View File

@@ -0,0 +1,187 @@
@REM ----------------------------------------------------------------------------
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing,
@REM software distributed under the License is distributed on an
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@REM KIND, either express or implied. See the License for the
@REM specific language governing permissions and limitations
@REM under the License.
@REM ----------------------------------------------------------------------------
@REM ----------------------------------------------------------------------------
@REM Apache Maven Wrapper startup batch script, version 3.3.4
@REM
@REM Required ENV vars:
@REM JAVA_HOME - location of a JDK home dir
@REM
@REM Optional ENV vars
@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
@REM e.g. to debug Maven itself, use
@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
@REM ----------------------------------------------------------------------------
@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
@echo off
@REM set title of command window
title %0
@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
@REM set %HOME% to equivalent of $HOME
if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
@REM Execute a user defined script before this one
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
@REM check for pre script, once with legacy .bat ending and once with .cmd ending
if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %*
if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %*
:skipRcPre
@setlocal
set ERROR_CODE=0
@REM To isolate internal variables from possible post scripts, we use another setlocal
@setlocal
@REM ==== START VALIDATION ====
if not "%JAVA_HOME%" == "" goto OkJHome
echo.
echo Error: JAVA_HOME not found in your environment. >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
:OkJHome
if exist "%JAVA_HOME%\bin\java.exe" goto init
echo.
echo Error: JAVA_HOME is set to an invalid directory. >&2
echo JAVA_HOME = "%JAVA_HOME%" >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
@REM ==== END VALIDATION ====
:init
@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
@REM Fallback to current working directory if not found.
set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
set EXEC_DIR=%CD%
set WDIR=%EXEC_DIR%
:findBaseDir
IF EXIST "%WDIR%"\.mvn goto baseDirFound
cd ..
IF "%WDIR%"=="%CD%" goto baseDirNotFound
set WDIR=%CD%
goto findBaseDir
:baseDirNotFound
set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
cd "%EXEC_DIR%"
goto endDetectBaseDir
:baseDirFound
set MAVEN_PROJECTBASEDIR=%WDIR%
cd "%EXEC_DIR%"
:endDetectBaseDir
IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
@setlocal EnableExtensions EnableDelayedExpansion
for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG=!JVM_CONFIG! %%a
@endlocal & set JVM_CONFIG=%JVM_CONFIG%
:endReadAdditionalConfig
SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar"
FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B
)
@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
if exist %WRAPPER_JAR% (
if "%MVNW_VERBOSE%" == "true" (
echo Found %WRAPPER_JAR%
)
) else (
if not "%MVNW_REPOURL%" == "" (
SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar"
)
if "%MVNW_VERBOSE%" == "true" (
echo Couldn't find %WRAPPER_JAR%, downloading it ...
echo Downloading from: %WRAPPER_URL%
)
powershell -Command "&{"^
"$webclient = new-object System.Net.WebClient;"^
"if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
"$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
"}"^
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^
"}"
if "%MVNW_VERBOSE%" == "true" (
echo Finished downloading %WRAPPER_JAR%
)
)
@REM End of extension
%MAVEN_JAVA_EXE% ^
%JVM_CONFIG% ^
--add-opens=java.base/java.lang=ALL-UNNAMED ^
--add-opens=java.base/java.util=ALL-UNNAMED ^
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED ^
--add-opens=java.base/java.net=ALL-UNNAMED ^
--add-opens=java.base/java.io=ALL-UNNAMED ^
%MAVEN_OPTS% ^
%MAVEN_DEBUG_OPTS% ^
-classpath %WRAPPER_JAR% ^
"-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^
%WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
if ERRORLEVEL 1 goto error
goto end
:error
set ERROR_CODE=1
:end
@endlocal & set ERROR_CODE=%ERROR_CODE%
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
@REM check for post script, once with legacy .bat ending and once with .cmd ending
if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat"
if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd"
:skipRcPost
@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
if "%MAVEN_BATCH_PAUSE%" == "on" pause
if "%MAVEN_VERBOSE%" == "on" echo %ERROR_CODE%
exit /B %ERROR_CODE%

434
pom.xml
View File

@@ -4,23 +4,112 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.auction</groupId>
<artifactId>troostwijk-scraper</artifactId>
<groupId>auctiora</groupId>
<artifactId>auctiora</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Troostwijk Auction Scraper</name>
<description>Web scraper for Troostwijk Auctions with object detection and notifications</description>
<repositories>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
</repositories>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<maven.compiler.source>25</maven.compiler.source>
<maven.compiler.target>25</maven.compiler.target>
<maven.compiler.release>25</maven.compiler.release>
<jackson.version>2.17.0</jackson.version>
<opencv.version>4.9.0-0</opencv.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<quarkus.platform.version>3.17.7</quarkus.platform.version>
<asm.version>9.8</asm.version>
<lombok.version>1.18.40</lombok.version>
<!--this is not a bug, its feature -->
<lombok-version>${lombok.version}</lombok-version>
<lombok-maven-version>1.18.20.0</lombok-maven-version>
<maven-compiler-plugin-version>3.14.0</maven-compiler-plugin-version>
<versions-maven-plugin.version>2.19.0</versions-maven-plugin.version>
<jandex-maven-plugin-version>3.5.0</jandex-maven-plugin-version>
<jdbi.version>3.47.0</jdbi.version>
<maven.compiler.args>
--enable-native-access=ALL-UNNAMED
--add-opens java.base/sun.misc=ALL-UNNAMED
-Xdiags:verbose
-Xlint:all
</maven.compiler.args>
<uberJar>true</uberJar> <!-- Your existing properties... -->
<quarkus.package.jar.type>uber-jar</quarkus.package.jar.type>
<quarkus.package.jar.enabled>true</quarkus.package.jar.enabled>
<maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>io.quarkus.platform</groupId>
<artifactId>quarkus-bom</artifactId>
<version>${quarkus.platform.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- Override ASM to support Java 25 -->
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm-commons</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm-tree</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm-util</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>4.1.124.Final</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- Force consistent versions -->
<dependency>
<groupId>org.opentest4j</groupId>
<artifactId>opentest4j</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.9</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>com.github.vladimir-bukhtoyarov.bucket4j</groupId>
<artifactId>bucket4j-core</artifactId>
<version>7.6.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.github.bucket4j/bucket4j -->
<!--<dependency>
<groupId>io.github.bucket4j</groupId>
<artifactId>bucket4j</artifactId>
<version>8.9.0</version>
</dependency>-->
<!-- JSoup for HTML parsing and HTTP client -->
<dependency>
<groupId>org.jsoup</groupId>
@@ -41,7 +130,19 @@
<artifactId>sqlite-jdbc</artifactId>
<version>3.45.1.0</version>
</dependency>
<!-- Lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven</artifactId>
<version>${lombok-maven-version}</version>
<type>pom</type>
</dependency>
<!-- JavaMail API for email notifications -->
<dependency>
<groupId>com.sun.mail</groupId>
@@ -49,52 +150,315 @@
<version>1.6.2</version>
</dependency>
<!-- OpenCV for image processing and object detection -->
<dependency>
<groupId>com.microsoft.playwright</groupId>
<artifactId>playwright</artifactId>
<version>1.40.0</version>
</dependency>
<!-- SLF4J API and implementation for logging -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.9</version>
</dependency>
<!-- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>2.0.9</version>
</dependency>-->
<!-- JUnit 5 for testing -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.10.1</version>
<scope>test</scope>
</dependency>
<!-- Mockito for mocking in tests -->
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5-mockito</artifactId>
<version>3.30.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>5.8.0</version>
<scope>test</scope>
</dependency>
<!-- Mockito JUnit Jupiter integration -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<version>5.8.0</version>
<scope>test</scope>
</dependency>
<!-- JDBI3 - Lightweight ORM for SQL -->
<dependency>
<groupId>org.jdbi</groupId>
<artifactId>jdbi3-core</artifactId>
<version>${jdbi.version}</version>
</dependency>
<dependency>
<groupId>org.jdbi</groupId>
<artifactId>jdbi3-sqlobject</artifactId>
<version>${jdbi.version}</version>
</dependency>
<!-- AssertJ for fluent assertions (optional but recommended) -->
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.24.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.vladsch.flexmark</groupId>
<artifactId>flexmark-all</artifactId>
<version>0.64.8</version>
</dependency>
<!-- Quarkus Core Dependencies -->
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-rest-jackson</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-arc</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-scheduler</artifactId>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Explicitly add cron-utils with slf4j excluded to avoid path warning -->
<dependency>
<groupId>com.cronutils</groupId>
<artifactId>cron-utils</artifactId>
<version>9.2.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Force Netty 4.1.124.Final to avoid sun.misc.Unsafe warnings -->
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec-http2</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver-dns</artifactId>
<version>4.1.124.Final</version>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-smallrye-health</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-config-yaml</artifactId>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-undertow</artifactId>
</dependency>
<!-- OSGi annotations -->
<dependency>
<groupId>org.osgi</groupId>
<artifactId>org.osgi.annotation.bundle</artifactId>
<version>2.0.0</version>
<scope>provided</scope>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-junit5</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.rest-assured</groupId>
<artifactId>rest-assured</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.openpnp</groupId>
<artifactId>opencv</artifactId>
<version>${opencv.version}</version>
<version>4.9.0-0</version>
<!--<classifier>windows-x86_64</classifier>-->
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<plugins>
<!-- Maven Compiler Plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.11.0</version>
<configuration>
<source>11</source>
<target>11</target>
</configuration>
</plugin>
<!-- Maven Assembly Plugin for creating executable JAR with dependencies -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<archive>
<manifest>
<mainClass>com.auction.scraper.TroostwijkScraper</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<groupId>io.quarkus.platform</groupId>
<artifactId>quarkus-maven-plugin</artifactId>
<version>${quarkus.platform.version}</version>
<extensions>true</extensions>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
<goal>build</goal>
<goal>generate-code</goal>
<goal>generate-code-tests</goal>
</goals>
</execution>
</executions>
<configuration>
<properties>
<build.timestamp>${maven.build.timestamp}</build.timestamp>
</properties>
<jvmArgs>--enable-native-access=ALL-UNNAMED --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true</jvmArgs>
</configuration>
</plugin>
<plugin>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven-plugin</artifactId>
<version>${lombok-maven-version}</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>delombok</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Maven Exec Plugin for running with native access -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin-version}</version>
<configuration>
<release>${maven.compiler.release}</release>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok-version}</version>
</path>
<path>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-extension-processor</artifactId>
<version>${quarkus.platform.version}</version>
</path>
</annotationProcessorPaths>
<compilerArgs>
<arg>-Xdiags:verbose</arg>
<arg>-Xlint:all</arg>
</compilerArgs>
<fork>true</fork>
<excludes>
<exclude>module-info.java</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
<version>${versions-maven-plugin.version}</version>
</plugin>
<!-- Maven Surefire Plugin for tests with native access -->
<plugin>
<groupId>io.smallrye</groupId>
<artifactId>jandex-maven-plugin</artifactId>
<version>${jandex-maven-plugin-version}</version>
<executions>
<execution>
<id>make-index</id>
<goals>
<goal>jandex</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.2.5</version>
<configuration>
<!-- Enable ByteBuddy experimental mode for Java 25 support -->
<!-- Mockito requires this for Java 24+ -->
<argLine>
--enable-native-access=ALL-UNNAMED
--add-opens java.base/sun.misc=ALL-UNNAMED
-Dnet.bytebuddy.experimental=true
--add-opens java.base/java.lang=ALL-UNNAMED
--add-opens java.base/java.util=ALL-UNNAMED
</argLine>
</configuration>
</plugin>
</plugins>
</build>
<!-- In your pom.xml, alongside <build> and <dependencies> -->
<distributionManagement>
<repository>
<id>gitea</id>
<url>https://git.appmodel.nl/api/packages/Tour/maven</url>
</repository>
<snapshotRepository>
<id>gitea</id>
<url>https://git.appmodel.nl/api/packages/Tour/maven</url>
</snapshotRepository>
</distributionManagement>
</project>

206
scripts/README.md Normal file
View File

@@ -0,0 +1,206 @@
# Auctiora Scripts
Utility scripts for managing the Auctiora auction monitoring system.
## 📦 Available Scripts
### 1. Production Data Sync
Sync production database and images from `athena.lan` to your local development environment.
#### Quick Start
**Linux/Mac (Bash)**:
```bash
# Make executable (first time only)
chmod +x scripts/sync-production-data.sh
# Sync database only
./scripts/sync-production-data.sh --db-only
# Sync everything
./scripts/sync-production-data.sh --all
# Sync images only
./scripts/sync-production-data.sh --images-only
```
## 🔧 Prerequisites
### Required
- **SSH Client**: OpenSSH or equivalent
- Windows: Built-in on Windows 10+, or install [Git Bash](https://git-scm.com/downloads)
- Linux/Mac: Pre-installed
- **SCP**: Secure copy (usually comes with SSH)
- **SSH Access**: SSH key configured for `tour@athena.lan`
### Optional
- **rsync**: For efficient incremental image sync
- Windows: Install via [WSL](https://docs.microsoft.com/en-us/windows/wsl/install) or [Cygwin](https://www.cygwin.com/)
- Linux/Mac: Usually pre-installed
- **sqlite3**: For showing database statistics
- Windows: Download from [sqlite.org](https://www.sqlite.org/download.html)
- Linux: `sudo apt install sqlite3`
- Mac: Pre-installed
## 📊 What Gets Synced
### Database (`cache.db`)
- **Size**: ~8.9 GB (as of Dec 2024)
- **Contains**:
- Auctions metadata
- Lots (kavels) with bid information
- Images metadata and URLs
- HTTP cache for scraper
- **Local Path**: `c:\mnt\okcomputer\cache.db`
### Images Directory
- **Size**: Varies (can be large)
- **Contains**:
- Downloaded lot images
- Organized by lot ID
- **Local Path**: `c:\mnt\okcomputer\images\`
## 🚀 Usage Examples
## 📁 File Locations
### Remote (Production)
```
athena.lan
├── Docker Volume: shared-auction-data
│ ├── /data/cache.db (SQLite database)
│ └── /data/images/ (Image files)
└── /tmp/ (Temporary staging area)
```
### Local (Development)
```
c:\mnt\okcomputer\
├── cache.db (SQLite database)
├── cache.db.backup-* (Automatic backups)
└── images\ (Image files)
```
## 🔒 Safety Features
### Automatic Backups
- Existing local database is automatically backed up before sync
- Backup format: `cache.db.backup-YYYYMMDD-HHMMSS`
- Keep recent backups manually or clean up old ones
### Confirmation Prompts
- PowerShell script prompts for confirmation (unless `-Force` is used)
- Shows configuration before executing
- Safe to cancel at any time
### Error Handling
- Validates SSH connection before starting
- Cleans up temporary files on remote server
- Reports clear error messages
## ⚡ Performance Tips
### Faster Image Sync with rsync
Install rsync for incremental image sync (only new/changed files):
**Windows (WSL)**:
```powershell
wsl --install
wsl -d Ubuntu
sudo apt install rsync
```
**Windows (Chocolatey)**:
```powershell
choco install rsync
```
**Benefit**: First sync downloads everything, subsequent syncs only transfer changed files.
Images can be synced separately when needed for image processing tests.
## 🐛 Troubleshooting
### SSH Connection Issues
```powershell
# Test SSH connection
ssh tour@athena.lan "echo 'Connection OK'"
# Check SSH key
ssh-add -l
```
### Permission Denied
```bash
# Add SSH key (Linux/Mac)
chmod 600 ~/.ssh/id_rsa
ssh-add ~/.ssh/id_rsa
# Windows: Use PuTTY or OpenSSH for Windows
```
### Database Locked Error
```powershell
# Make sure no other process is using the database
Get-Process | Where-Object {$_.Path -like "*java*"} | Stop-Process
# Or restart the monitor
```
### Slow Image Sync
- Use rsync instead of scp (see Performance Tips)
- Consider syncing only database for code development
- Images only needed for object detection testing
## 📝 Script Details
### sync-production-data.sh (Bash)
- **Platform**: Linux, Mac, Git Bash on Windows
- **Best for**: Unix-like environments
- **Features**: Color output, progress bars, statistics
## 🔄 Automation
### Linux/Mac Cron
```bash
# Edit crontab
crontab -e
# Add daily sync at 7 AM
0 7 * * * /path/to/auctiora/scripts/sync-production-data.sh --db-only
```
## 🆘 Support
### Getting Help
```bash
# Bash
./scripts/sync-production-data.sh --help
```
### Common Commands
```powershell
# Check database size
ls c:\mnt\okcomputer\cache.db -h
# View database contents
sqlite3 c:\mnt\okcomputer\cache.db
.tables
.schema lots
SELECT COUNT(*) FROM lots;
.quit
# Check image count
(Get-ChildItem c:\mnt\okcomputer\images -Recurse -File).Count
```
## 📚 Related Documentation
- [Database Architecture](../wiki/DATABASE_ARCHITECTURE.md)
- [Integration Flowchart](../docs/INTEGRATION_FLOWCHART.md)
- [Main README](../README.md)
---
**Last Updated**: December 2025
**Maintainer**: Auctiora Development Team

160
scripts/cleanup-database.sh Normal file
View File

@@ -0,0 +1,160 @@
#!/bin/bash
#
# Database Cleanup Utility
#
# Removes invalid/old data from the local database
#
# Usage:
# ./scripts/cleanup-database.sh [--dry-run]
#
# Options:
# --dry-run Show what would be deleted without actually deleting
#
set -e
# Configuration
LOCAL_DB_PATH="${1:-c:/mnt/okcomputer/cache.db}"
DRY_RUN=false
# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m'
# Parse arguments
if [ "$1" = "--dry-run" ] || [ "$2" = "--dry-run" ]; then
DRY_RUN=true
fi
if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then
grep '^#' "$0" | sed 's/^# \?//'
exit 0
fi
echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║ Database Cleanup - Auctiora Monitor ║${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
echo ""
if [ ! -f "${LOCAL_DB_PATH}" ]; then
echo -e "${RED}Error: Database not found at ${LOCAL_DB_PATH}${NC}"
exit 1
fi
# Backup database before cleanup
if [ "$DRY_RUN" = false ]; then
BACKUP_PATH="${LOCAL_DB_PATH}.backup-before-cleanup-$(date +%Y%m%d-%H%M%S)"
echo -e "${YELLOW}Creating backup: ${BACKUP_PATH}${NC}"
cp "${LOCAL_DB_PATH}" "${BACKUP_PATH}"
echo ""
fi
# Show current state
echo -e "${BLUE}Current database state:${NC}"
sqlite3 "${LOCAL_DB_PATH}" <<EOF
.mode box
SELECT
'Total lots' as metric,
COUNT(*) as count
FROM lots
UNION ALL
SELECT
'Valid lots (with auction_id)',
COUNT(*)
FROM lots
WHERE auction_id IS NOT NULL AND auction_id != ''
UNION ALL
SELECT
'Invalid lots (missing auction_id)',
COUNT(*)
FROM lots
WHERE auction_id IS NULL OR auction_id = '';
EOF
echo ""
# Count items to be deleted
echo -e "${YELLOW}Analyzing data to clean up...${NC}"
INVALID_LOTS=$(sqlite3 "${LOCAL_DB_PATH}" "SELECT COUNT(*) FROM lots WHERE auction_id IS NULL OR auction_id = '';")
ORPHANED_IMAGES=$(sqlite3 "${LOCAL_DB_PATH}" "SELECT COUNT(*) FROM images WHERE lot_id NOT IN (SELECT lot_id FROM lots);")
echo -e " ${RED}→ Invalid lots to delete: ${INVALID_LOTS}${NC}"
echo -e " ${YELLOW}→ Orphaned images to delete: ${ORPHANED_IMAGES}${NC}"
echo ""
if [ "$INVALID_LOTS" -eq 0 ] && [ "$ORPHANED_IMAGES" -eq 0 ]; then
echo -e "${GREEN}✓ Database is clean! No cleanup needed.${NC}"
exit 0
fi
if [ "$DRY_RUN" = true ]; then
echo -e "${BLUE}DRY RUN MODE - No changes will be made${NC}"
echo ""
echo "Would delete:"
echo " - $INVALID_LOTS invalid lots"
echo " - $ORPHANED_IMAGES orphaned images"
echo ""
echo "Run without --dry-run to perform cleanup"
exit 0
fi
# Confirm cleanup
echo -e "${YELLOW}This will permanently delete the above records.${NC}"
read -p "Continue? (y/N) " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
echo "Cleanup cancelled"
exit 0
fi
# Perform cleanup
echo ""
echo -e "${YELLOW}Cleaning up database...${NC}"
# Delete invalid lots
if [ "$INVALID_LOTS" -gt 0 ]; then
echo -e " ${BLUE}[1/2] Deleting invalid lots...${NC}"
sqlite3 "${LOCAL_DB_PATH}" "DELETE FROM lots WHERE auction_id IS NULL OR auction_id = '';"
echo -e " ${GREEN}✓ Deleted ${INVALID_LOTS} invalid lots${NC}"
fi
# Delete orphaned images
if [ "$ORPHANED_IMAGES" -gt 0 ]; then
echo -e " ${BLUE}[2/2] Deleting orphaned images...${NC}"
sqlite3 "${LOCAL_DB_PATH}" "DELETE FROM images WHERE lot_id NOT IN (SELECT lot_id FROM lots);"
echo -e " ${GREEN}✓ Deleted ${ORPHANED_IMAGES} orphaned images${NC}"
fi
# Vacuum database to reclaim space
echo -e " ${BLUE}[3/3] Compacting database...${NC}"
sqlite3 "${LOCAL_DB_PATH}" "VACUUM;"
echo -e " ${GREEN}✓ Database compacted${NC}"
echo ""
echo -e "${GREEN}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Cleanup completed successfully ║${NC}"
echo -e "${GREEN}╚════════════════════════════════════════════════════════╝${NC}"
echo ""
# Show final state
echo -e "${BLUE}Final database state:${NC}"
sqlite3 "${LOCAL_DB_PATH}" <<EOF
.mode box
SELECT
'Total lots' as metric,
COUNT(*) as count
FROM lots
UNION ALL
SELECT
'Total images',
COUNT(*)
FROM images;
EOF
echo ""
DB_SIZE=$(du -h "${LOCAL_DB_PATH}" | cut -f1)
echo -e "${BLUE}Database size: ${DB_SIZE}${NC}"
echo ""

15
scripts/smb.ps1 Normal file
View File

@@ -0,0 +1,15 @@
# PowerShell: map the remote share, copy the folder, then clean up
$remote = '\\192.168.1.159\shared-auction-data'
$local = 'C:\mnt\okcomputer\output\models'
# (1) create/verify the PSDrive (prompts for password if needed)
if (-not (Get-PSDrive -Name Z -ErrorAction SilentlyContinue)) {
$cred = Get-Credential -UserName 'tour' -Message 'SMB password for tour@192.168.1.159'
New-PSDrive -Name Z -PSProvider FileSystem -Root $remote -Credential $cred -Persist | Out-Null
}
# (2) copy the local folder into the share
Copy-Item -Path $local -Destination 'Z:\' -Recurse -Force
# (3) optional cleanup
Remove-PSDrive -Name Z -Force

View File

@@ -0,0 +1,200 @@
#!/bin/bash
#
# Sync Production Data to Local
#
# This script copies the production SQLite database and images from the remote
# server (athena.lan) to your local development environment.
#
# Usage:
# ./scripts/sync-production-data.sh [--db-only|--images-only|--all]
#
# Options:
# --db-only Only sync the database (default)
# --images-only Only sync the images
# --all Sync both database and images
# --help Show this help message
#
set -e # Exit on error
# Configuration
REMOTE_HOST="tour@athena.lan"
REMOTE_VOLUME="shared-auction-data"
LOCAL_DB_PATH="c:/mnt/okcomputer/output/cache.db"
LOCAL_IMAGES_PATH="c:/mnt/okcomputer/images"
REMOTE_TMP="/tmp"
# Colors for output
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color
# Parse arguments
SYNC_MODE="db" # Default: database only
case "${1:-}" in
--db-only)
SYNC_MODE="db"
;;
--images-only)
SYNC_MODE="images"
;;
--all)
SYNC_MODE="all"
;;
--help|-h)
grep '^#' "$0" | sed 's/^# \?//'
exit 0
;;
"")
SYNC_MODE="db"
;;
*)
echo -e "${RED}Error: Unknown option '$1'${NC}"
echo "Use --help for usage information"
exit 1
;;
esac
echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║ Production Data Sync - Auctiora Monitor ║${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
echo ""
# Function to sync database
sync_database() {
echo -e "${YELLOW}[1/3] Copying database from Docker volume to /tmp...${NC}"
ssh ${REMOTE_HOST} "docker run --rm -v ${REMOTE_VOLUME}:/data -v ${REMOTE_TMP}:${REMOTE_TMP} alpine cp /data/cache.db ${REMOTE_TMP}/cache.db"
echo -e "${YELLOW}[2/3] Downloading database from remote server...${NC}"
# Create backup and remove old local database
if [ -f "${LOCAL_DB_PATH}" ]; then
BACKUP_PATH="${LOCAL_DB_PATH}.backup-$(date +%Y%m%d-%H%M%S)"
echo -e "${BLUE} Backing up existing database to: ${BACKUP_PATH}${NC}"
cp "${LOCAL_DB_PATH}" "${BACKUP_PATH}"
echo -e "${BLUE} Removing old local database...${NC}"
rm -f "${LOCAL_DB_PATH}"
fi
# Download new database
scp ${REMOTE_HOST}:${REMOTE_TMP}/cache.db "${LOCAL_DB_PATH}"
echo -e "${YELLOW}[3/3] Cleaning up remote /tmp...${NC}"
ssh ${REMOTE_HOST} "rm -f ${REMOTE_TMP}/cache.db"
# Show database info
DB_SIZE=$(du -h "${LOCAL_DB_PATH}" | cut -f1)
echo -e "${GREEN}✓ Database synced successfully (${DB_SIZE})${NC}"
# Show table counts
echo -e "${BLUE} Database statistics:${NC}"
sqlite3 "${LOCAL_DB_PATH}" <<EOF
.mode box
SELECT
'auctions' as table_name, COUNT(*) as count FROM auctions
UNION ALL
SELECT 'lots', COUNT(*) FROM lots
UNION ALL
SELECT 'images', COUNT(*) FROM images
UNION ALL
SELECT 'cache', COUNT(*) FROM cache;
EOF
# Show data quality report
echo -e "${BLUE} Data quality:${NC}"
sqlite3 "${LOCAL_DB_PATH}" <<EOF
.mode box
SELECT
'Valid lots' as metric,
COUNT(*) as count,
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%' as percentage
FROM lots
WHERE auction_id IS NOT NULL AND auction_id != ''
UNION ALL
SELECT
'Invalid lots (missing auction_id)',
COUNT(*),
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
FROM lots
WHERE auction_id IS NULL OR auction_id = ''
UNION ALL
SELECT
'Lots with intelligence fields',
COUNT(*),
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
FROM lots
WHERE followers_count IS NOT NULL OR estimated_min IS NOT NULL;
EOF
}
# Function to sync images
sync_images() {
echo -e "${YELLOW}[1/4] Getting image directory structure from Docker volume...${NC}"
# Create local images directory if it doesn't exist
mkdir -p "${LOCAL_IMAGES_PATH}"
echo -e "${YELLOW}[2/4] Copying images from Docker volume to /tmp...${NC}"
# Copy entire images directory from volume to /tmp
ssh ${REMOTE_HOST} "docker run --rm -v ${REMOTE_VOLUME}:/data -v ${REMOTE_TMP}:${REMOTE_TMP} alpine sh -c 'mkdir -p ${REMOTE_TMP}/auction-images && cp -r /data/images/* ${REMOTE_TMP}/auction-images/ 2>/dev/null || true'"
echo -e "${YELLOW}[3/4] Syncing images to local directory (this may take a while)...${NC}"
# Use rsync for efficient incremental sync
if command -v rsync &> /dev/null; then
echo -e "${BLUE} Using rsync for efficient transfer...${NC}"
rsync -avz --progress ${REMOTE_HOST}:${REMOTE_TMP}/auction-images/ "${LOCAL_IMAGES_PATH}/"
else
echo -e "${BLUE} Using scp for transfer (install rsync for faster incremental sync)...${NC}"
scp -r ${REMOTE_HOST}:${REMOTE_TMP}/auction-images/* "${LOCAL_IMAGES_PATH}/"
fi
echo -e "${YELLOW}[4/4] Cleaning up remote /tmp...${NC}"
ssh ${REMOTE_HOST} "rm -rf ${REMOTE_TMP}/auction-images"
# Show image stats
IMAGE_COUNT=$(find "${LOCAL_IMAGES_PATH}" -type f 2>/dev/null | wc -l)
IMAGE_SIZE=$(du -sh "${LOCAL_IMAGES_PATH}" 2>/dev/null | cut -f1)
echo -e "${GREEN}✓ Images synced successfully${NC}"
echo -e "${BLUE} Total images: ${IMAGE_COUNT}${NC}"
echo -e "${BLUE} Total size: ${IMAGE_SIZE}${NC}"
}
# Execute sync based on mode
START_TIME=$(date +%s)
case "$SYNC_MODE" in
db)
echo -e "${BLUE}Mode: Database only${NC}"
echo ""
sync_database
;;
images)
echo -e "${BLUE}Mode: Images only${NC}"
echo ""
sync_images
;;
all)
echo -e "${BLUE}Mode: Database + Images${NC}"
echo ""
sync_database
echo ""
sync_images
;;
esac
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))
echo ""
echo -e "${GREEN}╔════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Sync completed successfully in ${DURATION} seconds ║${NC}"
echo -e "${GREEN}╚════════════════════════════════════════════════════════╝${NC}"
echo ""
echo -e "${BLUE}Next steps:${NC}"
echo -e " 1. Verify data: sqlite3 ${LOCAL_DB_PATH} 'SELECT COUNT(*) FROM lots;'"
echo -e " 2. Start monitor: mvn quarkus:dev"
echo -e " 3. Open dashboard: http://localhost:8080"
echo ""

View File

@@ -0,0 +1,19 @@
package auctiora;
import java.time.LocalDateTime;
/**
* Represents auction metadata (veiling informatie)
* Data typically populated by the external scraper process
*/
public record AuctionInfo(
long auctionId, // Unique auction ID (from URL)
String title, // Auction title
String location, // Location (e.g., "Amsterdam, NL")
String city, // City name
String country, // Country code (e.g., "NL")
String url, // Full auction URL
String typePrefix, // Auction type (A1 or A7)
int lotCount, // Number of lots/kavels
LocalDateTime firstLotClosingTime // Closing time if available
) { }

View File

@@ -0,0 +1,82 @@
package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.eclipse.microprofile.health.HealthCheck;
import org.eclipse.microprofile.health.HealthCheckResponse;
import org.eclipse.microprofile.health.Liveness;
import org.eclipse.microprofile.health.Readiness;
import org.eclipse.microprofile.health.Startup;
import java.nio.file.Files;
import java.nio.file.Paths;
@ApplicationScoped
public class AuctionMonitorHealthCheck {
@Liveness
public static class LivenessCheck
implements HealthCheck {
@Override public HealthCheckResponse call() {
return HealthCheckResponse.up("Auction Monitor is alive");
}
}
@Readiness
@ApplicationScoped
public static class ReadinessCheck
implements HealthCheck {
@Inject DatabaseService db;
@Override
public HealthCheckResponse call() {
try {
var auctions = db.getAllAuctions();
var dbPath = Paths.get("C:\\mnt\\okcomputer\\output\\cache.db");
if (!Files.exists(dbPath.getParent())) {
return HealthCheckResponse.down("Database directory does not exist");
}
return HealthCheckResponse.named("database")
.up()
.withData("auctions", auctions.size())
.build();
} catch (Exception e) {
return HealthCheckResponse.named("database")
.down()
.withData("error", e.getMessage())
.build();
}
}
}
@Startup
@ApplicationScoped
public static class StartupCheck
implements HealthCheck {
@Inject DatabaseService db;
@Override
public HealthCheckResponse call() {
try {
// Verify database schema
db.ensureSchema();
return HealthCheckResponse.named("startup")
.up()
.withData("message", "Database schema initialized")
.build();
} catch (Exception e) {
return HealthCheckResponse.named("startup")
.down()
.withData("error", e.getMessage())
.build();
}
}
}
}

View File

@@ -0,0 +1,61 @@
package auctiora;
import io.quarkus.runtime.Startup;
import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.inject.Produces;
import jakarta.inject.Singleton;
import nu.pattern.OpenCV;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import org.opencv.core.Core;
import java.io.IOException;
import java.sql.SQLException;
/**
* CDI Producer for auction monitor services.
* Creates and configures singleton instances of core services.
*/
@Startup
@ApplicationScoped
public class AuctionMonitorProducer {
private static final Logger LOG = Logger.getLogger(AuctionMonitorProducer.class);
@PostConstruct void init() {
try {
OpenCV.loadLocally();
LOG.info("✓ OpenCV loaded successfully");
} catch (Exception e) {
LOG.warn("⚠️ OpenCV not available - image detection will be disabled: " + e.getMessage());
}
}
@Produces @Singleton public DatabaseService produceDatabaseService(
@ConfigProperty(name = "auction.database.path") String dbPath) throws SQLException {
var db = new DatabaseService(dbPath);
db.ensureSchema();
return db;
}
@Produces @Singleton public NotificationService produceNotificationService(
@ConfigProperty(name = "auction.notification.config") String config) {
return new NotificationService(config);
}
@Produces @Singleton public ObjectDetectionService produceObjectDetectionService(
@ConfigProperty(name = "auction.yolo.config") String cfgPath,
@ConfigProperty(name = "auction.yolo.weights") String weightsPath,
@ConfigProperty(name = "auction.yolo.classes") String classesPath) throws IOException {
return new ObjectDetectionService(cfgPath, weightsPath, classesPath);
}
@Produces @Singleton public ImageProcessingService produceImageProcessingService(
DatabaseService db,
ObjectDetectionService detector) {
return new ImageProcessingService(db, detector);
}
}

View File

@@ -0,0 +1,925 @@
package auctiora;
import jakarta.inject.Inject;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.jboss.logging.Logger;
import java.time.LocalDateTime;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.stream.Collectors;
/**
* REST API for Auction Monitor control and status.
* Provides endpoints for:
* - Status checking
* - Manual workflow triggers
* - Statistics
*/
@Path("/api/monitor")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public class AuctionMonitorResource {
private static final Logger LOG = Logger.getLogger(AuctionMonitorResource.class);
@Inject DatabaseService db;
@Inject QuarkusWorkflowScheduler scheduler;
@Inject NotificationService notifier;
@Inject RateLimitedHttpClient httpClient;
@Inject LotEnrichmentService enrichmentService;
/**
* GET /api/monitor/status
* Returns current monitoring status
*/
@GET
@Path("/status")
public Response getStatus() {
try {
Map<String, Object> status = new HashMap<>();
status.put("running", true);
status.put("auctions", db.getAllAuctions().size());
status.put("lots", db.getAllLots().size());
status.put("images", db.getImageCount());
// Count closing soon (within 30 minutes, excluding already-closed)
var closingSoon = 0;
for (var lot : db.getAllLots()) {
if (lot.closingTime() != null) {
long minutes = lot.minutesUntilClose();
if (minutes > 0 && minutes < 30) {
closingSoon++;
}
}
}
status.put("closingSoon", closingSoon);
return Response.ok(status).build();
} catch (Exception e) {
LOG.error("Failed to get status", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/statistics
* Returns detailed statistics
*/
@GET
@Path("/statistics")
public Response getStatistics() {
try {
Map<String, Object> stats = new HashMap<>();
var auctions = db.getAllAuctions();
var lots = db.getAllLots();
stats.put("totalAuctions", auctions.size());
stats.put("totalLots", lots.size());
stats.put("totalImages", db.getImageCount());
// Lot statistics
var activeLots = 0;
var lotsWithBids = 0;
double totalBids = 0;
var hotLots = 0;
var sleeperLots = 0;
var bargainLots = 0;
var lotsClosing1h = 0;
var lotsClosing6h = 0;
double totalBidVelocity = 0;
int velocityCount = 0;
for (var lot : lots) {
long minutesLeft = lot.closingTime() != null ? lot.minutesUntilClose() : Long.MAX_VALUE;
if (lot.closingTime() != null && minutesLeft > 0) {
activeLots++;
// Time-based counts
if (minutesLeft < 60) lotsClosing1h++;
if (minutesLeft < 360) lotsClosing6h++;
}
if (lot.currentBid() > 0) {
lotsWithBids++;
totalBids += lot.currentBid();
}
// Intelligence metrics (require GraphQL enrichment)
if (lot.followersCount() != null && lot.followersCount() > 20) {
hotLots++;
}
if (lot.isSleeperLot()) {
sleeperLots++;
}
if (lot.isBelowEstimate()) {
bargainLots++;
}
// Bid velocity
if (lot.bidVelocity() != null && lot.bidVelocity() > 0) {
totalBidVelocity += lot.bidVelocity();
velocityCount++;
}
}
// Calculate bids per hour (average velocity across all lots with velocity data)
double bidsPerHour = velocityCount > 0 ? totalBidVelocity / velocityCount : 0;
stats.put("activeLots", activeLots);
stats.put("lotsWithBids", lotsWithBids);
stats.put("totalBidValue", String.format("€%.2f", totalBids));
stats.put("averageBid", lotsWithBids > 0 ? String.format("€%.2f", totalBids / lotsWithBids) : "€0.00");
// Bidding intelligence
stats.put("bidsPerHour", String.format("%.1f", bidsPerHour));
stats.put("hotLots", hotLots);
stats.put("sleeperLots", sleeperLots);
stats.put("bargainLots", bargainLots);
stats.put("lotsClosing1h", lotsClosing1h);
stats.put("lotsClosing6h", lotsClosing6h);
// Conversion rate
double conversionRate = activeLots > 0 ? (lotsWithBids * 100.0 / activeLots) : 0;
stats.put("conversionRate", String.format("%.1f%%", conversionRate));
return Response.ok(stats).build();
} catch (Exception e) {
LOG.error("Failed to get statistics", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/closing-soon
* Returns lots closing within the next specified hours (default: 24 hours)
*/
@GET
@Path("/closing-soon")
public Response getClosingSoon(@QueryParam("hours") @DefaultValue("24") int hours) {
try {
var lots = db.getAllLots();
var closingSoon = lots.stream()
.filter(lot -> lot.closingTime() != null)
.filter(lot -> lot.minutesUntilClose() > 0 && lot.minutesUntilClose() <= hours * 60)
.sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose()))
.limit(100)
.toList();
return Response.ok(closingSoon).build();
} catch (Exception e) {
LOG.error("Failed to get closing soon lots", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/lots/{lotId}/bid-history
* Returns bid history for a specific lot
*/
@GET
@Path("/lots/{lotId}/bid-history")
public Response getBidHistory(@PathParam("lotId") String lotId) {
try {
var history = db.getBidHistory(lotId);
return Response.ok(history).build();
} catch (Exception e) {
LOG.error("Failed to get bid history for lot {}", lotId, e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/trigger/scraper-import
* Manually trigger scraper import workflow
*/
@POST
@Path("/trigger/scraper-import")
public Response triggerScraperImport() {
try {
scheduler.importScraperData();
return Response.ok(Map.of("message", "Scraper import triggered successfully")).build();
} catch (Exception e) {
LOG.error("Failed to trigger scraper import", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/trigger/image-processing
* Manually trigger image processing workflow
*/
@POST
@Path("/trigger/image-processing")
public Response triggerImageProcessing() {
try {
scheduler.processImages();
return Response.ok(Map.of("message", "Image processing triggered successfully")).build();
} catch (Exception e) {
LOG.error("Failed to trigger image processing", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/trigger/bid-monitoring
* Manually trigger bid monitoring workflow
*/
@POST
@Path("/trigger/bid-monitoring")
public Response triggerBidMonitoring() {
try {
scheduler.monitorBids();
return Response.ok(Map.of("message", "Bid monitoring triggered successfully")).build();
} catch (Exception e) {
LOG.error("Failed to trigger bid monitoring", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/trigger/closing-alerts
* Manually trigger closing alerts workflow
*/
@POST
@Path("/trigger/closing-alerts")
public Response triggerClosingAlerts() {
try {
scheduler.checkClosingTimes();
return Response.ok(Map.of("message", "Closing alerts triggered successfully")).build();
} catch (Exception e) {
LOG.error("Failed to trigger closing alerts", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/trigger/graphql-enrichment
* Manually trigger GraphQL enrichment for all lots or lots closing soon
*/
@POST
@Path("/trigger/graphql-enrichment")
public Response triggerGraphQLEnrichment(@QueryParam("hoursUntilClose") @DefaultValue("24") int hours) {
try {
int enriched;
if (hours > 0) {
enriched = enrichmentService.enrichClosingSoonLots(hours);
return Response.ok(Map.of(
"message", "GraphQL enrichment triggered for lots closing within " + hours + " hours",
"enrichedCount", enriched
)).build();
} else {
enriched = enrichmentService.enrichAllActiveLots();
return Response.ok(Map.of(
"message", "GraphQL enrichment triggered for all lots",
"enrichedCount", enriched
)).build();
}
} catch (Exception e) {
LOG.error("Failed to trigger GraphQL enrichment", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/auctions
* Returns list of all auctions
*/
@GET
@Path("/auctions")
public Response getAuctions(@QueryParam("country") String country) {
try {
var auctions = country != null && !country.isEmpty()
? db.getAuctionsByCountry(country)
: db.getAllAuctions();
return Response.ok(auctions).build();
} catch (Exception e) {
LOG.error("Failed to get auctions", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/lots
* Returns list of active lots
*/
@GET
@Path("/lots")
public Response getActiveLots() {
try {
var lots = db.getActiveLots();
return Response.ok(lots).build();
} catch (Exception e) {
LOG.error("Failed to get lots", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/lots/closing-soon
* Returns lots closing within specified minutes (default 30)
*/
@GET
@Path("/lots/closing-soon")
public Response getLotsClosingSoon(@QueryParam("minutes") @DefaultValue("30") int minutes) {
try {
var allLots = db.getActiveLots();
var closingSoon = allLots.stream()
.filter(lot -> lot.closingTime() != null)
.filter(lot -> {
long minutesLeft = lot.minutesUntilClose();
return minutesLeft > 0 && minutesLeft < minutes;
})
.sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose()))
.toList();
return Response.ok(closingSoon).build();
} catch (Exception e) {
LOG.error("Failed to get closing lots", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/lots/{lotId}/images
* Returns images for a specific lot
*/
@GET
@Path("/lots/{lotId}/images")
public Response getLotImages(@PathParam("lotId") int lotId) {
try {
var images = db.getImagesForLot(lotId);
return Response.ok(images).build();
} catch (Exception e) {
LOG.error("Failed to get lot images", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* POST /api/monitor/test-notification
* Send a test notification
*/
@POST
@Path("/test-notification")
public Response sendTestNotification(Map<String, String> request) {
try {
var message = request.getOrDefault("message", "Test notification from Auction Monitor");
var title = request.getOrDefault("title", "Test Notification");
var priority = Integer.parseInt(request.getOrDefault("priority", "0"));
notifier.sendNotification(message, title, priority);
return Response.ok(Map.of("message", "Test notification sent successfully")).build();
} catch (Exception e) {
LOG.error("Failed to send test notification", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/rate-limit/stats
* Returns HTTP rate limiting statistics for all hosts
*/
@GET
@Path("/rate-limit/stats")
public Response getRateLimitStats() {
try {
var stats = httpClient.getAllStats();
Map<String, Object> response = new HashMap<>();
response.put("hosts", stats.size());
Map<String, Object> hostStats = new HashMap<>();
for (var entry : stats.entrySet()) {
var stat = entry.getValue();
hostStats.put(entry.getKey(), Map.of(
"totalRequests", stat.getTotalRequests(),
"successfulRequests", stat.getSuccessfulRequests(),
"failedRequests", stat.getFailedRequests(),
"rateLimitedRequests", stat.getRateLimitedRequests(),
"averageDurationMs", stat.getAverageDurationMs()
));
}
response.put("statistics", hostStats);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get rate limit stats", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/rate-limit/stats/{host}
* Returns HTTP rate limiting statistics for a specific host
*/
@GET
@Path("/rate-limit/stats/{host}")
public Response getRateLimitStatsForHost(@PathParam("host") String host) {
try {
var stat = httpClient.getStats(host);
if (stat == null) {
return Response.status(Response.Status.NOT_FOUND)
.entity(Map.of("error", "No statistics found for host: " + host))
.build();
}
Map<String, Object> response = Map.of(
"host", stat.getHost(),
"totalRequests", stat.getTotalRequests(),
"successfulRequests", stat.getSuccessfulRequests(),
"failedRequests", stat.getFailedRequests(),
"rateLimitedRequests", stat.getRateLimitedRequests(),
"averageDurationMs", stat.getAverageDurationMs()
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get rate limit stats for host", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/charts/country-distribution
* Returns dynamic country distribution for charts
*/
@GET
@Path("/charts/country-distribution")
public Response getCountryDistribution() {
try {
var auctions = db.getAllAuctions();
Map<String, Long> distribution = auctions.stream()
.filter(a -> a.country() != null && !a.country().isEmpty())
.collect(Collectors.groupingBy(
AuctionInfo::country,
Collectors.counting()
));
return Response.ok(distribution).build();
} catch (Exception e) {
LOG.error("Failed to get country distribution", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/charts/category-distribution
* Returns dynamic category distribution with intelligence for charts
*/
@GET
@Path("/charts/category-distribution")
public Response getCategoryDistribution() {
try {
var lots = db.getAllLots();
// Category distribution
Map<String, Long> distribution = lots.stream()
.filter(l -> l.category() != null && !l.category().isEmpty())
.collect(Collectors.groupingBy(
l -> l.category().length() > 20 ? l.category().substring(0, 20) + "..." : l.category(),
Collectors.counting()
));
// Find top category by count
var topCategory = distribution.entrySet().stream()
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse("N/A");
// Calculate average bids per category
Map<String, Double> avgBidsByCategory = lots.stream()
.filter(l -> l.category() != null && !l.category().isEmpty() && l.currentBid() > 0)
.collect(Collectors.groupingBy(
l -> l.category().length() > 20 ? l.category().substring(0, 20) + "..." : l.category(),
Collectors.averagingDouble(Lot::currentBid)
));
double overallAvgBid = lots.stream()
.filter(l -> l.currentBid() > 0)
.mapToDouble(Lot::currentBid)
.average()
.orElse(0.0);
Map<String, Object> response = new HashMap<>();
response.put("distribution", distribution);
response.put("topCategory", topCategory);
response.put("categoryCount", distribution.size());
response.put("averageBidOverall", String.format("€%.2f", overallAvgBid));
response.put("avgBidsByCategory", avgBidsByCategory);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get category distribution", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/charts/bidding-trend
* Returns time series data for last N hours
*/
@GET
@Path("/charts/bidding-trend")
public Response getBiddingTrend(@QueryParam("hours") @DefaultValue("24") int hours) {
try {
var lots = db.getAllLots();
Map<Integer, TrendHour> trends = new HashMap<>();
// Initialize hours
LocalDateTime now = LocalDateTime.now();
for (int i = hours - 1; i >= 0; i--) {
LocalDateTime hour = now.minusHours(i);
int hourKey = hour.getHour();
trends.put(hourKey, new TrendHour(hourKey, 0, 0));
}
// Count lots and bids per hour (mock implementation - in real app, use timestamp data)
// This is a simplified version - you'd need actual timestamps in DB
for (var lot : lots) {
if (lot.closingTime() != null) {
int hour = lot.closingTime().getHour();
TrendHour trend = trends.getOrDefault(hour, new TrendHour(hour, 0, 0));
trend.lots++;
if (lot.currentBid() > 0) trend.bids++;
}
}
return Response.ok(trends.values()).build();
} catch (Exception e) {
LOG.error("Failed to get bidding trend", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/charts/insights
* Returns intelligent insights
*/
@GET
@Path("/charts/insights")
public Response getInsights() {
try {
var lots = db.getAllLots();
var auctions = db.getAllAuctions();
List<Map<String, String>> insights = new ArrayList<>();
// Calculate insights
long criticalCount = lots.stream().filter(l -> l.minutesUntilClose() < 30).count();
if (criticalCount > 10) {
insights.add(Map.of(
"icon", "fa-exclamation-circle",
"title", criticalCount + " lots closing soon",
"description", "High urgency items require attention"
));
}
double bidRate = lots.stream().filter(l -> l.currentBid() > 0).count() * 100.0 / lots.size();
if (bidRate > 60) {
insights.add(Map.of(
"icon", "fa-chart-line",
"title", String.format("%.1f%% bid rate", bidRate),
"description", "Strong market engagement detected"
));
}
long imageCoverage = db.getImageCount() * 100 / Math.max(lots.size(), 1);
if (imageCoverage < 80) {
insights.add(Map.of(
"icon", "fa-images",
"title", imageCoverage + "% image coverage",
"description", "Consider processing more images"
));
}
// Add geographic insight (filter out null countries)
String topCountry = auctions.stream()
.filter(a -> a.country() != null)
.collect(Collectors.groupingBy(AuctionInfo::country, Collectors.counting()))
.entrySet().stream()
.max(Map.Entry.comparingByValue())
.map(Map.Entry::getKey)
.orElse("N/A");
if (!"N/A".equals(topCountry)) {
insights.add(Map.of(
"icon", "fa-globe",
"title", topCountry + " leading",
"description", "Top performing country"
));
}
// Add sleeper lots insight
long sleeperCount = lots.stream().filter(Lot::isSleeperLot).count();
if (sleeperCount > 0) {
insights.add(Map.of(
"icon", "fa-eye",
"title", sleeperCount + " sleeper lots",
"description", "High interest, low bids - opportunity?"
));
}
// Add bargain insight
long bargainCount = lots.stream().filter(Lot::isBelowEstimate).count();
if (bargainCount > 5) {
insights.add(Map.of(
"icon", "fa-tag",
"title", bargainCount + " bargains",
"description", "Priced below auction house estimates"
));
}
// Add watch/followers insight
long highWatchCount = lots.stream()
.filter(l -> l.followersCount() != null && l.followersCount() > 20)
.count();
if (highWatchCount > 0) {
insights.add(Map.of(
"icon", "fa-fire",
"title", highWatchCount + " hot lots",
"description", "High follower count, strong competition"
));
}
return Response.ok(insights).build();
} catch (Exception e) {
LOG.error("Failed to get insights", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/intelligence/sleepers
* Returns "sleeper" lots (high watch count, low bids)
*/
@GET
@Path("/intelligence/sleepers")
public Response getSleeperLots(@QueryParam("minFollowers") @DefaultValue("10") int minFollowers) {
try {
var allLots = db.getAllLots();
var sleepers = allLots.stream()
.filter(Lot::isSleeperLot)
.toList();
Map<String, Object> response = Map.of(
"count", sleepers.size(),
"lots", sleepers
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get sleeper lots", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/intelligence/bargains
* Returns lots priced below auction house estimates
*/
@GET
@Path("/intelligence/bargains")
public Response getBargains() {
try {
var allLots = db.getAllLots();
var bargains = allLots.stream()
.filter(Lot::isBelowEstimate)
.sorted((a, b) -> {
Double ratioA = a.getPriceVsEstimateRatio();
Double ratioB = b.getPriceVsEstimateRatio();
if (ratioA == null) return 1;
if (ratioB == null) return -1;
return ratioA.compareTo(ratioB);
})
.toList();
Map<String, Object> response = Map.of(
"count", bargains.size(),
"lots", bargains
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get bargains", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/intelligence/popular
* Returns lots by popularity level
*/
@GET
@Path("/intelligence/popular")
public Response getPopularLots(@QueryParam("level") @DefaultValue("HIGH") String level) {
try {
var allLots = db.getAllLots();
var popular = allLots.stream()
.filter(lot -> level.equalsIgnoreCase(lot.getPopularityLevel()))
.sorted((a, b) -> {
Integer followersA = a.followersCount() != null ? a.followersCount() : 0;
Integer followersB = b.followersCount() != null ? b.followersCount() : 0;
return followersB.compareTo(followersA);
})
.toList();
Map<String, Object> response = Map.of(
"count", popular.size(),
"level", level,
"lots", popular
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get popular lots", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/intelligence/price-analysis
* Returns price vs estimate analysis
*/
@GET
@Path("/intelligence/price-analysis")
public Response getPriceAnalysis() {
try {
var allLots = db.getAllLots();
long belowEstimate = allLots.stream().filter(Lot::isBelowEstimate).count();
long aboveEstimate = allLots.stream().filter(Lot::isAboveEstimate).count();
long withEstimates = allLots.stream()
.filter(lot -> lot.estimatedMin() != null && lot.estimatedMax() != null)
.count();
double avgPriceVsEstimate = allLots.stream()
.map(Lot::getPriceVsEstimateRatio)
.filter(ratio -> ratio != null)
.mapToDouble(Double::doubleValue)
.average()
.orElse(0.0);
Map<String, Object> response = Map.of(
"totalLotsWithEstimates", withEstimates,
"belowEstimate", belowEstimate,
"aboveEstimate", aboveEstimate,
"averagePriceVsEstimatePercent", Math.round(avgPriceVsEstimate),
"bargainOpportunities", belowEstimate
);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Failed to get price analysis", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/lots/{lotId}/intelligence
* Returns detailed intelligence for a specific lot
*/
@GET
@Path("/lots/{lotId}/intelligence")
public Response getLotIntelligence(@PathParam("lotId") long lotId) {
try {
var lot = db.getAllLots().stream()
.filter(l -> l.lotId() == lotId)
.findFirst()
.orElse(null);
if (lot == null) {
return Response.status(Response.Status.NOT_FOUND)
.entity(Map.of("error", "Lot not found"))
.build();
}
Map<String, Object> intelligence = new HashMap<>();
intelligence.put("lotId", lot.lotId());
intelligence.put("followersCount", lot.followersCount());
intelligence.put("popularityLevel", lot.getPopularityLevel());
intelligence.put("estimatedMidpoint", lot.getEstimatedMidpoint());
intelligence.put("priceVsEstimatePercent", lot.getPriceVsEstimateRatio());
intelligence.put("isBargain", lot.isBelowEstimate());
intelligence.put("isOvervalued", lot.isAboveEstimate());
intelligence.put("isSleeperLot", lot.isSleeperLot());
intelligence.put("nextBidAmount", lot.calculateNextBid());
intelligence.put("totalCostWithFees", lot.calculateTotalCost());
intelligence.put("viewCount", lot.viewCount());
intelligence.put("bidVelocity", lot.bidVelocity());
intelligence.put("condition", lot.condition());
intelligence.put("vat", lot.vat());
intelligence.put("buyerPremium", lot.buyerPremiumPercentage());
return Response.ok(intelligence).build();
} catch (Exception e) {
LOG.error("Failed to get lot intelligence", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* GET /api/monitor/charts/watch-distribution
* Returns follower/watch count distribution
*/
@GET
@Path("/charts/watch-distribution")
public Response getWatchDistribution() {
try {
var lots = db.getAllLots();
Map<String, Long> distribution = new HashMap<>();
distribution.put("0 watchers", lots.stream().filter(l -> l.followersCount() == null || l.followersCount() == 0).count());
distribution.put("1-5 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 1 && l.followersCount() <= 5).count());
distribution.put("6-20 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 6 && l.followersCount() <= 20).count());
distribution.put("21-50 watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() >= 21 && l.followersCount() <= 50).count());
distribution.put("50+ watchers", lots.stream().filter(l -> l.followersCount() != null && l.followersCount() > 50).count());
return Response.ok(distribution).build();
} catch (Exception e) {
LOG.error("Failed to get watch distribution", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
// Helper class for trend data
public static class TrendHour {
public int hour;
public int lots;
public int bids;
public TrendHour(int hour, int lots, int bids) {
this.hour = hour;
this.lots = lots;
this.bids = bids;
}
}
}

View File

@@ -0,0 +1,16 @@
package auctiora;
import java.time.LocalDateTime;
/**
* Represents a bid in the bid history
*/
public record BidHistory(
int id,
String lotId,
double bidAmount,
LocalDateTime bidTime,
boolean isAutobid,
String bidderId,
Integer bidderNumber
) {}

View File

@@ -0,0 +1,218 @@
package auctiora;
import auctiora.db.*;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Jdbi;
import java.util.List;
/**
* Refactored database service using repository pattern and JDBI3.
* Delegates operations to specialized repositories for better separation of concerns.
*
* @deprecated Legacy methods maintained for backward compatibility.
* New code should use repositories directly via dependency injection.
*/
@Slf4j
public class DatabaseService {
private final Jdbi jdbi;
private final LotRepository lotRepository;
private final AuctionRepository auctionRepository;
private final ImageRepository imageRepository;
/**
* Constructor for programmatic instantiation (tests, CLI tools).
*/
public DatabaseService(String dbPath) {
String url = "jdbc:sqlite:" + dbPath + "?journal_mode=WAL&busy_timeout=10000";
this.jdbi = Jdbi.create(url);
// Initialize schema
DatabaseSchema.ensureSchema(jdbi);
// Create repositories
this.lotRepository = new LotRepository(jdbi);
this.auctionRepository = new AuctionRepository(jdbi);
this.imageRepository = new ImageRepository(jdbi);
}
/**
* Constructor with JDBI instance (for dependency injection).
*/
public DatabaseService(Jdbi jdbi) {
this.jdbi = jdbi;
DatabaseSchema.ensureSchema(jdbi);
this.lotRepository = new LotRepository(jdbi);
this.auctionRepository = new AuctionRepository(jdbi);
this.imageRepository = new ImageRepository(jdbi);
}
// ==================== LEGACY COMPATIBILITY METHODS ====================
// These methods delegate to repositories for backward compatibility
void ensureSchema() {
DatabaseSchema.ensureSchema(jdbi);
}
synchronized void upsertAuction(AuctionInfo auction) {
auctionRepository.upsert(auction);
}
synchronized List<AuctionInfo> getAllAuctions() {
return auctionRepository.getAll();
}
synchronized List<AuctionInfo> getAuctionsByCountry(String countryCode) {
return auctionRepository.getByCountry(countryCode);
}
synchronized void upsertLot(Lot lot) {
lotRepository.upsert(lot);
}
synchronized void upsertLotWithIntelligence(Lot lot) {
lotRepository.upsertWithIntelligence(lot);
}
synchronized void updateLotCurrentBid(Lot lot) {
lotRepository.updateCurrentBid(lot);
}
synchronized void updateLotNotificationFlags(Lot lot) {
lotRepository.updateNotificationFlags(lot);
}
synchronized List<Lot> getActiveLots() {
return lotRepository.getActiveLots();
}
synchronized List<Lot> getAllLots() {
return lotRepository.getAllLots();
}
synchronized List<BidHistory> getBidHistory(String lotId) {
return lotRepository.getBidHistory(lotId);
}
synchronized void insertBidHistory(List<BidHistory> bidHistory) {
lotRepository.insertBidHistory(bidHistory);
}
synchronized void insertImage(long lotId, String url, String filePath, List<String> labels) {
imageRepository.insert(lotId, url, filePath, labels);
}
synchronized void updateImageLabels(int imageId, List<String> labels) {
imageRepository.updateLabels(imageId, labels);
}
synchronized List<String> getImageLabels(int imageId) {
return imageRepository.getLabels(imageId);
}
synchronized List<ImageRecord> getImagesForLot(long lotId) {
return imageRepository.getImagesForLot(lotId)
.stream()
.map(img -> new ImageRecord(img.id(), img.lotId(), img.url(), img.filePath(), img.labels()))
.toList();
}
synchronized List<ImageDetectionRecord> getImagesNeedingDetection() {
return imageRepository.getImagesNeedingDetection()
.stream()
.map(img -> new ImageDetectionRecord(img.id(), img.lotId(), img.filePath()))
.toList();
}
synchronized int getImageCount() {
return imageRepository.getImageCount();
}
synchronized List<AuctionInfo> importAuctionsFromScraper() {
return jdbi.withHandle(handle -> {
var sql = """
SELECT
l.auction_id,
MIN(l.title) as title,
MIN(l.location) as location,
MIN(l.url) as url,
COUNT(*) as lots_count,
MIN(l.closing_time) as first_lot_closing_time,
MIN(l.scraped_at) as scraped_at
FROM lots l
WHERE l.auction_id IS NOT NULL
GROUP BY l.auction_id
""";
return handle.createQuery(sql)
.map((rs, ctx) -> {
try {
var auction = ScraperDataAdapter.fromScraperAuction(rs);
if (auction.auctionId() != 0L) {
auctionRepository.upsert(auction);
return auction;
}
} catch (Exception e) {
log.warn("Failed to import auction: {}", e.getMessage());
}
return null;
})
.list()
.stream()
.filter(a -> a != null)
.toList();
});
}
synchronized List<Lot> importLotsFromScraper() {
return jdbi.withHandle(handle -> {
var sql = "SELECT * FROM lots";
return handle.createQuery(sql)
.map((rs, ctx) -> {
try {
var lot = ScraperDataAdapter.fromScraperLot(rs);
if (lot.lotId() != 0L && lot.saleId() != 0L) {
lotRepository.upsert(lot);
return lot;
}
} catch (Exception e) {
log.warn("Failed to import lot: {}", e.getMessage());
}
return null;
})
.list()
.stream()
.filter(l -> l != null)
.toList();
});
}
// ==================== DIRECT REPOSITORY ACCESS ====================
// Expose repositories for modern usage patterns
public LotRepository lots() {
return lotRepository;
}
public AuctionRepository auctions() {
return auctionRepository;
}
public ImageRepository images() {
return imageRepository;
}
public Jdbi getJdbi() {
return jdbi;
}
// ==================== LEGACY RECORDS ====================
// Keep records for backward compatibility with existing code
public record ImageRecord(int id, long lotId, String url, String filePath, String labels) {}
public record ImageDetectionRecord(int id, long lotId, String filePath) {}
}

View File

@@ -0,0 +1,55 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public record ImageProcessingService(DatabaseService db, ObjectDetectionService detector) {
boolean processImage(int id, String path, long lot) {
try {
path = path.replace('\\', '/');
var f = new java.io.File(path);
if (!f.exists() || !f.canRead()) {
log.warn("Image not accessible: {}", path);
return false;
}
if (f.length() > 50L * 1024 * 1024) {
log.warn("Image too large: {}", path);
return false;
}
var labels = detector.detectObjects(path);
db.updateImageLabels(id, labels);
if (!labels.isEmpty())
log.info("Lot {}: {}", lot, String.join(", ", labels));
return true;
} catch (Exception e) {
log.error("Process fail {}: {}", id, e.getMessage());
return false;
}
}
void processPendingImages() {
try {
var images = db.getImagesNeedingDetection();
log.info("Pending {}", images.size());
int processed = 0, detected = 0;
for (var i : images) {
if (processImage(i.id(), i.filePath(), i.lotId())) {
processed++;
var lbl = db.getImageLabels(i.id());
if (lbl != null && !lbl.isEmpty()) detected++;
}
}
log.info("Processed {}, detected {}", processed, detected);
} catch (Exception e) {
log.error("Batch fail: {}", e.getMessage());
}
}
}

View File

@@ -0,0 +1,153 @@
package auctiora;
import lombok.With;
import java.time.Duration;
import java.time.LocalDateTime;
/// Represents a lot (kavel) in an auction.
/// Data typically populated by the external scraper process.
/// This project enriches the data with image analysis and monitoring.
@With
public record Lot(
long saleId,
long lotId,
String displayId, // Full lot ID string (e.g., "A1-34732-49") for GraphQL queries
String title,
String description,
String manufacturer,
String type,
int year,
String category,
double currentBid,
String currency,
String url,
LocalDateTime closingTime,
boolean closingNotified,
// HIGH PRIORITY FIELDS from GraphQL API
Integer followersCount, // Watch count - direct competition indicator
Double estimatedMin, // Auction house min estimate (cents)
Double estimatedMax, // Auction house max estimate (cents)
Long nextBidStepInCents, // Exact bid increment from API
String condition, // Direct condition field
String categoryPath, // Structured category (e.g., "Vehicles > Cars > Classic")
String cityLocation, // Structured location
String countryCode, // ISO country code
// MEDIUM PRIORITY FIELDS
String biddingStatus, // More detailed than minimumBidAmountMet
String appearance, // Visual condition notes
String packaging, // Packaging details
Long quantity, // Lot quantity (bulk lots)
Double vat, // VAT percentage
Double buyerPremiumPercentage, // Buyer premium
String remarks, // Viewing/pickup notes
// BID INTELLIGENCE FIELDS
Double startingBid, // Starting/opening bid
Double reservePrice, // Reserve price (if disclosed)
Boolean reserveMet, // Reserve met status
Double bidIncrement, // Calculated bid increment
Integer viewCount, // Number of views
LocalDateTime firstBidTime, // First bid timestamp
LocalDateTime lastBidTime, // Last bid timestamp
Double bidVelocity, // Bids per hour
Double condition_score,
//Integer manufacturing_year,
Integer provenance_docs
) {
public Integer provenanceDocs() { return provenance_docs; }
/// manufacturing_year
public Integer manufacturingYear() { return year; }
public Double conditionScore() { return condition_score; }
public long minutesUntilClose() {
if (closingTime == null) return Long.MAX_VALUE;
return Duration.between(LocalDateTime.now(), closingTime).toMinutes();
}
// Intelligence Methods
/// Calculate total cost including VAT and buyer premium
public double calculateTotalCost() {
double base = currentBid > 0 ? currentBid : 0;
if (vat != null && vat > 0) {
base += (base * vat / 100.0);
}
if (buyerPremiumPercentage != null && buyerPremiumPercentage > 0) {
base += (base * buyerPremiumPercentage / 100.0);
}
return base;
}
/// Calculate next bid amount using API-provided increment
public double calculateNextBid() {
if (nextBidStepInCents != null && nextBidStepInCents > 0) {
return currentBid + (nextBidStepInCents / 100.0);
} else if (bidIncrement != null && bidIncrement > 0) {
return currentBid + bidIncrement;
}
// Fallback: 5% increment
return currentBid * 1.05;
}
/// Check if current bid is below estimate (potential bargain)
public boolean isBelowEstimate() {
if (estimatedMin == null || estimatedMin == 0) return false;
return currentBid < (estimatedMin / 100.0);
}
/// Check if current bid exceeds estimate (overvalued)
public boolean isAboveEstimate() {
if (estimatedMax == null || estimatedMax == 0) return false;
return currentBid > (estimatedMax / 100.0);
}
/// Calculate interest-to-bid conversion rate
public double getInterestToBidRatio() {
if (followersCount == null || followersCount == 0) return 0.0;
return currentBid > 0 ? 100.0 : 0.0;
}
/// Determine lot popularity level
public String getPopularityLevel() {
if (followersCount == null) return "UNKNOWN";
if (followersCount > 50) return "HIGH";
if (followersCount > 20) return "MEDIUM";
if (followersCount > 5) return "LOW";
return "MINIMAL";
}
/// Check if lot is a "sleeper" (high interest, low bids)
public boolean isSleeperLot() {
return followersCount != null && followersCount > 10 && currentBid < 100;
}
/// Calculate estimated value range midpoint
public Double getEstimatedMidpoint() {
if (estimatedMin == null || estimatedMax == null) return null;
return (estimatedMin + estimatedMax) / 200.0; // Convert from cents
}
/// Calculate price vs estimate ratio (for analytics)
public Double getPriceVsEstimateRatio() {
Double midpoint = getEstimatedMidpoint();
if (midpoint == null || midpoint == 0 || currentBid == 0) return null;
return (currentBid / midpoint) * 100.0;
}
/// Factory method for creating a basic Lot without intelligence fields (for tests and backward compatibility)
public static Lot basic(
long saleId, long lotId, String title, String description,
String manufacturer, String type, int year, String category,
double currentBid, String currency, String url,
LocalDateTime closingTime, boolean closingNotified) {
return new Lot(
saleId, lotId, null, title, description, manufacturer, type, year, category,
currentBid, currency, url, closingTime, closingNotified,
null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null,
null, null
);
}
}

View File

@@ -0,0 +1,81 @@
package auctiora;
import io.quarkus.scheduler.Scheduled;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
/**
* Scheduled tasks for enriching lots with GraphQL intelligence data.
* Uses dynamic frequencies based on lot closing times:
* - Critical (< 1 hour): Every 5 minutes
* - Urgent (< 6 hours): Every 30 minutes
* - Normal (< 24 hours): Every 2 hours
* - All lots: Every 6 hours
*/
@Slf4j
@ApplicationScoped
public class LotEnrichmentScheduler {
@Inject LotEnrichmentService enrichmentService;
/**
* Enriches lots closing within 1 hour - HIGH PRIORITY
* Runs every 5 minutes
*/
@Scheduled(cron = "0 */5 * * * ?")
public void enrichCriticalLots() {
try {
log.debug("Enriching critical lots (closing < 1 hour)");
int enriched = enrichmentService.enrichClosingSoonLots(1);
if (enriched > 0) log.info("Enriched {} critical lots", enriched);
} catch (Exception e) {
log.error("Failed to enrich critical lots", e);
}
}
/**
* Enriches lots closing within 6 hours - MEDIUM PRIORITY
* Runs every 30 minutes
*/
@Scheduled(cron = "0 */30 * * * ?")
public void enrichUrgentLots() {
try {
log.debug("Enriching urgent lots (closing < 6 hours)");
int enriched = enrichmentService.enrichClosingSoonLots(6);
if (enriched > 0) log.info("Enriched {} urgent lots", enriched);
} catch (Exception e) {
log.error("Failed to enrich urgent lots", e);
}
}
/**
* Enriches lots closing within 24 hours - NORMAL PRIORITY
* Runs every 2 hours
*/
@Scheduled(cron = "0 0 */2 * * ?")
public void enrichDailyLots() {
try {
log.debug("Enriching daily lots (closing < 24 hours)");
int enriched = enrichmentService.enrichClosingSoonLots(24);
if (enriched > 0) log.info("Enriched {} daily lots", enriched);
} catch (Exception e) {
log.error("Failed to enrich daily lots", e);
}
}
/**
* Enriches all active lots - LOW PRIORITY
* Runs every 6 hours to keep all data fresh
*/
@Scheduled(cron = "0 0 */6 * * ?")
public void enrichAllLots() {
try {
log.info("Starting full enrichment of all lots");
int enriched = enrichmentService.enrichAllActiveLots();
log.info("Full enrichment complete: {} lots updated", enriched);
} catch (Exception e) {
log.error("Failed to enrich all lots", e);
}
}
}

View File

@@ -0,0 +1,201 @@
package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Service for enriching lots with intelligence data from GraphQL API.
* Updates existing lot records with followers, estimates, velocity, etc.
*/
@Slf4j
@ApplicationScoped
public class LotEnrichmentService {
@Inject TroostwijkGraphQLClient graphQLClient;
@Inject DatabaseService db;
/**
* Enriches a single lot with GraphQL intelligence data
*/
public boolean enrichLot(Lot lot) {
if (lot.displayId() == null || lot.displayId().isBlank()) {
log.debug("Cannot enrich lot {} - missing displayId", lot.lotId());
return false;
}
try {
var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId());
if (intelligence == null) {
log.debug("No intelligence data for lot {}", lot.displayId());
return false;
}
// Merge intelligence with existing lot data
var enrichedLot = mergeLotWithIntelligence(lot, intelligence);
db.upsertLotWithIntelligence(enrichedLot);
log.debug("Enriched lot {} with GraphQL data", lot.lotId());
return true;
} catch (Exception e) {
log.warn("Failed to enrich lot {}: {}", lot.lotId(), e.getMessage());
return false;
}
}
/**
* Enriches multiple lots sequentially
* @param lots List of lots to enrich
* @return Number of successfully enriched lots
*/
public int enrichLotsBatch(List<Lot> lots) {
if (lots.isEmpty()) {
return 0;
}
log.info("Enriching {} lots via GraphQL", lots.size());
int enrichedCount = 0;
for (var lot : lots) {
if (lot.displayId() == null || lot.displayId().isBlank()) {
log.debug("Skipping lot {} - missing displayId", lot.lotId());
continue;
}
try {
var intelligence = graphQLClient.fetchLotIntelligence(lot.displayId(), lot.lotId());
if (intelligence != null) {
var enrichedLot = mergeLotWithIntelligence(lot, intelligence);
db.upsertLotWithIntelligence(enrichedLot);
enrichedCount++;
} else {
log.debug("No intelligence data for lot {}", lot.displayId());
}
} catch (Exception e) {
log.warn("Failed to enrich lot {}: {}", lot.displayId(), e.getMessage());
}
// Small delay to respect rate limits (handled by RateLimitedHttpClient)
}
log.info("Successfully enriched {}/{} lots", enrichedCount, lots.size());
return enrichedCount;
}
/**
* Enriches lots closing soon (within specified hours) with higher priority
*/
public int enrichClosingSoonLots(int hoursUntilClose) {
try {
var allLots = db.getAllLots();
var closingSoon = allLots.stream()
.filter(lot -> lot.closingTime() != null)
.filter(lot -> {
long minutes = lot.minutesUntilClose();
return minutes > 0 && minutes <= hoursUntilClose * 60;
})
.toList();
if (closingSoon.isEmpty()) {
log.debug("No lots closing within {} hours", hoursUntilClose);
return 0;
}
log.info("Enriching {} lots closing within {} hours", closingSoon.size(), hoursUntilClose);
return enrichLotsBatch(closingSoon);
} catch (Exception e) {
log.error("Failed to enrich closing soon lots: {}", e.getMessage());
return 0;
}
}
/**
* Enriches all active lots (can be slow for large datasets)
*/
public int enrichAllActiveLots() {
try {
var allLots = db.getAllLots();
log.info("Enriching all {} active lots", allLots.size());
// Process in batches to avoid overwhelming the API
int batchSize = 50;
int totalEnriched = 0;
for (int i = 0; i < allLots.size(); i += batchSize) {
int end = Math.min(i + batchSize, allLots.size());
List<Lot> batch = allLots.subList(i, end);
int enriched = enrichLotsBatch(batch);
totalEnriched += enriched;
// Small delay between batches to respect rate limits
if (end < allLots.size()) {
Thread.sleep(1000);
}
}
log.info("Finished enriching all lots. Total enriched: {}/{}", totalEnriched, allLots.size());
return totalEnriched;
} catch (Exception e) {
log.error("Failed to enrich all lots: {}", e.getMessage());
return 0;
}
}
/**
* Merges existing lot data with GraphQL intelligence
*/
private Lot mergeLotWithIntelligence(Lot lot, LotIntelligence intel) {
return new Lot(
lot.saleId(),
lot.lotId(),
lot.displayId(), // Preserve displayId
lot.title(),
lot.description(),
lot.manufacturer(),
lot.type(),
lot.year(),
lot.category(),
lot.currentBid(),
lot.currency(),
lot.url(),
lot.closingTime(),
lot.closingNotified(),
// HIGH PRIORITY FIELDS from GraphQL
intel.followersCount(),
intel.estimatedMin(),
intel.estimatedMax(),
intel.nextBidStepInCents(),
intel.condition(),
intel.categoryPath(),
intel.cityLocation(),
intel.countryCode(),
// MEDIUM PRIORITY FIELDS
intel.biddingStatus(),
intel.appearance(),
intel.packaging(),
intel.quantity(),
intel.vat(),
intel.buyerPremiumPercentage(),
intel.remarks(),
// BID INTELLIGENCE FIELDS
intel.startingBid(),
intel.reservePrice(),
intel.reserveMet(),
intel.bidIncrement(),
intel.viewCount(),
intel.firstBidTime(),
intel.lastBidTime(),
intel.bidVelocity(),
null, // condition_score (computed separately)
null // provenance_docs (computed separately)
);
}
}

View File

@@ -0,0 +1,33 @@
package auctiora;
import java.time.LocalDateTime;
/**
* Record holding enriched intelligence data fetched from GraphQL API
*/
public record LotIntelligence(
long lotId,
Integer followersCount,
Double estimatedMin,
Double estimatedMax,
Long nextBidStepInCents,
String condition,
String categoryPath,
String cityLocation,
String countryCode,
String biddingStatus,
String appearance,
String packaging,
Long quantity,
Double vat,
Double buyerPremiumPercentage,
String remarks,
Double startingBid,
Double reservePrice,
Boolean reserveMet,
Double bidIncrement,
Integer viewCount,
LocalDateTime firstBidTime,
LocalDateTime lastBidTime,
Double bidVelocity
) {}

View File

@@ -0,0 +1,110 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import javax.mail.*;
import javax.mail.internet.*;
import java.awt.*;
import java.util.Date;
import java.util.Properties;
@Slf4j
public record NotificationService(Config cfg) {
// Extra convenience constructor: raw string → Config
public NotificationService(String raw) {
this(Config.parse(raw));
}
public void sendNotification(String msg, String title, int prio) {
if (cfg.useDesktop()) sendDesktop(title, msg, prio);
if (cfg.useEmail()) sendEmail(title, msg, prio);
}
private void sendDesktop(String title, String msg, int prio) {
try {
if (!SystemTray.isSupported()) {
log.info("Desktop not supported: {}", title);
return;
}
var tray = SystemTray.getSystemTray();
var icon = new TrayIcon(
Toolkit.getDefaultToolkit().createImage(new byte[0]),
"notify"
);
icon.setImageAutoSize(true);
tray.add(icon);
var type = prio > 0 ? TrayIcon.MessageType.WARNING : TrayIcon.MessageType.INFO;
icon.displayMessage(title, msg, type);
Thread.sleep(2000);
tray.remove(icon);
log.info("Desktop notification: {}", title);
} catch (Exception e) {
log.warn("Desktop failed: {}", e.getMessage());
}
}
private void sendEmail(String title, String msg, int prio) {
try {
var props = new Properties();
props.put("mail.smtp.auth", "true");
props.put("mail.smtp.starttls.enable", "true");
props.put("mail.smtp.host", "smtp.gmail.com");
props.put("mail.smtp.port", "587");
props.put("mail.smtp.ssl.trust", "smtp.gmail.com");
var session = Session.getInstance(props, new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(cfg.smtpUsername(), cfg.smtpPassword());
}
});
var m = new MimeMessage(session);
m.setFrom(new InternetAddress(cfg.smtpUsername()));
m.setRecipients(Message.RecipientType.TO, InternetAddress.parse(cfg.toEmail()));
m.setSubject("[Troostwijk] " + title);
m.setText(msg);
m.setSentDate(new Date());
if (prio > 0) {
m.setHeader("X-Priority", "1");
m.setHeader("Importance", "High");
}
Transport.send(m);
log.info("Email notification: {}", title);
} catch (Exception e) {
log.warn("Email failed: {}", e.getMessage());
}
}
public record Config(
boolean useDesktop,
boolean useEmail,
String smtpUsername,
String smtpPassword,
String toEmail
) {
public static Config parse(String raw) {
if ("desktop".equalsIgnoreCase(raw)) {
return new Config(true, false, null, null, null);
}
if (raw != null && raw.startsWith("smtp:")) {
var p = raw.split(":", -1);
if (p.length < 4) {
throw new IllegalArgumentException("Format: smtp:username:password:toEmail");
}
return new Config(true, true, p[1], p[2], p[3]);
}
throw new IllegalArgumentException("Use 'desktop' or 'smtp:username:password:toEmail'");
}
}
}

View File

@@ -0,0 +1,244 @@
package auctiora;
import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import lombok.extern.slf4j.Slf4j;
import nu.pattern.OpenCV;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgcodecs.Imgcodecs;
import java.io.Console;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import static org.opencv.dnn.Dnn.DNN_BACKEND_OPENCV;
import static org.opencv.dnn.Dnn.DNN_TARGET_CPU;
/**
* Service for performing object detection on images using OpenCV's DNN
* module. The DNN module can load pretrained models from several
* frameworks (Darknet, TensorFlow, ONNX, etc.). Here
* we load a YOLO model (Darknet) by specifying the configuration and
* weights files. For each image we run a forward pass and return a
* list of detected class labels.
*
* If model files are not found, the service operates in disabled mode
* and returns empty lists.
*/
@Slf4j
public class ObjectDetectionService {
private Net net;
private List<String> classNames;
private boolean enabled;
private int warnCount = 0;
private static final int MAX_WARNINGS = 5;
private static boolean openCvLoaded = false;
private final String cfgPath;
private final String weightsPath;
private final String classNamesPath;
ObjectDetectionService(String cfgPath, String weightsPath, String classNamesPath) throws IOException {
this.cfgPath = cfgPath;
this.weightsPath = weightsPath;
this.classNamesPath = classNamesPath;
}
@PostConstruct
void init() {
// Load OpenCV native libraries first
if (!openCvLoaded) {
try {
OpenCV.loadLocally();
openCvLoaded = true;
log.info("✓ OpenCV {} loaded successfully", Core.VERSION);
} catch (Exception e) {
log.warn("⚠️ Object detection disabled: OpenCV native libraries not loaded");
enabled = false;
net = null;
classNames = new ArrayList<>();
return;
}
}
initializeModel();
}
private void initializeModel() {
// Check if model files exist
var cfgFile = Paths.get(cfgPath);
var weightsFile = Paths.get(weightsPath);
var classNamesFile = Paths.get(classNamesPath);
if (!Files.exists(cfgFile) || !Files.exists(weightsFile) || !Files.exists(classNamesFile)) {
log.info("⚠️ Object detection disabled: YOLO model files not found");
log.info(" Expected files:");
log.info(" - {}", cfgPath);
log.info(" - {}", weightsPath);
log.info(" - {}", classNamesPath);
log.info(" Scraper will continue without image analysis.");
enabled = false;
net = null;
classNames = new ArrayList<>();
return;
}
try {
// Load network
net = Dnn.readNetFromDarknet(cfgPath, weightsPath);
// Try to use GPU/CUDA if available, fallback to CPU
try {
net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA);
net.setPreferableTarget(Dnn.DNN_TARGET_CUDA);
log.info("✓ Object detection enabled with YOLO (CUDA/GPU acceleration)");
} catch (Exception e) {
// CUDA not available, try Vulkan for AMD GPUs
try {
net.setPreferableBackend(Dnn.DNN_BACKEND_VKCOM);
net.setPreferableTarget(Dnn.DNN_TARGET_VULKAN);
log.info("✓ Object detection enabled with YOLO (Vulkan/GPU acceleration)");
} catch (Exception e2) {
// GPU not available, fallback to CPU
net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(DNN_TARGET_CPU);
log.info("✓ Object detection enabled with YOLO (CPU only)");
}
}
// Load class names (one per line)
classNames = Files.readAllLines(classNamesFile);
enabled = true;
} catch (UnsatisfiedLinkError e) {
log.error("⚠️ Object detection disabled: OpenCV native libraries not loaded", e);
enabled = false;
net = null;
classNames = new ArrayList<>();
} catch (Exception e) {
log.error("⚠️ Object detection disabled: " + e.getMessage(), e);
enabled = false;
net = null;
classNames = new ArrayList<>();
}
}
/**
* Detects objects in the given image file and returns a list of
* humanreadable labels. Only detections above a confidence
* threshold are returned. For brevity this method omits drawing
* bounding boxes. See the OpenCV DNN documentation for details on
* postprocessing【784097309529506†L324-L344】.
*
* @param imagePath absolute path to the image
* @return list of detected class names (empty if detection disabled)
*/
List<String> detectObjects(String imagePath) {
if (!enabled) {
return new ArrayList<>();
}
List<String> labels = new ArrayList<>();
var image = Imgcodecs.imread(imagePath);
if (image.empty()) return labels;
// Create a 4D blob from the image
var blob = Dnn.blobFromImage(image, 1.0 / 255.0, new Size(416, 416), new Scalar(0, 0, 0), true, false);
net.setInput(blob);
List<Mat> outs = new ArrayList<>();
var outNames = getOutputLayerNames(net);
net.forward(outs, outNames);
// Postprocess: for each detection compute score and choose class
var confThreshold = 0.5f;
for (var out : outs) {
// YOLO output shape: [num_detections, 85] where 85 = 4 (bbox) + 1 (objectness) + 80 (classes)
int numDetections = out.rows();
int numElements = out.cols();
int expectedLength = 5 + classNames.size();
if (numElements < expectedLength) {
// Rate-limit warnings to prevent thread blocking from excessive logging
if (warnCount < MAX_WARNINGS) {
log.warn("Output matrix has wrong dimensions: expected {} columns, got {}. Output shape: [{}, {}]",
expectedLength, numElements, numDetections, numElements);
warnCount++;
if (warnCount == MAX_WARNINGS) {
log.warn("Suppressing further dimension warnings (reached {} warnings)", MAX_WARNINGS);
}
}
continue;
}
for (var i = 0; i < numDetections; i++) {
// Get entire row (all 85 elements)
var data = new double[numElements];
for (int j = 0; j < numElements; j++) {
data[j] = out.get(i, j)[0];
}
// Extract objectness score (index 4) and class scores (index 5+)
double objectness = data[4];
if (objectness < confThreshold) {
continue; // Skip low-confidence detections
}
// Extract class scores
var scores = new double[classNames.size()];
System.arraycopy(data, 5, scores, 0, Math.min(scores.length, data.length - 5));
var classId = argMax(scores);
var confidence = scores[classId] * objectness; // Combine objectness with class confidence
if (confidence > confThreshold) {
var label = classNames.get(classId);
if (!labels.contains(label)) {
labels.add(label);
}
}
}
}
// Release resources
image.release();
blob.release();
for (var out : outs) {
out.release();
}
return labels;
}
/**
* Returns the indexes of the output layers in the network. YOLO
* automatically discovers its output layers; other models may require
* manually specifying them【784097309529506†L356-L365】.
*/
private List<String> getOutputLayerNames(Net net) {
List<String> names = new ArrayList<>();
var outLayers = net.getUnconnectedOutLayers().toList();
var layersNames = net.getLayerNames();
for (var i : outLayers) {
names.add(layersNames.get(i - 1));
}
return names;
}
/**
* Returns the index of the maximum value in the array.
*/
private int argMax(double[] array) {
var best = 0;
var max = array[0];
for (var i = 1; i < array.length; i++) {
if (array[i] > max) {
max = array[i];
best = i;
}
}
return best;
}
}

View File

@@ -0,0 +1,309 @@
package auctiora;
import io.quarkus.runtime.StartupEvent;
import io.quarkus.scheduler.Scheduled;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import java.util.List;
/**
* Quarkus-based Workflow Scheduler using @Scheduled annotations.
* Replaces the manual ScheduledExecutorService with Quarkus Scheduler.
*
* This class coordinates all scheduled workflows using Quarkus's built-in
* scheduling capabilities with cron expressions.
*/
@ApplicationScoped
public class QuarkusWorkflowScheduler {
private static final Logger LOG = Logger.getLogger(QuarkusWorkflowScheduler.class);
@Inject DatabaseService db;
@Inject NotificationService notifier;
@Inject ObjectDetectionService detector;
@Inject ImageProcessingService imageProcessor;
@Inject LotEnrichmentService enrichmentService;
@ConfigProperty(name = "auction.database.path") String databasePath;
/**
* Triggered on application startup to enrich existing lots with bid intelligence
*/
void onStart(@Observes StartupEvent ev) {
LOG.info("🚀 Application started - triggering initial lot enrichment...");
// Run enrichment in background thread to not block startup
new Thread(() -> {
try {
Thread.sleep(5000); // Wait 5 seconds for application to fully start
LOG.info("Starting full lot enrichment in background...");
int enriched = enrichmentService.enrichAllActiveLots();
LOG.infof("✓ Startup enrichment complete: %d lots enriched", enriched);
} catch (Exception e) {
LOG.errorf(e, "❌ Startup enrichment failed: %s", e.getMessage());
}
}).start();
}
/**
* Workflow 1: Import Scraper Data
* Cron: Every 30 minutes (0 -/30 - - - ?)
* Purpose: Import new auctions and lots from external scraper
*/
@Scheduled(cron = "{auction.workflow.scraper-import.cron}", identity = "scraper-import")
void importScraperData() {
try {
LOG.info("📥 [WORKFLOW 1] Importing scraper data...");
var start = System.currentTimeMillis();
// Import auctions
var auctions = db.importAuctionsFromScraper();
LOG.infof(" → Imported %d auctions", auctions.size());
// Import lots
var lots = db.importLotsFromScraper();
LOG.infof(" → Imported %d lots", lots.size());
// Check for images needing detection
var images = db.getImagesNeedingDetection();
LOG.infof(" → Found %d images needing detection", images.size());
var duration = System.currentTimeMillis() - start;
LOG.infof(" ✓ Scraper import completed in %dms", duration);
// Trigger notification if significant data imported
if (auctions.size() > 0 || lots.size() > 10) {
notifier.sendNotification(
String.format("Imported %d auctions, %d lots", auctions.size(), lots.size()),
"Data Import Complete",
0
);
}
} catch (Exception e) {
LOG.errorf(e, " ❌ Scraper import failed: %s", e.getMessage());
}
}
/**
* Workflow 2: Process Pending Images
* Cron: Every 1 hour (0 0 * * * ?)
* Purpose: Run object detection on images already downloaded by scraper
*/
@Scheduled(cron = "{auction.workflow.image-processing.cron}", identity = "image-processing")
void processImages() {
try {
LOG.info("🖼️ [WORKFLOW 2] Processing pending images...");
var start = System.currentTimeMillis();
// Get images that have been downloaded but need object detection
var pendingImages = db.getImagesNeedingDetection();
if (pendingImages.isEmpty()) {
LOG.info(" → No pending images to process");
return;
}
// Limit batch size to prevent thread blocking (max 100 images per run)
final int MAX_BATCH_SIZE = 100;
int totalPending = pendingImages.size();
if (totalPending > MAX_BATCH_SIZE) {
LOG.infof(" → Found %d pending images, processing first %d (batch limit)",
totalPending, MAX_BATCH_SIZE);
pendingImages = pendingImages.subList(0, MAX_BATCH_SIZE);
} else {
LOG.infof(" → Processing %d images", totalPending);
}
var processed = 0;
var detected = 0;
var failed = 0;
for (var image : pendingImages) {
try {
// Run object detection on already-downloaded image
if (imageProcessor.processImage(image.id(), image.filePath(), image.lotId())) {
processed++;
// Check if objects were detected
var labels = db.getImageLabels(image.id());
if (labels != null && !labels.isEmpty()) {
detected++;
// Send notification for interesting detections
if (labels.size() >= 3) {
notifier.sendNotification(
String.format("Lot %d: Detected %s",
image.lotId(),
String.join(", ", labels)),
"Objects Detected",
0
);
}
}
} else {
failed++;
}
// Rate limiting (lighter since no network I/O)
Thread.sleep(100);
} catch (Exception e) {
failed++;
LOG.warnf(" ⚠️ Failed to process image: %s", e.getMessage());
}
}
var duration = System.currentTimeMillis() - start;
LOG.infof(" ✓ Processed %d/%d images, detected objects in %d, failed %d (%.1fs)",
processed, totalPending, detected, failed, duration / 1000.0);
if (totalPending > MAX_BATCH_SIZE) {
LOG.infof(" → %d images remaining for next run", totalPending - MAX_BATCH_SIZE);
}
} catch (Exception e) {
LOG.errorf(e, " ❌ Image processing failed: %s", e.getMessage());
}
}
/**
* Workflow 3: Monitor Bids
* Cron: Every 15 minutes (0 -/15 * * * ?)
* Purpose: Check for bid changes and send notifications
*/
@Scheduled(cron = "{auction.workflow.bid-monitoring.cron}", identity = "bid-monitoring")
void monitorBids() {
try {
LOG.info("💰 [WORKFLOW 3] Monitoring bids...");
var start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
LOG.infof(" → Checking %d active lots", activeLots.size());
// Note: In production, this would call Troostwijk API
// For now, we just track what's in the database
// The external scraper updates bids, we just notify
var duration = System.currentTimeMillis() - start;
LOG.infof(" ✓ Bid monitoring completed in %dms", duration);
} catch (Exception e) {
LOG.errorf(e, " ❌ Bid monitoring failed: %s", e.getMessage());
}
}
/**
* Workflow 4: Check Closing Times
* Cron: Every 5 minutes (0 -/5 * * * ?)
* Purpose: Send alerts for lots closing soon
*/
@Scheduled(cron = "{auction.workflow.closing-alerts.cron}", identity = "closing-alerts")
void checkClosingTimes() {
try {
LOG.info("⏰ [WORKFLOW 4] Checking closing times...");
var start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
var alertsSent = 0;
for (var lot : activeLots) {
if (lot.closingTime() == null) continue;
var minutesLeft = lot.minutesUntilClose();
// Alert for lots closing in 5 minutes
if (minutesLeft <= 5 && minutesLeft > 0 && !lot.closingNotified()) {
var message = String.format("Kavel %d sluit binnen %d min.",
lot.lotId(), minutesLeft);
notifier.sendNotification(message, "Lot Closing Soon", 1);
// Mark as notified
var updated = Lot.basic(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
lot.currentBid(), lot.currency(), lot.url(),
lot.closingTime(), true
);
db.updateLotNotificationFlags(updated);
alertsSent++;
}
}
var duration = System.currentTimeMillis() - start;
LOG.infof(" → Sent %d closing alerts in %dms", alertsSent, duration);
} catch (Exception e) {
LOG.errorf(e, " ❌ Closing alerts failed: %s", e.getMessage());
}
}
/**
* Event-driven trigger: New auction discovered
*/
public void onNewAuctionDiscovered(AuctionInfo auction) {
LOG.infof("📣 EVENT: New auction discovered - %s", auction.title());
try {
db.upsertAuction(auction);
notifier.sendNotification(
String.format("New auction: %s\nLocation: %s\nLots: %d",
auction.title(), auction.location(), auction.lotCount()),
"New Auction Discovered",
0
);
} catch (Exception e) {
LOG.errorf(e, " ❌ Failed to handle new auction: %s", e.getMessage());
}
}
/**
* Event-driven trigger: Bid change detected
*/
public void onBidChange(Lot lot, double previousBid, double newBid) {
LOG.infof("📣 EVENT: Bid change on lot %d (€%.2f → €%.2f)",
lot.lotId(), previousBid, newBid);
try {
db.updateLotCurrentBid(lot);
notifier.sendNotification(
String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previousBid),
"Kavel Bieding Update",
0
);
} catch (Exception e) {
LOG.errorf(e, " ❌ Failed to handle bid change: %s", e.getMessage());
}
}
/**
* Event-driven trigger: Objects detected in image
*/
public void onObjectsDetected(int lotId, List<String> labels) {
LOG.infof("📣 EVENT: Objects detected in lot %d - %s",
lotId, String.join(", ", labels));
try {
if (labels.size() >= 2) {
notifier.sendNotification(
String.format("Lot %d contains: %s", lotId, String.join(", ", labels)),
"Objects Detected",
0
);
}
} catch (Exception e) {
LOG.errorf(e, " ❌ Failed to send detection notification: %s", e.getMessage());
}
}
}

View File

@@ -0,0 +1,246 @@
package auctiora;
import jakarta.enterprise.context.ApplicationScoped;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/**
* Rate-limited HTTP client that enforces per-host request limits.
*
* Features:
* - Per-host rate limiting (configurable max requests per second)
* - Request counting and monitoring
* - Thread-safe using semaphores
* - Automatic host extraction from URLs
*
* This prevents overloading external services like Troostwijk and getting blocked.
*/
@ApplicationScoped
public class RateLimitedHttpClient {
private static final Logger LOG = Logger.getLogger(RateLimitedHttpClient.class);
private final HttpClient httpClient;
private final Map<String, RateLimiter> rateLimiters;
private final Map<String, RequestStats> requestStats;
@ConfigProperty(name = "auction.http.rate-limit.default-max-rps", defaultValue = "2")
int defaultMaxRequestsPerSecond;
@ConfigProperty(name = "auction.http.rate-limit.troostwijk-max-rps", defaultValue = "1")
int troostwijkMaxRequestsPerSecond;
@ConfigProperty(name = "auction.http.timeout-seconds", defaultValue = "30")
int timeoutSeconds;
public RateLimitedHttpClient() {
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(30))
.build();
this.rateLimiters = new ConcurrentHashMap<>();
this.requestStats = new ConcurrentHashMap<>();
}
/**
* Sends a GET request with automatic rate limiting based on host.
*/
public HttpResponse<String> sendGet(String url) throws IOException, InterruptedException {
var request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofString());
}
/**
* Sends a request for binary data (like images) with rate limiting.
*/
public HttpResponse<byte[]> sendGetBytes(String url) throws IOException, InterruptedException {
var request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.GET()
.build();
return send(request, HttpResponse.BodyHandlers.ofByteArray());
}
/**
* Sends any HTTP request with automatic rate limiting.
*/
public <T> HttpResponse<T> send(HttpRequest request, HttpResponse.BodyHandler<T> bodyHandler)
throws IOException, InterruptedException {
var host = extractHost(request.uri());
var limiter = getRateLimiter(host);
var stats = getRequestStats(host);
// Enforce rate limit (blocks if necessary)
limiter.acquire();
// Track request
stats.incrementTotal();
var startTime = System.currentTimeMillis();
try {
var response = httpClient.send(request, bodyHandler);
var duration = System.currentTimeMillis() - startTime;
stats.recordSuccess(duration);
LOG.debugf("HTTP %d %s %s (%dms)",
response.statusCode(), request.method(), host, duration);
// Track rate limit violations (429 = Too Many Requests)
if (response.statusCode() == 429) {
stats.incrementRateLimited();
LOG.warnf("⚠️ Rate limited by %s (HTTP 429)", host);
}
return response;
} catch (IOException | InterruptedException e) {
stats.incrementFailed();
LOG.warnf("❌ HTTP request failed for %s: %s", host, e.getMessage());
throw e;
}
}
/**
* Gets or creates a rate limiter for a specific host.
*/
private RateLimiter getRateLimiter(String host) {
return rateLimiters.computeIfAbsent(host, h -> {
var maxRps = getMaxRequestsPerSecond(h);
LOG.infof("Initializing rate limiter for %s: %d req/s", h, maxRps);
return new RateLimiter(maxRps);
});
}
/**
* Gets or creates request stats for a specific host.
*/
private RequestStats getRequestStats(String host) {
return requestStats.computeIfAbsent(host, h -> new RequestStats(h));
}
/**
* Determines max requests per second for a given host.
*/
private int getMaxRequestsPerSecond(String host) {
return host.contains("troostwijk") ? troostwijkMaxRequestsPerSecond : defaultMaxRequestsPerSecond;
}
private String extractHost(URI uri) {
return uri.getHost() != null ? uri.getHost() : uri.toString();
}
public Map<String, RequestStats> getAllStats() {
return Map.copyOf(requestStats);
}
public RequestStats getStats(String host) {
return requestStats.get(host);
}
/**
* Rate limiter implementation using token bucket algorithm.
* Allows burst traffic up to maxRequestsPerSecond, then enforces steady rate.
*/
private static class RateLimiter {
private final Semaphore semaphore;
private final int maxRequestsPerSecond;
private final long intervalNanos;
RateLimiter(int maxRequestsPerSecond) {
this.maxRequestsPerSecond = maxRequestsPerSecond;
this.intervalNanos = TimeUnit.SECONDS.toNanos(1) / maxRequestsPerSecond;
this.semaphore = new Semaphore(maxRequestsPerSecond);
// Refill tokens periodically
startRefillThread();
}
void acquire() throws InterruptedException {
semaphore.acquire();
// Enforce minimum delay between requests
var delayMillis = intervalNanos / 1_000_000;
if (delayMillis > 0) {
Thread.sleep(delayMillis);
}
}
private void startRefillThread() {
var refillThread = new Thread(() -> {
while (!Thread.currentThread().isInterrupted()) {
try {
Thread.sleep(1000); // Refill every second
var toRelease = maxRequestsPerSecond - semaphore.availablePermits();
if (toRelease > 0) {
semaphore.release(toRelease);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}, "RateLimiter-Refill");
refillThread.setDaemon(true);
refillThread.start();
}
}
public static final class RequestStats {
private final String host;
private final AtomicLong totalRequests = new AtomicLong(0);
private final AtomicLong successfulRequests = new AtomicLong(0);
private final AtomicLong failedRequests = new AtomicLong(0);
private final AtomicLong rateLimitedRequests = new AtomicLong(0);
private final AtomicLong totalDurationMs = new AtomicLong(0);
RequestStats(String host) {
this.host = host;
}
void incrementTotal() { totalRequests.incrementAndGet(); }
void recordSuccess(long durationMs) {
successfulRequests.incrementAndGet();
totalDurationMs.addAndGet(durationMs);
}
void incrementFailed() { failedRequests.incrementAndGet(); }
void incrementRateLimited() { rateLimitedRequests.incrementAndGet(); }
public String getHost() { return host; }
public long getTotalRequests() { return totalRequests.get(); }
public long getSuccessfulRequests() { return successfulRequests.get(); }
public long getFailedRequests() { return failedRequests.get(); }
public long getRateLimitedRequests() { return rateLimitedRequests.get(); }
public long getAverageDurationMs() {
var successful = successfulRequests.get();
return successful > 0 ? totalDurationMs.get() / successful : 0;
}
@Override
public String toString() {
return String.format("%s: %d total, %d success, %d failed, %d rate-limited, avg %dms",
host, getTotalRequests(), getSuccessfulRequests(),
getFailedRequests(), getRateLimitedRequests(), getAverageDurationMs());
}
}
}

View File

@@ -0,0 +1,176 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
@Slf4j
public class ScraperDataAdapter {
private static final DateTimeFormatter[] TIMESTAMP_FORMATS = {
DateTimeFormatter.ISO_LOCAL_DATE_TIME,
DateTimeFormatter.ISO_DATE_TIME,
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
};
static AuctionInfo fromScraperAuction(ResultSet rs) throws SQLException {
// Parse "A7-39813" → auctionId=39813, type="A7"
var auctionIdStr = rs.getString("auction_id");
var auctionId = extractNumericId(auctionIdStr);
var type = extractTypePrefix(auctionIdStr);
// Split "Cluj-Napoca, RO" → city="Cluj-Napoca", country="RO"
var location = rs.getString("location");
var locationParts = parseLocation(location);
var city = locationParts[0];
var country = locationParts[1];
// Map field names
var lotCount = getIntOrDefault(rs, "lots_count", 0);
var closingTime = parseTimestamp(getStringOrNull(rs, "first_lot_closing_time"));
return new AuctionInfo(
auctionId,
rs.getString("title"),
location,
city,
country,
rs.getString("url"),
type,
lotCount,
closingTime
);
}
public static Lot fromScraperLot(ResultSet rs) throws SQLException {
var lotIdStr = rs.getString("lot_id"); // Full display ID (e.g., "A1-34732-49")
var lotId = extractNumericId(lotIdStr);
var saleId = extractNumericId(rs.getString("auction_id"));
var bidStr = getStringOrNull(rs, "current_bid");
var bid = parseBidAmount(bidStr);
var currency = parseBidCurrency(bidStr);
var closing = parseTimestamp(getStringOrNull(rs, "closing_time"));
return new Lot(
saleId,
lotId,
lotIdStr, // Store full displayId for GraphQL queries
rs.getString("title"),
getStringOrDefault(rs, "description", ""),
getStringOrDefault(rs, "manufacturer", ""),
getStringOrDefault(rs, "type", ""),
getIntOrDefault(rs, "year", 0),
getStringOrDefault(rs, "category", ""),
bid,
currency,
rs.getString("url"),
closing,
getBooleanOrDefault(rs, "closing_notified", false),
// New intelligence fields - set to null for now
null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null,
null, null
);
}
public static long extractNumericId(String id) {
if (id == null || id.isBlank()) return 0L;
// Remove the type prefix (e.g., "A7-") first, then extract all digits
// "A7-39813" → "39813" → 39813
// "A1-28505-5" → "28505-5" → "285055"
var afterPrefix = id.indexOf('-') >= 0 ? id.substring(id.indexOf('-') + 1) : id;
var digits = afterPrefix.replaceAll("\\D+", "");
if (digits.isEmpty()) return 0L;
// Check if number is too large for long (> 19 digits or value > Long.MAX_VALUE)
if (digits.length() > 19) {
log.debug("ID too large for long, skipping: {}", id);
return 0L;
}
try {
return Long.parseLong(digits);
} catch (NumberFormatException e) {
log.debug("Invalid numeric ID, skipping: {}", id);
return 0L;
}
}
private static String extractTypePrefix(String id) {
if (id == null) return "";
var idx = id.indexOf('-');
return idx > 0 ? id.substring(0, idx) : "";
}
private static String[] parseLocation(String location) {
if (location == null || location.isBlank()) return new String[]{ "", "" };
var parts = location.split(",\\s*");
var city = parts[0].trim();
var country = parts.length > 1 ? parts[parts.length - 1].trim() : "";
return new String[]{ city, country };
}
private static double parseBidAmount(String bid) {
if (bid == null || bid.isBlank() || bid.toLowerCase().contains("no")) return 0.0;
var cleaned = bid.replaceAll("[^0-9.]", "");
try {
return cleaned.isEmpty() ? 0.0 : Double.parseDouble(cleaned);
} catch (NumberFormatException e) {
return 0.0;
}
}
private static String parseBidCurrency(String bid) {
if (bid == null) return "EUR";
return bid.contains("") ? "EUR"
: bid.contains("$") ? "USD"
: bid.contains("£") ? "GBP"
: "EUR";
}
private static LocalDateTime parseTimestamp(String ts) {
if (ts == null || ts.isBlank()) return null;
// Filter out known invalid values
String tsLower = ts.toLowerCase().trim();
if (tsLower.equals("gap") || tsLower.equals("null") || tsLower.equals("n/a") ||
tsLower.equals("unknown") || tsLower.equals("tbd") || tsLower.length() < 8) {
log.debug("Skipping invalid timestamp value: {}", ts);
return null;
}
for (var fmt : TIMESTAMP_FORMATS) {
try {
return LocalDateTime.parse(ts, fmt);
} catch (DateTimeParseException ignored) { }
}
log.debug("Unable to parse timestamp: {}", ts);
return null;
}
private static String getStringOrNull(ResultSet rs, String col) throws SQLException {
return rs.getString(col);
}
private static String getStringOrDefault(ResultSet rs, String col, String def) throws SQLException {
var v = rs.getString(col);
return v != null ? v : def;
}
private static int getIntOrDefault(ResultSet rs, String col, int def) throws SQLException {
var v = rs.getInt(col);
return rs.wasNull() ? def : v;
}
private static boolean getBooleanOrDefault(ResultSet rs, String col, boolean def) throws SQLException {
var v = rs.getInt(col);
return rs.wasNull() ? def : v != 0;
}
}

View File

@@ -0,0 +1,74 @@
package auctiora;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import lombok.extern.slf4j.Slf4j;
import nu.pattern.OpenCV;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.opencv.core.Core;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Map;
@Slf4j
@Path("/api")
public class StatusResource {
private static final DateTimeFormatter FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss z")
.withZone(ZoneId.systemDefault());
@ConfigProperty(name = "application.version", defaultValue = "1.0-SNAPSHOT") String appVersion;
@ConfigProperty(name = "application.groupId") String groupId;
@ConfigProperty(name = "application.artifactId") String artifactId;
@ConfigProperty(name = "application.version") String version;
public record StatusResponse(
String groupId,
String artifactId,
String version,
String status,
String timestamp,
String mvnVersion,
String javaVersion,
String os,
String openCvVersion
) { }
@GET
@Path("/status")
@Produces(MediaType.APPLICATION_JSON)
public StatusResponse getStatus() {
return new StatusResponse(groupId, artifactId, version,
"running",
FORMATTER.format(Instant.now()),
appVersion,
System.getProperty("java.version"),
System.getProperty("os.name") + " " + System.getProperty("os.arch"),
getOpenCvVersion()
);
}
@GET
@Path("/hello")
@Produces(MediaType.APPLICATION_JSON)
public Map<String, String> sayHello() {
return Map.of(
"message", "Hello from Scrape-UI!",
"timestamp", FORMATTER.format(Instant.now()),
"openCvVersion", getOpenCvVersion()
);
}
private String getOpenCvVersion() {
try {
// OpenCV is already loaded by AuctionMonitorProducer
return Core.VERSION;
} catch (Exception e) {
return "Not loaded";
}
}
}

View File

@@ -0,0 +1,378 @@
package auctiora;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
/**
* GraphQL client for fetching enriched lot data from Troostwijk API.
* Fetches intelligence fields: followers, estimates, bid velocity, condition, etc.
*/
@Slf4j
@ApplicationScoped
public class TroostwijkGraphQLClient {
private static final String GRAPHQL_ENDPOINT = "https://storefront.tbauctions.com/storefront/graphql";
private static final String LOCALE = "nl";
private static final String PLATFORM = "TWK";
private static final ObjectMapper objectMapper = new ObjectMapper();
@Inject
RateLimitedHttpClient rateLimitedClient;
/**
* Fetches enriched lot data from GraphQL API
* @param displayId The lot display ID (e.g., "A1-34732-49")
* @param lotId The numeric lot ID for mapping back to database
* @return LotIntelligence with enriched fields, or null if failed
*/
public LotIntelligence fetchLotIntelligence(String displayId, long lotId) {
if (displayId == null || displayId.isBlank()) {
log.debug("Cannot fetch intelligence for null/blank displayId");
return null;
}
try {
var query = buildLotQuery();
var variables = buildVariables(displayId);
// Proper GraphQL request format with query and variables
var requestBody = String.format(
"{\"query\":\"%s\",\"variables\":%s}",
escapeJson(query),
variables
);
var request = java.net.http.HttpRequest.newBuilder()
.uri(java.net.URI.create(GRAPHQL_ENDPOINT))
.header("Content-Type", "application/json")
.header("Accept", "application/json")
.POST(java.net.http.HttpRequest.BodyPublishers.ofString(requestBody))
.build();
var response = rateLimitedClient.send(
request,
java.net.http.HttpResponse.BodyHandlers.ofString()
);
if (response == null || response.body() == null) {
log.debug("No response from GraphQL for lot {}", displayId);
return null;
}
log.debug("GraphQL response for lot {}: {}", displayId, response.body().substring(0, Math.min(200, response.body().length())));
return parseLotIntelligence(response.body(), lotId);
} catch (Exception e) {
log.warn("Failed to fetch lot intelligence for {}: {}", lotId, e.getMessage());
return null;
}
}
/**
* Batch fetch multiple lots in a single query (more efficient)
*/
public List<LotIntelligence> fetchBatchLotIntelligence(List<Long> lotIds) {
List<LotIntelligence> results = new ArrayList<>();
// Split into batches of 50 to avoid query size limits
var batchSize = 50;
for (var i = 0; i < lotIds.size(); i += batchSize) {
var end = Math.min(i + batchSize, lotIds.size());
var batch = lotIds.subList(i, end);
try {
var query = buildBatchLotQuery(batch);
var requestBody = String.format("{\"query\":\"%s\"}",
escapeJson(query));
var request = java.net.http.HttpRequest.newBuilder()
.uri(java.net.URI.create(GRAPHQL_ENDPOINT))
.header("Content-Type", "application/json")
.POST(java.net.http.HttpRequest.BodyPublishers.ofString(requestBody))
.build();
var response = rateLimitedClient.send(
request,
java.net.http.HttpResponse.BodyHandlers.ofString()
);
if (response != null && response.body() != null) {
results.addAll(parseBatchLotIntelligence(response.body(), batch));
}
} catch (Exception e) {
log.warn("Failed to fetch batch lot intelligence: {}", e.getMessage());
}
}
return results;
}
private String buildLotQuery() {
// Match Python scraper's LOT_BIDDING_QUERY structure
// Uses lotDetails(displayId:...) instead of lot(id:...)
return """
query LotBiddingData($lotDisplayId: String!, $locale: String!, $platform: Platform!) {
lotDetails(displayId: $lotDisplayId, locale: $locale, platform: $platform) {
id
displayId
followersCount
currentBidInCents
nextBidStepInCents
condition
description
biddingStatus
buyersPremium
viewCount
estimatedValueInCentsMin
estimatedValueInCentsMax
categoryPath
location {
city
country
}
biddingStatistics {
numberOfBids
}
}
}
""".replaceAll("\\s+", " ");
}
private String buildVariables(String displayId) {
return String.format("""
{
"lotDisplayId": "%s",
"locale": "%s",
"platform": "%s"
}
""", displayId, LOCALE, PLATFORM).replaceAll("\\s+", " ");
}
private String buildBatchLotQuery(List<Long> lotIds) {
var query = new StringBuilder("query {");
for (var i = 0; i < lotIds.size(); i++) {
query.append(String.format("""
lot%d: lot(id: %d) {
id
followersCount
estimatedMin
estimatedMax
nextBidStepInCents
condition
categoryPath
city
countryCode
biddingStatus
vat
buyerPremiumPercentage
viewCount
bidsCount
}
""", i, lotIds.get(i)));
}
query.append("}");
return query.toString().replaceAll("\\s+", " ");
}
private LotIntelligence parseLotIntelligence(String json, long lotId) {
try {
// Check if response is HTML (error page) instead of JSON
if (json != null && json.trim().startsWith("<")) {
log.debug("GraphQL API returned HTML instead of JSON - likely auth required or wrong endpoint");
return null;
}
var root = objectMapper.readTree(json);
var lotNode = root.path("data").path("lotDetails");
if (lotNode.isMissingNode()) {
log.debug("No lotDetails in GraphQL response");
return null;
}
// Extract location from nested object
var locationNode = lotNode.path("location");
var city = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "city");
var countryCode = locationNode.isMissingNode() ? null : getStringOrNull(locationNode, "country");
// Extract bids count from nested biddingStatistics
var statsNode = lotNode.path("biddingStatistics");
var bidsCount = statsNode.isMissingNode() ? null : getIntOrNull(statsNode, "numberOfBids");
// Convert cents to euros for estimates
var estimatedMinCents = getLongOrNull(lotNode, "estimatedValueInCentsMin");
var estimatedMaxCents = getLongOrNull(lotNode, "estimatedValueInCentsMax");
var estimatedMin = estimatedMinCents != null ? estimatedMinCents.doubleValue() : null;
var estimatedMax = estimatedMaxCents != null ? estimatedMaxCents.doubleValue() : null;
return new LotIntelligence(
lotId,
getIntOrNull(lotNode, "followersCount"),
estimatedMin,
estimatedMax,
getLongOrNull(lotNode, "nextBidStepInCents"),
getStringOrNull(lotNode, "condition"),
getStringOrNull(lotNode, "categoryPath"),
city,
countryCode,
getStringOrNull(lotNode, "biddingStatus"),
null, // appearance - not in API response
null, // packaging - not in API response
null, // quantity - not in API response
null, // vat - not in API response
null, // buyerPremiumPercentage - could extract from buyersPremium
null, // remarks - not in API response
null, // startingBid - not in API response
null, // reservePrice - not in API response
null, // reserveMet - not in API response
null, // bidIncrement - not in API response
getIntOrNull(lotNode, "viewCount"),
null, // firstBidTime - not in API response
null, // lastBidTime - not in API response
null // bidVelocity - could calculate from bidsCount if we had timing data
);
} catch (Exception e) {
log.warn("Failed to parse lot intelligence: {}", e.getMessage());
return null;
}
}
private List<LotIntelligence> parseBatchLotIntelligence(String json, List<Long> lotIds) {
List<LotIntelligence> results = new ArrayList<>();
try {
var root = objectMapper.readTree(json);
var data = root.path("data");
for (var i = 0; i < lotIds.size(); i++) {
var lotNode = data.path("lot" + i);
if (!lotNode.isMissingNode()) {
var intelligence = parseLotIntelligenceFromNode(lotNode, lotIds.get(i));
if (intelligence != null) {
results.add(intelligence);
}
}
}
} catch (Exception e) {
log.warn("Failed to parse batch lot intelligence: {}", e.getMessage());
}
return results;
}
private LotIntelligence parseLotIntelligenceFromNode(JsonNode lotNode, long lotId) {
try {
return new LotIntelligence(
lotId,
getIntOrNull(lotNode, "followersCount"),
getDoubleOrNull(lotNode, "estimatedMin"),
getDoubleOrNull(lotNode, "estimatedMax"),
getLongOrNull(lotNode, "nextBidStepInCents"),
getStringOrNull(lotNode, "condition"),
getStringOrNull(lotNode, "categoryPath"),
getStringOrNull(lotNode, "city"),
getStringOrNull(lotNode, "countryCode"),
getStringOrNull(lotNode, "biddingStatus"),
null, // appearance not in batch query
null, // packaging not in batch query
null, // quantity not in batch query
getDoubleOrNull(lotNode, "vat"),
getDoubleOrNull(lotNode, "buyerPremiumPercentage"),
null, // remarks not in batch query
null, // startingBid not in batch query
null, // reservePrice not in batch query
null, // reserveMet not in batch query
null, // bidIncrement not in batch query
getIntOrNull(lotNode, "viewCount"),
null, // firstBidTime not in batch query
null, // lastBidTime not in batch query
calculateBidVelocity(lotNode)
);
} catch (Exception e) {
log.warn("Failed to parse lot node: {}", e.getMessage());
return null;
}
}
private Double calculateBidVelocity(JsonNode lotNode) {
try {
var bidsCount = getIntOrNull(lotNode, "bidsCount");
var firstBidStr = getStringOrNull(lotNode, "firstBidTime");
if (bidsCount == null || firstBidStr == null || bidsCount == 0) {
return null;
}
var firstBid = parseDateTime(firstBidStr);
if (firstBid == null) return null;
var hoursElapsed = java.time.Duration.between(firstBid, LocalDateTime.now()).toHours();
if (hoursElapsed == 0) return (double) bidsCount;
return (double) bidsCount / hoursElapsed;
} catch (Exception e) {
return null;
}
}
private LocalDateTime parseDateTime(String dateStr) {
if (dateStr == null || dateStr.isBlank()) return null;
try {
return LocalDateTime.parse(dateStr, DateTimeFormatter.ISO_DATE_TIME);
} catch (Exception e) {
return null;
}
}
private String escapeJson(String str) {
return str.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r")
.replace("\t", "\\t");
}
private Integer getIntOrNull(JsonNode node, String field) {
var fieldNode = node.path(field);
return fieldNode.isNumber() ? fieldNode.asInt() : null;
}
private Long getLongOrNull(JsonNode node, String field) {
var fieldNode = node.path(field);
return fieldNode.isNumber() ? fieldNode.asLong() : null;
}
private Double getDoubleOrNull(JsonNode node, String field) {
var fieldNode = node.path(field);
return fieldNode.isNumber() ? fieldNode.asDouble() : null;
}
private String getStringOrNull(JsonNode node, String field) {
var fieldNode = node.path(field);
return fieldNode.isTextual() ? fieldNode.asText() : null;
}
private Boolean getBooleanOrNull(JsonNode node, String field) {
var fieldNode = node.path(field);
return fieldNode.isBoolean() ? fieldNode.asBoolean() : null;
}
}

View File

@@ -0,0 +1,132 @@
package auctiora;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.sql.SQLException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Slf4j
public class TroostwijkMonitor {
private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list";
RateLimitedHttpClient httpClient;
ObjectMapper objectMapper;
@Getter DatabaseService db;
NotificationService notifier;
ObjectDetectionService detector;
ImageProcessingService imageProcessor;
ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
var t = new Thread(r, "troostwijk-monitor-thread");
t.setDaemon(true);
return t;
});
public TroostwijkMonitor(String databasePath,
String notificationConfig,
String yoloCfgPath,
String yoloWeightsPath,
String classNamesPath)
throws SQLException, IOException {
httpClient = new RateLimitedHttpClient();
objectMapper = new ObjectMapper();
db = new DatabaseService(databasePath);
notifier = new NotificationService(notificationConfig);
detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
imageProcessor = new ImageProcessingService(db, detector);
db.ensureSchema();
}
public void scheduleMonitoring() {
scheduler.scheduleAtFixedRate(this::monitorAllLots, 0, 1, TimeUnit.HOURS);
log.info("✓ Monitoring service started");
}
private void monitorAllLots() {
try {
var activeLots = db.getActiveLots();
log.info("Monitoring {} active lots …", activeLots.size());
for (var lot : activeLots) {
checkAndUpdateLot(lot);
}
} catch (Exception e) {
log.error("Error during scheduled monitoring", e);
}
}
private void checkAndUpdateLot(Lot lot) {
refreshLotBid(lot);
var minutesLeft = lot.minutesUntilClose();
if (minutesLeft < 30) {
if (minutesLeft <= 5 && !lot.closingNotified()) {
notifier.sendNotification(
"Kavel " + lot.lotId() + " sluit binnen " + minutesLeft + " min.",
"Lot nearing closure", 1);
db.updateLotNotificationFlags(lot.withClosingNotified(true));
}
scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
}
}
private void refreshLotBid(Lot lot) {
try {
var url = LOT_API +
"?batchSize=1&listType=7&offset=0&sortOption=0" +
"&saleID=" + lot.saleId() +
"&parentID=0&relationID=0&buildversion=201807311" +
"&lotID=" + lot.lotId();
var resp = httpClient.sendGet(url);
if (resp.statusCode() != 200) return;
var root = objectMapper.readTree(resp.body());
var results = root.path("results");
if (results.isArray() && results.size() > 0) {
var newBid = results.get(0).path("cb").asDouble();
if (Double.compare(newBid, lot.currentBid()) > 0) {
var previous = lot.currentBid();
var updatedLot = lot.withCurrentBid(newBid);
db.updateLotCurrentBid(updatedLot);
var msg = String.format(
"Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previous);
notifier.sendNotification(msg, "Kavel bieding update", 0);
}
}
} catch (IOException | InterruptedException e) {
log.warn("Failed to refresh bid for lot {}", lot.lotId(), e);
if (e instanceof InterruptedException) Thread.currentThread().interrupt();
}
}
public void printDatabaseStats() {
try {
var allLots = db.getAllLots();
var imageCount = db.getImageCount();
log.info("📊 Database Summary: total lots = {}, total images = {}",
allLots.size(), imageCount);
if (!allLots.isEmpty()) {
var sum = allLots.stream().mapToDouble(Lot::currentBid).sum();
log.info("Total current bids: €{}", String.format("%.2f", sum));
}
} catch (Exception e) {
log.warn("Could not retrieve database stats", e);
}
}
public void processPendingImages() {
imageProcessor.processPendingImages();
}
}

View File

@@ -0,0 +1,378 @@
package auctiora;
import jakarta.inject.Inject;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.jboss.logging.Logger;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.stream.Collectors;
/**
* REST API for Auction Valuation Analytics
* Implements the mathematical framework for fair market value calculation,
* undervaluation detection, and bidding strategy recommendations.
*/
@Path("/api/analytics")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public class ValuationAnalyticsResource {
private static final Logger LOG = Logger.getLogger(ValuationAnalyticsResource.class);
@Inject
DatabaseService db;
/**
* GET /api/analytics/valuation/health
* Health check endpoint to verify API availability
*/
@GET
@Path("/valuation/health")
public Response healthCheck() {
return Response.ok(Map.of(
"status", "healthy",
"service", "valuation-analytics",
"timestamp", java.time.LocalDateTime.now().toString()
)).build();
}
/**
* POST /api/analytics/valuation
* Main valuation endpoint that calculates FMV, undervaluation score,
* predicted final price, and bidding strategy
*/
@POST
@Path("/valuation")
public Response calculateValuation(ValuationRequest request) {
try {
LOG.infof("Valuation request for lot: %s", request.lotId);
var startTime = System.currentTimeMillis();
// Step 1: Fetch comparable sales from database
var comparables = fetchComparables(request);
// Step 2: Calculate Fair Market Value (FMV)
var fmv = calculateFairMarketValue(request, comparables);
// Step 3: Calculate undervaluation score
var undervaluationScore = calculateUndervaluationScore(request, fmv.value);
// Step 4: Predict final price
var prediction = calculateFinalPrice(request, fmv.value);
// Step 5: Generate bidding strategy
var strategy = generateBiddingStrategy(request, fmv, prediction);
// Step 6: Compile response
var response = new ValuationResponse();
response.lotId = request.lotId;
response.timestamp = LocalDateTime.now().toString();
response.fairMarketValue = fmv;
response.undervaluationScore = undervaluationScore;
response.pricePrediction = prediction;
response.biddingStrategy = strategy;
response.parameters = request;
var duration = System.currentTimeMillis() - startTime;
LOG.infof("Valuation completed in %d ms", duration);
return Response.ok(response).build();
} catch (Exception e) {
LOG.error("Valuation calculation failed", e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
.entity(Map.of("error", e.getMessage()))
.build();
}
}
/**
* Fetches comparable lots from database based on category, manufacturer,
* year, and condition similarity
*/
private List<ComparableLot> fetchComparables(ValuationRequest req) {
// TODO: Replace with actual database query
// For now, return mock data simulating real comparables
return List.of(
new ComparableLot("NL-2023-4451", 8200.0, 8.0, 2016, 1, 30),
new ComparableLot("BE-2023-9823", 7800.0, 7.0, 2014, 0, 45),
new ComparableLot("DE-2024-1234", 8500.0, 9.0, 2017, 1, 60),
new ComparableLot("NL-2023-5678", 7500.0, 6.0, 2013, 0, 25),
new ComparableLot("BE-2024-7890", 7900.0, 7.5, 2015, 1, 15),
new ComparableLot("NL-2023-2345", 8100.0, 8.5, 2016, 0, 40),
new ComparableLot("DE-2024-4567", 8300.0, 7.0, 2015, 1, 55),
new ComparableLot("BE-2023-3456", 7700.0, 6.5, 2014, 0, 35)
);
}
/**
* Formula: FMV = Σ(P_i · ω_c · ω_t · ω_p · ω_h) / Σ(ω_c · ω_t · ω_p · ω_h)
* Where weights are exponential/logistic functions of similarity
*/
private FairMarketValue calculateFairMarketValue(ValuationRequest req, List<ComparableLot> comparables) {
var weightedSum = 0.0;
var weightSum = 0.0;
List<WeightedComparable> weightedComps = new ArrayList<>();
for (var comp : comparables) {
// Condition weight: ω_c = exp(-λ_c · |C_target - C_i|)
var omegaC = Math.exp(-0.693 * Math.abs(req.conditionScore - comp.conditionScore));
// Time weight: ω_t = exp(-λ_t · |T_target - T_i|)
var omegaT = Math.exp(-0.048 * Math.abs(req.manufacturingYear - comp.manufacturingYear));
// Provenance weight: ω_p = 1 + δ_p · (P_target - P_i)
var omegaP = 1 + 0.15 * ((req.provenanceDocs > 0 ? 1 : 0) - comp.hasProvenance);
// Historical weight: ω_h = 1 / (1 + e^(-kh · (D_i - D_median)))
var omegaH = 1.0 / (1 + Math.exp(-0.01 * (comp.daysAgo - 40)));
var totalWeight = omegaC * omegaT * omegaP * omegaH;
weightedSum += comp.finalPrice * totalWeight;
weightSum += totalWeight;
// Store for transparency
weightedComps.add(new WeightedComparable(comp, totalWeight, omegaC, omegaT, omegaP, omegaH));
}
var baseFMV = weightSum > 0 ? weightedSum / weightSum : (req.estimatedMin + req.estimatedMax) / 2;
// Apply condition multiplier: M_cond = exp(α_c · √C_target - β_c)
var conditionMultiplier = Math.exp(0.15 * Math.sqrt(req.conditionScore) - 0.40);
baseFMV *= conditionMultiplier;
// Apply provenance premium: Δ_prov = V_base · (η_0 + η_1 · ln(1 + N_docs))
if (req.provenanceDocs > 0) {
var provenancePremium = 0.08 + 0.035 * Math.log(1 + req.provenanceDocs);
baseFMV *= (1 + provenancePremium);
}
var fmv = new FairMarketValue();
fmv.value = Math.round(baseFMV * 100.0) / 100.0;
fmv.conditionMultiplier = Math.round(conditionMultiplier * 1000.0) / 1000.0;
fmv.provenancePremium = req.provenanceDocs > 0 ? 0.08 + 0.035 * Math.log(1 + req.provenanceDocs) : 0.0;
fmv.comparablesUsed = comparables.size();
fmv.confidence = calculateFMVConfidence(comparables.size(), weightSum);
fmv.weightedComparables = weightedComps;
return fmv;
}
/**
* Calculates undervaluation score:
* U_score = (FMV - P_current)/FMV · σ_market · (1 + B_velocity/10) · ln(1 + W_watch/W_bid)
*/
private double calculateUndervaluationScore(ValuationRequest req, double fmv) {
if (fmv <= 0) return 0.0;
var priceGap = (fmv - req.currentBid) / fmv;
var velocityFactor = 1 + req.bidVelocity / 10.0;
var watchRatio = Math.log(1 + req.watchCount / Math.max(req.bidCount, 1));
var uScore = priceGap * req.marketVolatility * velocityFactor * watchRatio;
return Math.max(0.0, Math.round(uScore * 1000.0) / 1000.0);
}
/**
* Predicts final price: P̂_final = FMV · (1 + ε_bid + ε_time + ε_comp)
* Where each epsilon represents auction dynamics
*/
private PricePrediction calculateFinalPrice(ValuationRequest req, double fmv) {
// Bid momentum error: ε_bid = tanh(φ_1 · Λ_b - φ_2 · P_current/FMV)
var epsilonBid = Math.tanh(0.15 * req.bidVelocity - 0.10 * (req.currentBid / fmv));
// Time pressure error: ε_time = ψ · exp(-t_close/30)
var epsilonTime = 0.20 * Math.exp(-req.minutesUntilClose / 30.0);
// Competition error: ε_comp = ρ · ln(1 + W_watch/50)
var epsilonComp = 0.08 * Math.log(1 + req.watchCount / 50.0);
var predictedPrice = fmv * (1 + epsilonBid + epsilonTime + epsilonComp);
// 95% confidence interval: ± 1.96 · σ_residual
var residualStdDev = fmv * 0.08; // Mock residual standard deviation
var ciLower = predictedPrice - 1.96 * residualStdDev;
var ciUpper = predictedPrice + 1.96 * residualStdDev;
var pred = new PricePrediction();
pred.predictedPrice = Math.round(predictedPrice * 100.0) / 100.0;
pred.confidenceIntervalLower = Math.round(ciLower * 100.0) / 100.0;
pred.confidenceIntervalUpper = Math.round(ciUpper * 100.0) / 100.0;
pred.components = Map.of(
"bidMomentum", Math.round(epsilonBid * 1000.0) / 1000.0,
"timePressure", Math.round(epsilonTime * 1000.0) / 1000.0,
"competition", Math.round(epsilonComp * 1000.0) / 1000.0
);
return pred;
}
/**
* Generates optimal bidding strategy based on market conditions
*/
private BiddingStrategy generateBiddingStrategy(ValuationRequest req, FairMarketValue fmv, PricePrediction pred) {
var strategy = new BiddingStrategy();
// Determine competition level
if (req.bidVelocity > 5.0) {
strategy.competitionLevel = "HIGH";
strategy.recommendedTiming = "FINAL_30_SECONDS";
strategy.maxBid = pred.predictedPrice + 50; // Slight overbid for hot lots
strategy.riskFactors = List.of("Bidding war likely", "Sniping detected");
} else if (req.minutesUntilClose < 10) {
strategy.competitionLevel = "EXTREME";
strategy.recommendedTiming = "FINAL_10_SECONDS";
strategy.maxBid = pred.predictedPrice * 1.02;
strategy.riskFactors = List.of("Last-minute sniping", "Price volatility");
} else {
strategy.competitionLevel = "MEDIUM";
strategy.recommendedTiming = "FINAL_10_MINUTES";
// Adjust max bid based on undervaluation
var undervaluationScore = calculateUndervaluationScore(req, fmv.value);
if (undervaluationScore > 0.25) {
// Aggressive strategy for undervalued lots
strategy.maxBid = fmv.value * (1 + 0.05); // Conservative overbid
strategy.analysis = "Significant undervaluation detected. Consider aggressive bidding.";
} else {
// Standard strategy
strategy.maxBid = fmv.value * (1 + 0.03);
}
strategy.riskFactors = List.of("Standard competition level");
}
// Generate detailed analysis
strategy.analysis = String.format(
"Bid velocity is %.1f bids/min with %d watchers. %s competition detected. " +
"Predicted final: €%.2f (%.0f%% confidence).",
req.bidVelocity,
req.watchCount,
strategy.competitionLevel,
pred.predictedPrice,
fmv.confidence * 100
);
// Round the max bid
strategy.maxBid = Math.round(strategy.maxBid * 100.0) / 100.0;
strategy.recommendedTimingText = strategy.recommendedTiming.replace("_", " ");
return strategy;
}
/**
* Calculates confidence score based on number and quality of comparables
*/
private double calculateFMVConfidence(int comparableCount, double totalWeight) {
var confidence = 0.5; // Base confidence
// Boost for more comparables
confidence += Math.min(comparableCount * 0.05, 0.3);
// Boost for high total weight (good matches)
confidence += Math.min(totalWeight / comparableCount * 0.1, 0.2);
// Cap at 0.95
return Math.min(confidence, 0.95);
}
// ================== DTO Classes ==================
public static class ValuationRequest {
public String lotId;
public double currentBid;
public double conditionScore; // C_target ∈ [0,10]
public int manufacturingYear; // T_target
public int watchCount; // W_watch
public int bidCount = 1; // W_bid (default 1 to avoid division by zero)
public double marketVolatility = 0.15; // σ_market ∈ [0,1]
public double bidVelocity; // Λ_b (bids/min)
public int minutesUntilClose; // t_close
public int provenanceDocs = 0; // N_docs
public double estimatedMin;
public double estimatedMax;
// Optional: override parameters for sensitivity analysis
public Map<String, Double> sensitivityParams;
}
public static class ValuationResponse {
public String lotId;
public String timestamp;
public FairMarketValue fairMarketValue;
public double undervaluationScore;
public PricePrediction pricePrediction;
public BiddingStrategy biddingStrategy;
public ValuationRequest parameters;
public long calculationTimeMs;
}
public static class FairMarketValue {
public double value;
public double conditionMultiplier;
public double provenancePremium;
public int comparablesUsed;
public double confidence; // [0,1]
public List<WeightedComparable> weightedComparables;
}
public static class WeightedComparable {
public String comparableLotId;
public double finalPrice;
public double totalWeight;
public Map<String, Double> components;
public WeightedComparable(ComparableLot comp, double totalWeight, double omegaC, double omegaT, double omegaP, double omegaH) {
this.comparableLotId = comp.lotId;
this.finalPrice = comp.finalPrice;
this.totalWeight = Math.round(totalWeight * 1000.0) / 1000.0;
this.components = Map.of(
"conditionWeight", Math.round(omegaC * 1000.0) / 1000.0,
"timeWeight", Math.round(omegaT * 1000.0) / 1000.0,
"provenanceWeight", Math.round(omegaP * 1000.0) / 1000.0,
"historicalWeight", Math.round(omegaH * 1000.0) / 1000.0
);
}
}
public static class PricePrediction {
public double predictedPrice;
public double confidenceIntervalLower;
public double confidenceIntervalUpper;
public Map<String, Double> components; // ε_bid, ε_time, ε_comp
}
public static class BiddingStrategy {
public String competitionLevel; // LOW, MEDIUM, HIGH, EXTREME
public double maxBid;
public String recommendedTiming; // FINAL_10_MINUTES, FINAL_30_SECONDS, etc.
public String recommendedTimingText;
public String analysis;
public List<String> riskFactors;
}
// Helper class for internal comparable representation
private static class ComparableLot {
String lotId;
double finalPrice;
double conditionScore;
int manufacturingYear;
int hasProvenance;
int daysAgo;
public ComparableLot(String lotId, double finalPrice, double conditionScore, int manufacturingYear, int hasProvenance, int daysAgo) {
this.lotId = lotId;
this.finalPrice = finalPrice;
this.conditionScore = conditionScore;
this.manufacturingYear = manufacturingYear;
this.hasProvenance = hasProvenance;
this.daysAgo = daysAgo;
}
}
}

View File

@@ -0,0 +1,433 @@
package auctiora;
import lombok.extern.slf4j.Slf4j;
import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* Orchestrates the complete workflow of auction monitoring, image processing,
* object detection, and notifications.
*
* This class coordinates all services and provides scheduled execution,
* event-driven triggers, and manual workflow execution.
*/
@Slf4j
public class WorkflowOrchestrator {
private final TroostwijkMonitor monitor;
private final DatabaseService db;
private final ImageProcessingService imageProcessor;
private final NotificationService notifier;
private final ObjectDetectionService detector;
private final ScheduledExecutorService scheduler;
private boolean isRunning = false;
/**
* Creates a workflow orchestrator with all necessary services.
*/
public WorkflowOrchestrator(String databasePath, String notificationConfig,
String yoloCfg, String yoloWeights, String yoloClasses)
throws SQLException, IOException {
log.info("🔧 Initializing Workflow Orchestrator...");
// Initialize core services
this.db = new DatabaseService(databasePath);
this.db.ensureSchema();
this.notifier = new NotificationService(notificationConfig);
this.detector = new ObjectDetectionService(yoloCfg, yoloWeights, yoloClasses);
this.imageProcessor = new ImageProcessingService(db, detector);
this.monitor = new TroostwijkMonitor(databasePath, notificationConfig,
yoloCfg, yoloWeights, yoloClasses);
this.scheduler = Executors.newScheduledThreadPool(3);
log.info("✓ Workflow Orchestrator initialized");
}
/**
* Starts all scheduled workflows.
* This is the main entry point for automated operation.
*/
public void startScheduledWorkflows() {
if (isRunning) {
log.info("⚠️ Workflows already running");
return;
}
log.info("\n🚀 Starting Scheduled Workflows...\n");
// Workflow 1: Import scraper data (every 30 minutes)
scheduleScraperDataImport();
// Workflow 2: Process pending images (every 1 hour)
scheduleImageProcessing();
// Workflow 3: Monitor bids (every 15 minutes)
scheduleBidMonitoring();
// Workflow 4: Check closing times (every 5 minutes)
scheduleClosingAlerts();
isRunning = true;
log.info("✓ All scheduled workflows started\n");
}
/**
* Workflow 1: Import Scraper Data
* Frequency: Every 30 minutes
* Purpose: Import new auctions and lots from external scraper
*/
private void scheduleScraperDataImport() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("📥 [WORKFLOW 1] Importing scraper data...");
var start = System.currentTimeMillis();
// Import auctions
var auctions = db.importAuctionsFromScraper();
log.info(" → Imported {} auctions", auctions.size());
// Import lots
var lots = db.importLotsFromScraper();
log.info(" → Imported {} lots", lots.size());
// Check for images needing detection
var images = db.getImagesNeedingDetection();
log.info(" → Found {} images needing detection", images.size());
var duration = System.currentTimeMillis() - start;
log.info(" ✓ Scraper import completed in {}ms\n", duration);
// Trigger notification if significant data imported
if (auctions.size() > 0 || lots.size() > 10) {
notifier.sendNotification(
String.format("Imported %d auctions, %d lots", auctions.size(), lots.size()),
"Data Import Complete",
0
);
}
} catch (Exception e) {
log.info(" ❌ Scraper import failed: {}", e.getMessage());
}
}, 0, 30, TimeUnit.MINUTES);
log.info(" ✓ Scheduled: Scraper Data Import (every 30 min)");
}
/**
* Workflow 2: Process Pending Images
* Frequency: Every 1 hour
* Purpose: Run object detection on images already downloaded by scraper
*/
private void scheduleImageProcessing() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("🖼️ [WORKFLOW 2] Processing pending images...");
var start = System.currentTimeMillis();
// Get images that have been downloaded but need object detection
var pendingImages = db.getImagesNeedingDetection();
if (pendingImages.isEmpty()) {
log.info(" → No pending images to process\n");
return;
}
log.info(" → Processing {} images", pendingImages.size());
var processed = 0;
var detected = 0;
for (var image : pendingImages) {
try {
// Run object detection on already-downloaded image
if (imageProcessor.processImage(image.id(), image.filePath(), image.lotId())) {
processed++;
// Check if objects were detected
var labels = db.getImageLabels(image.id());
if (labels != null && !labels.isEmpty()) {
detected++;
// Send notification for interesting detections
if (labels.size() >= 3) {
notifier.sendNotification(
String.format("Lot %d: Detected %s",
image.lotId(),
String.join(", ", labels)),
"Objects Detected",
0
);
}
}
}
// Rate limiting (lighter since no network I/O)
Thread.sleep(100);
} catch (Exception e) {
log.info("\uFE0F Failed to process image: {}", e.getMessage());
}
}
var duration = System.currentTimeMillis() - start;
log.info(String.format(" ✓ Processed %d images, detected objects in %d (%.1fs)\n",
processed, detected, duration / 1000.0));
} catch (Exception e) {
log.info(" ❌ Image processing failed: {}", e.getMessage());
}
}, 5, 60, TimeUnit.MINUTES);
log.info(" ✓ Scheduled: Image Processing (every 1 hour)");
}
/**
* Workflow 3: Monitor Bids
* Frequency: Every 15 minutes
* Purpose: Check for bid changes and send notifications
*/
private void scheduleBidMonitoring() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("💰 [WORKFLOW 3] Monitoring bids...");
var start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
log.info(" → Checking {} active lots", activeLots.size());
var bidChanges = 0;
for (var lot : activeLots) {
// Note: In production, this would call Troostwijk API
// For now, we just track what's in the database
// The external scraper updates bids, we just notify
}
var duration = System.currentTimeMillis() - start;
log.info(String.format(" ✓ Bid monitoring completed in %dms\n", duration));
} catch (Exception e) {
log.info(" ❌ Bid monitoring failed: {}", e.getMessage());
}
}, 2, 15, TimeUnit.MINUTES);
log.info(" ✓ Scheduled: Bid Monitoring (every 15 min)");
}
/**
* Workflow 4: Check Closing Times
* Frequency: Every 5 minutes
* Purpose: Send alerts for lots closing soon
*/
private void scheduleClosingAlerts() {
scheduler.scheduleAtFixedRate(() -> {
try {
log.info("⏰ [WORKFLOW 4] Checking closing times...");
var start = System.currentTimeMillis();
var activeLots = db.getActiveLots();
var alertsSent = 0;
for (var lot : activeLots) {
if (lot.closingTime() == null) continue;
var minutesLeft = lot.minutesUntilClose();
// Alert for lots closing in 5 minutes
if (minutesLeft <= 5 && minutesLeft > 0 && !lot.closingNotified()) {
var message = String.format("Kavel %d sluit binnen %d min.",
lot.lotId(), minutesLeft);
notifier.sendNotification(message, "Lot Closing Soon", 1);
// Mark as notified
var updated = Lot.basic(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
lot.currentBid(), lot.currency(), lot.url(),
lot.closingTime(), true
);
db.updateLotNotificationFlags(updated);
alertsSent++;
}
}
var duration = System.currentTimeMillis() - start;
log.info(String.format(" → Sent %d closing alerts in %dms\n",
alertsSent, duration));
} catch (Exception e) {
log.info(" ❌ Closing alerts failed: {}", e.getMessage());
}
}, 1, 5, TimeUnit.MINUTES);
log.info(" ✓ Scheduled: Closing Alerts (every 5 min)");
}
/**
* Manual trigger: Run complete workflow once
* Useful for testing or on-demand execution
*/
public void runCompleteWorkflowOnce() {
log.info("\n🔄 Running Complete Workflow (Manual Trigger)...\n");
try {
// Step 1: Import data
log.info("[1/4] Importing scraper data...");
var auctions = db.importAuctionsFromScraper();
var lots = db.importLotsFromScraper();
log.info(" ✓ Imported {} auctions, {} lots", auctions.size(), lots.size());
// Step 2: Process images
log.info("[2/4] Processing pending images...");
monitor.processPendingImages();
log.info(" ✓ Image processing completed");
// Step 3: Check bids
log.info("[3/4] Monitoring bids...");
var activeLots = db.getActiveLots();
log.info(" ✓ Monitored {} lots", activeLots.size());
// Step 4: Check closing times
log.info("[4/4] Checking closing times...");
var closingSoon = 0;
for (var lot : activeLots) {
if (lot.closingTime() != null && lot.minutesUntilClose() < 30) {
closingSoon++;
}
}
log.info(" ✓ Found {} lots closing soon", closingSoon);
log.info("\n✓ Complete workflow finished successfully\n");
} catch (Exception e) {
log.info("\n❌ Workflow failed: {}\n", e.getMessage());
}
}
/**
* Event-driven trigger: New auction discovered
*/
public void onNewAuctionDiscovered(AuctionInfo auction) {
log.info("\uD83D\uDCE3 EVENT: New auction discovered - {}", auction.title());
try {
db.upsertAuction(auction);
notifier.sendNotification(
String.format("New auction: %s\nLocation: %s\nLots: %d",
auction.title(), auction.location(), auction.lotCount()),
"New Auction Discovered",
0
);
} catch (Exception e) {
log.info(" ❌ Failed to handle new auction: {}", e.getMessage());
}
}
/**
* Event-driven trigger: Bid change detected
*/
public void onBidChange(Lot lot, double previousBid, double newBid) {
log.info(String.format("📣 EVENT: Bid change on lot %d (€%.2f → €%.2f)",
lot.lotId(), previousBid, newBid));
try {
db.updateLotCurrentBid(lot);
notifier.sendNotification(
String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), newBid, previousBid),
"Kavel Bieding Update",
0
);
} catch (Exception e) {
log.info(" ❌ Failed to handle bid change: {}", e.getMessage());
}
}
/**
* Event-driven trigger: Objects detected in image
*/
public void onObjectsDetected(int lotId, List<String> labels) {
log.info(String.format("📣 EVENT: Objects detected in lot %d - %s",
lotId, String.join(", ", labels)));
try {
if (labels.size() >= 2) {
notifier.sendNotification(
String.format("Lot %d contains: %s", lotId, String.join(", ", labels)),
"Objects Detected",
0
);
}
} catch (Exception e) {
log.info(" ❌ Failed to send detection notification: {}", e.getMessage());
}
}
/**
* Prints current workflow status
*/
public void printStatus() {
log.info("\n📊 Workflow Status:");
log.info(" Running: {}", isRunning ? "Yes" : "No");
try {
var auctions = db.getAllAuctions();
var lots = db.getAllLots();
var images = db.getImageCount();
log.info(" Auctions: {}", auctions.size());
log.info(" Lots: {}", lots.size());
log.info(" Images: {}", images);
// Count closing soon
var closingSoon = 0;
for (var lot : lots) {
if (lot.closingTime() != null && lot.minutesUntilClose() < 30) {
closingSoon++;
}
}
log.info(" Closing soon (< 30 min): {}", closingSoon);
} catch (Exception e) {
log.info("\uFE0F Could not retrieve status: {}", e.getMessage());
}
IO.println();
}
/**
* Gracefully shuts down all workflows
*/
public void shutdown() {
log.info("\n🛑 Shutting down workflows...");
isRunning = false;
scheduler.shutdown();
try {
if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) {
scheduler.shutdownNow();
}
log.info("✓ Workflows shut down successfully\n");
} catch (InterruptedException e) {
scheduler.shutdownNow();
Thread.currentThread().interrupt();
}
}
}

View File

@@ -0,0 +1,164 @@
package auctiora.db;
import auctiora.AuctionInfo;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Jdbi;
import java.time.LocalDateTime;
import java.util.List;
/**
* Repository for auction-related database operations using JDBI3.
* Handles CRUD operations and queries for auctions.
*/
@Slf4j
@RequiredArgsConstructor
public class AuctionRepository {
private final Jdbi jdbi;
/**
* Inserts or updates an auction record.
* Handles both auction_id conflicts and url uniqueness constraints.
*/
public void upsert(AuctionInfo auction) {
jdbi.useTransaction(handle -> {
try {
// Try INSERT with ON CONFLICT on auction_id
handle.createUpdate("""
INSERT INTO auctions (
auction_id, title, location, city, country, url, type, lot_count, closing_time, discovered_at
) VALUES (
:auctionId, :title, :location, :city, :country, :url, :type, :lotCount, :closingTime, :discoveredAt
)
ON CONFLICT(auction_id) DO UPDATE SET
title = excluded.title,
location = excluded.location,
city = excluded.city,
country = excluded.country,
url = excluded.url,
type = excluded.type,
lot_count = excluded.lot_count,
closing_time = excluded.closing_time
""")
.bind("auctionId", auction.auctionId())
.bind("title", auction.title())
.bind("location", auction.location())
.bind("city", auction.city())
.bind("country", auction.country())
.bind("url", auction.url())
.bind("type", auction.typePrefix())
.bind("lotCount", auction.lotCount())
.bind("closingTime", auction.firstLotClosingTime() != null ? auction.firstLotClosingTime().toString() : null)
.bind("discoveredAt", java.time.Instant.now().getEpochSecond())
.execute();
} catch (Exception e) {
// If UNIQUE constraint on url fails, try updating by url
String errMsg = e.getMessage();
if (errMsg != null && (errMsg.contains("UNIQUE constraint failed") ||
errMsg.contains("PRIMARY KEY constraint failed"))) {
log.debug("Auction conflict detected, attempting update by URL: {}", auction.url());
int updated = handle.createUpdate("""
UPDATE auctions SET
auction_id = :auctionId,
title = :title,
location = :location,
city = :city,
country = :country,
type = :type,
lot_count = :lotCount,
closing_time = :closingTime
WHERE url = :url
""")
.bind("auctionId", auction.auctionId())
.bind("title", auction.title())
.bind("location", auction.location())
.bind("city", auction.city())
.bind("country", auction.country())
.bind("type", auction.typePrefix())
.bind("lotCount", auction.lotCount())
.bind("closingTime", auction.firstLotClosingTime() != null ? auction.firstLotClosingTime().toString() : null)
.bind("url", auction.url())
.execute();
if (updated == 0) {
log.warn("Failed to update auction by URL: {}", auction.url());
}
} else {
log.error("Unexpected error upserting auction: {}", e.getMessage(), e);
throw e;
}
}
});
}
/**
* Retrieves all auctions from the database.
*/
public List<AuctionInfo> getAll() {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT * FROM auctions")
.map((rs, ctx) -> {
String closingStr = rs.getString("closing_time");
LocalDateTime closingTime = null;
if (closingStr != null && !closingStr.isBlank()) {
try {
closingTime = LocalDateTime.parse(closingStr);
} catch (Exception e) {
log.warn("Invalid closing_time format: {}", closingStr);
}
}
return new AuctionInfo(
rs.getLong("auction_id"),
rs.getString("title"),
rs.getString("location"),
rs.getString("city"),
rs.getString("country"),
rs.getString("url"),
rs.getString("type"),
rs.getInt("lot_count"),
closingTime
);
})
.list()
);
}
/**
* Retrieves auctions filtered by country code.
*/
public List<AuctionInfo> getByCountry(String countryCode) {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT * FROM auctions WHERE country = :country")
.bind("country", countryCode)
.map((rs, ctx) -> {
String closingStr = rs.getString("closing_time");
LocalDateTime closingTime = null;
if (closingStr != null && !closingStr.isBlank()) {
try {
closingTime = LocalDateTime.parse(closingStr);
} catch (Exception e) {
log.warn("Invalid closing_time format: {}", closingStr);
}
}
return new AuctionInfo(
rs.getLong("auction_id"),
rs.getString("title"),
rs.getString("location"),
rs.getString("city"),
rs.getString("country"),
rs.getString("url"),
rs.getString("type"),
rs.getInt("lot_count"),
closingTime
);
})
.list()
);
}
}

View File

@@ -0,0 +1,154 @@
package auctiora.db;
import lombok.experimental.UtilityClass;
import org.jdbi.v3.core.Jdbi;
/**
* Database schema DDL definitions for all tables and indexes.
* Uses text blocks (Java 15+) for clean SQL formatting.
*/
@UtilityClass
public class DatabaseSchema {
/**
* Initializes all database tables and indexes if they don't exist.
*/
public void ensureSchema(Jdbi jdbi) {
jdbi.useHandle(handle -> {
// Enable WAL mode for better concurrent access
handle.execute("PRAGMA journal_mode=WAL");
handle.execute("PRAGMA busy_timeout=10000");
handle.execute("PRAGMA synchronous=NORMAL");
createTables(handle);
createIndexes(handle);
});
}
private void createTables(org.jdbi.v3.core.Handle handle) {
// Cache table (for HTTP caching)
handle.execute("""
CREATE TABLE IF NOT EXISTS cache (
url TEXT PRIMARY KEY,
content BLOB,
timestamp REAL,
status_code INTEGER
)""");
// Auctions table (populated by external scraper)
handle.execute("""
CREATE TABLE IF NOT EXISTS auctions (
auction_id TEXT PRIMARY KEY,
url TEXT UNIQUE,
title TEXT,
location TEXT,
lots_count INTEGER,
first_lot_closing_time TEXT,
scraped_at TEXT,
city TEXT,
country TEXT,
type TEXT,
lot_count INTEGER DEFAULT 0,
closing_time TEXT,
discovered_at INTEGER
)""");
// Lots table (populated by external scraper)
handle.execute("""
CREATE TABLE IF NOT EXISTS lots (
lot_id TEXT PRIMARY KEY,
auction_id TEXT,
url TEXT UNIQUE,
title TEXT,
current_bid TEXT,
bid_count INTEGER,
closing_time TEXT,
viewing_time TEXT,
pickup_date TEXT,
location TEXT,
description TEXT,
category TEXT,
scraped_at TEXT,
sale_id INTEGER,
manufacturer TEXT,
type TEXT,
year INTEGER,
currency TEXT DEFAULT 'EUR',
closing_notified INTEGER DEFAULT 0,
starting_bid TEXT,
minimum_bid TEXT,
status TEXT,
brand TEXT,
model TEXT,
attributes_json TEXT,
first_bid_time TEXT,
last_bid_time TEXT,
bid_velocity REAL,
bid_increment REAL,
year_manufactured INTEGER,
condition_score REAL,
condition_description TEXT,
serial_number TEXT,
damage_description TEXT,
followers_count INTEGER DEFAULT 0,
estimated_min_price REAL,
estimated_max_price REAL,
lot_condition TEXT,
appearance TEXT,
estimated_min REAL,
estimated_max REAL,
next_bid_step_cents INTEGER,
condition TEXT,
category_path TEXT,
city_location TEXT,
country_code TEXT,
bidding_status TEXT,
packaging TEXT,
quantity INTEGER,
vat REAL,
buyer_premium_percentage REAL,
remarks TEXT,
reserve_price REAL,
reserve_met INTEGER,
view_count INTEGER,
FOREIGN KEY (auction_id) REFERENCES auctions(auction_id)
)""");
// Images table (populated by external scraper with URLs and local_path)
handle.execute("""
CREATE TABLE IF NOT EXISTS images (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT,
url TEXT,
local_path TEXT,
downloaded INTEGER DEFAULT 0,
labels TEXT,
processed_at INTEGER,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)""");
// Bid history table
handle.execute("""
CREATE TABLE IF NOT EXISTS bid_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
lot_id TEXT NOT NULL,
bid_amount REAL NOT NULL,
bid_time TEXT NOT NULL,
is_autobid INTEGER DEFAULT 0,
bidder_id TEXT,
bidder_number INTEGER,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (lot_id) REFERENCES lots(lot_id)
)""");
}
private void createIndexes(org.jdbi.v3.core.Handle handle) {
handle.execute("CREATE INDEX IF NOT EXISTS idx_timestamp ON cache(timestamp)");
handle.execute("CREATE INDEX IF NOT EXISTS idx_auctions_country ON auctions(country)");
handle.execute("CREATE INDEX IF NOT EXISTS idx_lots_sale_id ON lots(sale_id)");
handle.execute("CREATE INDEX IF NOT EXISTS idx_images_lot_id ON images(lot_id)");
handle.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_lot_url ON images(lot_id, url)");
handle.execute("CREATE INDEX IF NOT EXISTS idx_bid_history_lot_time ON bid_history(lot_id, bid_time)");
handle.execute("CREATE INDEX IF NOT EXISTS idx_bid_history_bidder ON bid_history(bidder_id)");
}
}

View File

@@ -0,0 +1,137 @@
package auctiora.db;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Jdbi;
import java.time.Instant;
import java.util.List;
/**
* Repository for image-related database operations using JDBI3.
* Handles image storage, object detection labels, and processing status.
*/
@Slf4j
@RequiredArgsConstructor
public class ImageRepository {
private final Jdbi jdbi;
/**
* Image record containing all image metadata.
*/
public record ImageRecord(int id, long lotId, String url, String filePath, String labels) {}
/**
* Minimal record for images needing object detection processing.
*/
public record ImageDetectionRecord(int id, long lotId, String filePath) {}
/**
* Inserts a complete image record (for testing/legacy compatibility).
* In production, scraper inserts with local_path, monitor updates labels via updateLabels.
*/
public void insert(long lotId, String url, String filePath, List<String> labels) {
jdbi.useHandle(handle ->
handle.createUpdate("""
INSERT INTO images (lot_id, url, local_path, labels, processed_at, downloaded)
VALUES (:lotId, :url, :localPath, :labels, :processedAt, 1)
""")
.bind("lotId", lotId)
.bind("url", url)
.bind("localPath", filePath)
.bind("labels", String.join(",", labels))
.bind("processedAt", Instant.now().getEpochSecond())
.execute()
);
}
/**
* Updates the labels field for an image after object detection.
*/
public void updateLabels(int imageId, List<String> labels) {
jdbi.useHandle(handle ->
handle.createUpdate("UPDATE images SET labels = :labels, processed_at = :processedAt WHERE id = :id")
.bind("labels", String.join(",", labels))
.bind("processedAt", Instant.now().getEpochSecond())
.bind("id", imageId)
.execute()
);
}
/**
* Gets the labels for a specific image.
*/
public List<String> getLabels(int imageId) {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT labels FROM images WHERE id = :id")
.bind("id", imageId)
.mapTo(String.class)
.findOne()
.map(labelsStr -> {
if (labelsStr != null && !labelsStr.isEmpty()) {
return List.of(labelsStr.split(","));
}
return List.<String>of();
})
.orElse(List.of())
);
}
/**
* Retrieves images for a specific lot.
*/
public List<ImageRecord> getImagesForLot(long lotId) {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT id, lot_id, url, local_path, labels FROM images WHERE lot_id = :lotId")
.bind("lotId", lotId)
.map((rs, ctx) -> new ImageRecord(
rs.getInt("id"),
rs.getLong("lot_id"),
rs.getString("url"),
rs.getString("local_path"),
rs.getString("labels")
))
.list()
);
}
/**
* Gets images that have been downloaded by the scraper but need object detection.
* Only returns images that have local_path set but no labels yet.
*/
public List<ImageDetectionRecord> getImagesNeedingDetection() {
return jdbi.withHandle(handle ->
handle.createQuery("""
SELECT i.id, i.lot_id, i.local_path
FROM images i
WHERE i.local_path IS NOT NULL
AND i.local_path != ''
AND (i.labels IS NULL OR i.labels = '')
""")
.map((rs, ctx) -> {
// Extract numeric lot ID from TEXT field (e.g., "A1-34732-49" -> 3473249)
String lotIdStr = rs.getString("lot_id");
long lotId = auctiora.ScraperDataAdapter.extractNumericId(lotIdStr);
return new ImageDetectionRecord(
rs.getInt("id"),
lotId,
rs.getString("local_path")
);
})
.list()
);
}
/**
* Gets the total number of images in the database.
*/
public int getImageCount() {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT COUNT(*) FROM images")
.mapTo(Integer.class)
.one()
);
}
}

View File

@@ -0,0 +1,275 @@
package auctiora.db;
import auctiora.Lot;
import auctiora.BidHistory;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Jdbi;
import java.time.LocalDateTime;
import java.util.List;
import static java.sql.Types.*;
/**
* Repository for lot-related database operations using JDBI3.
* Handles CRUD operations and queries for auction lots.
*/
@Slf4j
@RequiredArgsConstructor
public class LotRepository {
private final Jdbi jdbi;
/**
* Inserts or updates a lot (upsert operation).
* First tries UPDATE, then falls back to INSERT if lot doesn't exist.
*/
public void upsert(Lot lot) {
jdbi.useTransaction(handle -> {
// Try UPDATE first
int updated = handle.createUpdate("""
UPDATE lots SET
sale_id = :saleId,
auction_id = :auctionId,
title = :title,
description = :description,
manufacturer = :manufacturer,
type = :type,
year = :year,
category = :category,
current_bid = :currentBid,
currency = :currency,
url = :url,
closing_time = :closingTime
WHERE lot_id = :lotId
""")
.bind("saleId", String.valueOf(lot.saleId()))
.bind("auctionId", String.valueOf(lot.saleId())) // auction_id = sale_id
.bind("title", lot.title())
.bind("description", lot.description())
.bind("manufacturer", lot.manufacturer())
.bind("type", lot.type())
.bind("year", lot.year())
.bind("category", lot.category())
.bind("currentBid", lot.currentBid())
.bind("currency", lot.currency())
.bind("url", lot.url())
.bind("closingTime", lot.closingTime() != null ? lot.closingTime().toString() : null)
.bind("lotId", String.valueOf(lot.lotId()))
.execute();
if (updated == 0) {
// No rows updated, perform INSERT
handle.createUpdate("""
INSERT OR IGNORE INTO lots (
lot_id, sale_id, auction_id, title, description, manufacturer, type, year,
category, current_bid, currency, url, closing_time, closing_notified
) VALUES (
:lotId, :saleId, :auctionId, :title, :description, :manufacturer, :type, :year,
:category, :currentBid, :currency, :url, :closingTime, :closingNotified
)
""")
.bind("lotId", String.valueOf(lot.lotId()))
.bind("saleId", String.valueOf(lot.saleId()))
.bind("auctionId", String.valueOf(lot.saleId())) // auction_id = sale_id
.bind("title", lot.title())
.bind("description", lot.description())
.bind("manufacturer", lot.manufacturer())
.bind("type", lot.type())
.bind("year", lot.year())
.bind("category", lot.category())
.bind("currentBid", lot.currentBid())
.bind("currency", lot.currency())
.bind("url", lot.url())
.bind("closingTime", lot.closingTime() != null ? lot.closingTime().toString() : null)
.bind("closingNotified", lot.closingNotified() ? 1 : 0)
.execute();
}
});
}
/**
* Updates a lot with full intelligence data from GraphQL enrichment.
* Includes all 24+ intelligence fields from bidding platform.
*/
public void upsertWithIntelligence(Lot lot) {
jdbi.useHandle(handle -> {
var update = handle.createUpdate("""
UPDATE lots SET
sale_id = :saleId,
title = :title,
description = :description,
manufacturer = :manufacturer,
type = :type,
year = :year,
category = :category,
current_bid = :currentBid,
currency = :currency,
url = :url,
closing_time = :closingTime,
followers_count = :followersCount,
estimated_min = :estimatedMin,
estimated_max = :estimatedMax,
next_bid_step_cents = :nextBidStepInCents,
condition = :condition,
category_path = :categoryPath,
city_location = :cityLocation,
country_code = :countryCode,
bidding_status = :biddingStatus,
appearance = :appearance,
packaging = :packaging,
quantity = :quantity,
vat = :vat,
buyer_premium_percentage = :buyerPremiumPercentage,
remarks = :remarks,
starting_bid = :startingBid,
reserve_price = :reservePrice,
reserve_met = :reserveMet,
bid_increment = :bidIncrement,
view_count = :viewCount,
first_bid_time = :firstBidTime,
last_bid_time = :lastBidTime,
bid_velocity = :bidVelocity
WHERE lot_id = :lotId
""")
.bind("saleId", lot.saleId())
.bind("title", lot.title())
.bind("description", lot.description())
.bind("manufacturer", lot.manufacturer())
.bind("type", lot.type())
.bind("year", lot.year())
.bind("category", lot.category())
.bind("currentBid", lot.currentBid())
.bind("currency", lot.currency())
.bind("url", lot.url())
.bind("closingTime", lot.closingTime() != null ? lot.closingTime().toString() : null)
.bind("followersCount", lot.followersCount())
.bind("estimatedMin", lot.estimatedMin())
.bind("estimatedMax", lot.estimatedMax())
.bind("nextBidStepInCents", lot.nextBidStepInCents())
.bind("condition", lot.condition())
.bind("categoryPath", lot.categoryPath())
.bind("cityLocation", lot.cityLocation())
.bind("countryCode", lot.countryCode())
.bind("biddingStatus", lot.biddingStatus())
.bind("appearance", lot.appearance())
.bind("packaging", lot.packaging())
.bind("quantity", lot.quantity())
.bind("vat", lot.vat())
.bind("buyerPremiumPercentage", lot.buyerPremiumPercentage())
.bind("remarks", lot.remarks())
.bind("startingBid", lot.startingBid())
.bind("reservePrice", lot.reservePrice())
.bind("reserveMet", lot.reserveMet() != null && lot.reserveMet() ? 1 : null)
.bind("bidIncrement", lot.bidIncrement())
.bind("viewCount", lot.viewCount())
.bind("firstBidTime", lot.firstBidTime() != null ? lot.firstBidTime().toString() : null)
.bind("lastBidTime", lot.lastBidTime() != null ? lot.lastBidTime().toString() : null)
.bind("bidVelocity", lot.bidVelocity())
.bind("lotId", lot.lotId());
int updated = update.execute();
if (updated == 0) {
log.warn("Failed to update lot {} - lot not found in database", lot.lotId());
}
});
}
/**
* Updates only the current bid for a lot (lightweight update).
*/
public void updateCurrentBid(Lot lot) {
jdbi.useHandle(handle ->
handle.createUpdate("UPDATE lots SET current_bid = :bid WHERE lot_id = :lotId")
.bind("bid", lot.currentBid())
.bind("lotId", String.valueOf(lot.lotId()))
.execute()
);
}
/**
* Updates notification flags for a lot.
*/
public void updateNotificationFlags(Lot lot) {
jdbi.useHandle(handle ->
handle.createUpdate("UPDATE lots SET closing_notified = :notified WHERE lot_id = :lotId")
.bind("notified", lot.closingNotified() ? 1 : 0)
.bind("lotId", String.valueOf(lot.lotId()))
.execute()
);
}
/**
* Retrieves all active lots.
* Note: Despite the name, this returns ALL lots (legacy behavior for backward compatibility).
*/
public List<Lot> getActiveLots() {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT * FROM lots")
.map((rs, ctx) -> auctiora.ScraperDataAdapter.fromScraperLot(rs))
.list()
);
}
/**
* Retrieves all lots from the database.
*/
public List<Lot> getAllLots() {
return jdbi.withHandle(handle ->
handle.createQuery("SELECT * FROM lots")
.map((rs, ctx) -> auctiora.ScraperDataAdapter.fromScraperLot(rs))
.list()
);
}
/**
* Retrieves bid history for a specific lot.
*/
public List<BidHistory> getBidHistory(String lotId) {
return jdbi.withHandle(handle ->
handle.createQuery("""
SELECT id, lot_id, bid_amount, bid_time, is_autobid, bidder_id, bidder_number
FROM bid_history
WHERE lot_id = :lotId
ORDER BY bid_time DESC
""")
.bind("lotId", lotId)
.map((rs, ctx) -> new BidHistory(
rs.getInt("id"),
rs.getString("lot_id"),
rs.getDouble("bid_amount"),
LocalDateTime.parse(rs.getString("bid_time")),
rs.getInt("is_autobid") != 0,
rs.getString("bidder_id"),
(Integer) rs.getObject("bidder_number")
))
.list()
);
}
/**
* Inserts bid history records in batch.
*/
public void insertBidHistory(List<BidHistory> bidHistory) {
jdbi.useHandle(handle -> {
var batch = handle.prepareBatch("""
INSERT OR IGNORE INTO bid_history (
lot_id, bid_amount, bid_time, is_autobid, bidder_id, bidder_number
) VALUES (:lotId, :bidAmount, :bidTime, :isAutobid, :bidderId, :bidderNumber)
""");
bidHistory.forEach(bid ->
batch.bind("lotId", bid.lotId())
.bind("bidAmount", bid.bidAmount())
.bind("bidTime", bid.bidTime().toString())
.bind("isAutobid", bid.isAutobid() ? 1 : 0)
.bind("bidderId", bid.bidderId())
.bind("bidderNumber", bid.bidderNumber())
.add()
);
batch.execute();
});
}
}

View File

@@ -1,8 +0,0 @@
package com.auction.scraper;
public class Main {
public static void main(String[] args) {
System.out.println("Troostwijk Auction Scraper");
System.out.println("Use TroostwijkScraper class to run the scraper.");
}
}

View File

@@ -1,801 +0,0 @@
package com.auction.scraper;
/*
* TroostwijkScraper
*
* This example shows how you could build a Javabased scraper for the Dutch
* auctions on Troostwijk Auctions. The scraper uses a combination of
* HTTP requests and HTML parsing with the jsoup library to discover active
* auctions, calls Troostwijk's internal JSON API to fetch lot (kavel) data
* efficiently, writes the results into a local SQLite database, performs
* object detection on lot images using OpenCV's DNN module, and sends
* desktop/email notifications when bids change or lots are about to expire.
* The implementation uses well known open source libraries for each of these
* concerns. You can adjust the API endpoints and CSS selectors as
* Troostwijk's site evolves. The code is organised into small helper
* classes to make it easier to maintain.
*
* Dependencies (add these to your Maven/Gradle project):
*
* - org.jsoup:jsoup:1.17.2 HTML parser and HTTP client.
* - com.fasterxml.jackson.core:jackson-databind:2.17.0 JSON parsing.
* - org.xerial:sqlite-jdbc:3.45.1.0 SQLite JDBC driver.
* - com.sun.mail:javax.mail:1.6.2 JavaMail for email notifications (free).
* - org.openpnp:opencv:4.9.0-0 (with native libraries) OpenCV for image
* processing and object detection.
*
* Before running this program you must ensure that the native OpenCV
* binaries are on your library path (e.g. via -Djava.library.path).
* Desktop notifications work out of the box on Windows, macOS, and Linux.
* For email notifications, you need a Gmail account with an app password
* (free, requires 2FA enabled). See https://support.google.com/accounts/answer/185833
*
* The scraper performs four major tasks:
* 1. Discover all auctions located in the Netherlands.
* 2. For each auction, fetch all lots (kavels) including images and
* bidding information, and persist the data into SQLite tables.
* 3. Monitor bidding and closing times on a schedule and send desktop/email
* notifications when bids change or lots are about to expire.
* 4. Run object detection on downloaded lot images to automatically
* label objects using a YOLO model. The results are stored in the
* database for later search.
*/
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgcodecs.Imgcodecs;
import static org.opencv.dnn.Dnn.DNN_BACKEND_OPENCV;
import static org.opencv.dnn.Dnn.DNN_TARGET_CPU;
/**
* Main scraper class. It encapsulates the logic for scraping auctions,
* persisting data, scheduling updates, and performing object detection.
*/
public class TroostwijkScraper {
// Base URLs adjust these if Troostwijk changes their site structure
private static final String AUCTIONS_PAGE = "https://www.troostwijkauctions.com/nl/auctions";
private static final String LOT_API = "https://api.troostwijkauctions.com/lot/7/list";
// HTTP client used for API calls
private final HttpClient httpClient;
private final ObjectMapper objectMapper;
private final DatabaseService db;
private final NotificationService notifier;
private final ObjectDetectionService detector;
/**
* Constructor. Creates supporting services and ensures the database
* tables exist.
*
* @param databasePath Path to SQLite database file
* @param notificationConfig "desktop" for desktop only, or "smtp:user:pass:toEmail" for email
* @param unused Unused parameter (kept for compatibility)
* @param yoloCfgPath Path to YOLO configuration file
* @param yoloWeightsPath Path to YOLO weights file
* @param classNamesPath Path to file containing class names
*/
public TroostwijkScraper(String databasePath, String notificationConfig, String unused,
String yoloCfgPath, String yoloWeightsPath, String classNamesPath) throws SQLException, IOException {
this.httpClient = HttpClient.newHttpClient();
this.objectMapper = new ObjectMapper();
this.db = new DatabaseService(databasePath);
this.notifier = new NotificationService(notificationConfig, unused);
this.detector = new ObjectDetectionService(yoloCfgPath, yoloWeightsPath, classNamesPath);
// initialize DB
db.ensureSchema();
}
/**
* Discovers all active Dutch auctions by crawling the auctions page.
*
* Troostwijk lists auctions for many countries on one page. We parse
* the page with jsoup (an HTML parser that fetches and parses realworld
* HTML easily【438902460386021†L14-L24】) and filter auctions whose location
* contains ", NL" (indicating the Netherlands). Each auction link
* contains a unique sale ID which we extract from its URL.
*
* @return a list of sale identifiers for auctions located in NL
*/
public List<Integer> discoverDutchAuctions() {
List<Integer> saleIds = new ArrayList<>();
try {
// Fetch the auctions overview page
Document doc = Jsoup.connect(AUCTIONS_PAGE).get();
// Select all anchor elements that represent an auction listing.
// The exact selector may change; inspect the page with your browsers
// developer tools and update accordingly.
Elements auctionLinks = doc.select("a[href][data-id]");
for (Element link : auctionLinks) {
Element locationElement = link.selectFirst(".auction-location");
String location = locationElement != null ? locationElement.text() : "";
if (location.contains(", NL")) {
// Extract saleID from the data-id attribute or href
String saleIdStr = link.attr("data-id");
if (saleIdStr.isEmpty()) {
// Fallback: parse from URL path, e.g. /nl/sale/27213/machines
String href = link.attr("href");
String[] parts = href.split("/");
for (String p : parts) {
if (p.matches("\\d+")) {
saleIdStr = p;
break;
}
}
}
try {
int saleId = Integer.parseInt(saleIdStr);
saleIds.add(saleId);
} catch (NumberFormatException ignored) {
// not a sale ID
}
}
}
} catch (IOException e) {
System.err.println("Failed to discover auctions: " + e.getMessage());
}
return saleIds;
}
/**
* Retrieves all lots for a given sale ID using Troostwijks internal JSON
* API. The API accepts parameters such as batchSize, offset, and saleID.
* A large batchSize returns many lots at once【610752406306016†L124-L134】. We loop
* until no further results are returned. Each JSON result is mapped to
* our Lot domain object and persisted to the database.
*
* @param saleId the sale identifier
*/
public void fetchLotsForSale(int saleId) {
int batchSize = 200;
int offset = 0;
boolean more = true;
while (more) {
try {
String url = LOT_API + "?batchSize=" + batchSize
+ "&listType=7&offset=" + offset
+ "&sortOption=0&saleID=" + saleId
+ "&parentID=0&relationID=0&buildversion=201807311";
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.header("Accept", "application/json")
.GET()
.build();
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() != 200) {
System.err.println("API call failed for sale " + saleId + " with status " + response.statusCode());
break;
}
JsonNode root = objectMapper.readTree(response.body());
JsonNode results = root.path("results");
if (!results.isArray() || results.isEmpty()) {
more = false;
break;
}
for (JsonNode node : results) {
Lot lot = new Lot();
lot.saleId = saleId;
lot.lotId = node.path("lotID").asInt();
lot.title = node.path("t").asText();
lot.description = node.path("d").asText();
lot.manufacturer = node.path("mf").asText();
lot.type = node.path("typ").asText();
lot.year = node.path("yb").asInt();
lot.category = node.path("lc").asText();
// Current bid; field names may differ (e.g. currentBid or cb)
lot.currentBid = node.path("cb").asDouble();
lot.currency = node.path("cu").asText();
lot.url = "https://www.troostwijkauctions.com/nl" + node.path("url").asText();
// Save basic lot info into DB
db.upsertLot(lot);
// Download images and perform object detection
List<String> imageUrls = new ArrayList<>();
JsonNode imgs = node.path("imgs");
if (imgs.isArray()) {
for (JsonNode imgNode : imgs) {
String imgUrl = imgNode.asText();
imageUrls.add(imgUrl);
}
}
for (String imgUrl : imageUrls) {
String fileName = downloadImage(imgUrl, saleId, lot.lotId);
if (fileName != null) {
// run object detection once per image
List<String> labels = detector.detectObjects(fileName);
db.insertImage(lot.lotId, imgUrl, fileName, labels);
}
}
}
offset += batchSize;
} catch (IOException | InterruptedException e) {
System.err.println("Error fetching lots for sale " + saleId + ": " + e.getMessage());
more = false;
} catch (SQLException e) {
System.err.println("Database error: " + e.getMessage());
}
}
}
/**
* Downloads an image from the given URL to a local directory. Images
* are stored under "images/<saleId>/<lotId>/" to keep them organised.
*
* @param imageUrl remote image URL
* @param saleId sale identifier
* @param lotId lot identifier
* @return absolute path to saved file or null on failure
*/
private String downloadImage(String imageUrl, int saleId, int lotId) {
try {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(imageUrl))
.GET()
.build();
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
if (response.statusCode() == 200) {
Path dir = Paths.get("images", String.valueOf(saleId), String.valueOf(lotId));
Files.createDirectories(dir);
String fileName = Paths.get(imageUrl).getFileName().toString();
Path dest = dir.resolve(fileName);
Files.copy(response.body(), dest);
return dest.toAbsolutePath().toString();
}
} catch (IOException | InterruptedException e) {
System.err.println("Failed to download image " + imageUrl + ": " + e.getMessage());
}
return null;
}
/**
* Schedules periodic monitoring of all lots. The scheduler runs every
* hour to refresh current bids and closing times. For lots that
* are within 30 minutes of closing, it increases the polling frequency
* automatically. When a new bid is detected or a lot is about to
* expire, a Pushover notification is sent to the configured user.
* Note: In production, ensure proper shutdown handling for the scheduler.
*/
public ScheduledExecutorService scheduleMonitoring() {
ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
scheduler.scheduleAtFixedRate(() -> {
try {
List<Lot> activeLots = db.getActiveLots();
for (Lot lot : activeLots) {
// refresh the lot's bidding information via API
refreshLotBid(lot);
// check closing time to adjust monitoring
long minutesLeft = lot.minutesUntilClose();
if (minutesLeft < 30) {
// send warning when within 5 minutes
if (minutesLeft <= 5 && !lot.closingNotified) {
notifier.sendNotification("Kavel " + lot.lotId + " sluit binnen " + minutesLeft + " min.",
"Lot nearing closure", 1);
lot.closingNotified = true;
db.updateLotNotificationFlags(lot);
}
// schedule additional quick check for this lot
scheduler.schedule(() -> refreshLotBid(lot), 5, TimeUnit.MINUTES);
}
}
} catch (SQLException e) {
System.err.println("Error during scheduled monitoring: " + e.getMessage());
}
}, 0, 1, TimeUnit.HOURS);
return scheduler;
}
/**
* Refreshes the bid for a single lot and sends notification if it has
* changed since the last check. The method calls the same API used for
* initial scraping but only extracts the current bid for the given lot.
*
* @param lot the lot to refresh
*/
private void refreshLotBid(Lot lot) {
try {
String url = LOT_API + "?batchSize=1&listType=7&offset=0&sortOption=0&saleID=" + lot.saleId
+ "&parentID=0&relationID=0&buildversion=201807311&lotID=" + lot.lotId;
HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build();
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() != 200) return;
JsonNode root = objectMapper.readTree(response.body());
JsonNode results = root.path("results");
if (results.isArray() && !results.isEmpty()) {
JsonNode node = results.get(0);
double newBid = node.path("cb").asDouble();
if (Double.compare(newBid, lot.currentBid) > 0) {
double previous = lot.currentBid;
lot.currentBid = newBid;
db.updateLotCurrentBid(lot);
String msg = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)", lot.lotId, newBid, previous);
notifier.sendNotification(msg, "Kavel bieding update", 0);
}
}
} catch (IOException | InterruptedException | SQLException e) {
System.err.println("Failed to refresh bid for lot " + lot.lotId + ": " + e.getMessage());
}
}
/**
* Entry point. Configure database location, notification settings, and
* YOLO model paths here before running. Once started the scraper
* discovers Dutch auctions, scrapes lots, and begins monitoring.
*/
public static void main(String[] args) throws Exception {
// Configuration parameters (replace with your own values)
String databaseFile = "troostwijk.db";
// Notification configuration - choose one:
// Option 1: Desktop notifications only (free, no setup required)
String notificationConfig = System.getenv().getOrDefault("NOTIFICATION_CONFIG", "desktop");
// Option 2: Desktop + Email via Gmail (free, requires Gmail app password)
// Format: "smtp:username:appPassword:toEmail"
// Example: "smtp:your.email@gmail.com:abcd1234efgh5678:recipient@example.com"
// Get app password: Google Account > Security > 2-Step Verification > App passwords
String yoloCfg = "models/yolov4.cfg"; // path to YOLO config file
String yoloWeights = "models/yolov4.weights"; // path to YOLO weights file
String yoloClasses = "models/coco.names"; // list of class names
// Load native OpenCV library
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
TroostwijkScraper scraper = new TroostwijkScraper(databaseFile, notificationConfig, "",
yoloCfg, yoloWeights, yoloClasses);
// Step 1: Discover auctions in NL
List<Integer> auctions = scraper.discoverDutchAuctions();
System.out.println("Found auctions: " + auctions);
// Step 2: Fetch lots for each auction
for (int saleId : auctions) {
scraper.fetchLotsForSale(saleId);
}
// Step 3: Start monitoring bids and closures
scraper.scheduleMonitoring();
}
// ----------------------------------------------------------------------
// Domain classes and services
// ----------------------------------------------------------------------
/**
* Simple POJO representing a lot (kavel) in an auction. It keeps track
* of the sale it belongs to, current bid and closing time. The method
* minutesUntilClose computes how many minutes remain until the lot closes.
*/
static class Lot {
int saleId;
int lotId;
String title;
String description;
String manufacturer;
String type;
int year;
String category;
double currentBid;
String currency;
String url;
LocalDateTime closingTime; // null if unknown
boolean closingNotified;
long minutesUntilClose() {
if (closingTime == null) return Long.MAX_VALUE;
return java.time.Duration.between(LocalDateTime.now(), closingTime).toMinutes();
}
}
/**
* Service for persisting auctions, lots, images, and object labels into
* a SQLite database. Uses the Xerial JDBC driver which connects to
* SQLite via a URL of the form "jdbc:sqlite:path_to_file"【329850066306528†L40-L63】.
*/
static class DatabaseService {
private final String url;
DatabaseService(String dbPath) {
this.url = "jdbc:sqlite:" + dbPath;
}
/**
* Creates tables if they do not already exist. The schema includes
* tables for sales, lots, images, and object labels. This method is
* idempotent; it can be called multiple times.
*/
void ensureSchema() throws SQLException {
try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
// Sales table
stmt.execute("CREATE TABLE IF NOT EXISTS sales ("
+ "sale_id INTEGER PRIMARY KEY,"
+ "title TEXT,"
+ "location TEXT,"
+ "closing_time TEXT"
+ ")");
// Lots table
stmt.execute("CREATE TABLE IF NOT EXISTS lots ("
+ "lot_id INTEGER PRIMARY KEY,"
+ "sale_id INTEGER,"
+ "title TEXT,"
+ "description TEXT,"
+ "manufacturer TEXT,"
+ "type TEXT,"
+ "year INTEGER,"
+ "category TEXT,"
+ "current_bid REAL,"
+ "currency TEXT,"
+ "url TEXT,"
+ "closing_time TEXT,"
+ "closing_notified INTEGER DEFAULT 0,"
+ "FOREIGN KEY (sale_id) REFERENCES sales(sale_id)"
+ ")");
// Images table
stmt.execute("CREATE TABLE IF NOT EXISTS images ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "lot_id INTEGER,"
+ "url TEXT,"
+ "file_path TEXT,"
+ "labels TEXT,"
+ "FOREIGN KEY (lot_id) REFERENCES lots(lot_id)"
+ ")");
}
}
/**
* Inserts or updates a lot record. Uses INSERT OR REPLACE to
* implement upsert semantics so that existing rows are replaced.
*/
synchronized void upsertLot(Lot lot) throws SQLException {
String sql = "INSERT INTO lots (lot_id, sale_id, title, description, manufacturer, type, year, category, current_bid, currency, url, closing_time, closing_notified)"
+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+ " ON CONFLICT(lot_id) DO UPDATE SET "
+ "sale_id = excluded.sale_id, title = excluded.title, description = excluded.description, "
+ "manufacturer = excluded.manufacturer, type = excluded.type, year = excluded.year, category = excluded.category, "
+ "current_bid = excluded.current_bid, currency = excluded.currency, url = excluded.url, closing_time = excluded.closing_time";
try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(sql)) {
ps.setInt(1, lot.lotId);
ps.setInt(2, lot.saleId);
ps.setString(3, lot.title);
ps.setString(4, lot.description);
ps.setString(5, lot.manufacturer);
ps.setString(6, lot.type);
ps.setInt(7, lot.year);
ps.setString(8, lot.category);
ps.setDouble(9, lot.currentBid);
ps.setString(10, lot.currency);
ps.setString(11, lot.url);
ps.setString(12, lot.closingTime != null ? lot.closingTime.toString() : null);
ps.setInt(13, lot.closingNotified ? 1 : 0);
ps.executeUpdate();
}
}
/**
* Inserts a new image record. Each image is associated with a lot and
* stores both the original URL and the local file path. Detected
* labels are stored as a comma separated string.
*/
synchronized void insertImage(int lotId, String url, String filePath, List<String> labels) throws SQLException {
String sql = "INSERT INTO images (lot_id, url, file_path, labels) VALUES (?, ?, ?, ?)";
try (Connection conn = DriverManager.getConnection(this.url); PreparedStatement ps = conn.prepareStatement(sql)) {
ps.setInt(1, lotId);
ps.setString(2, url);
ps.setString(3, filePath);
ps.setString(4, String.join(",", labels));
ps.executeUpdate();
}
}
/**
* Retrieves all lots that are still active (i.e., have a closing time
* in the future or unknown). Only these lots need to be monitored.
*/
synchronized List<Lot> getActiveLots() throws SQLException {
List<Lot> list = new ArrayList<>();
String sql = "SELECT lot_id, sale_id, current_bid, currency, closing_time, closing_notified FROM lots";
try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) {
ResultSet rs = stmt.executeQuery(sql);
while (rs.next()) {
Lot lot = new Lot();
lot.lotId = rs.getInt("lot_id");
lot.saleId = rs.getInt("sale_id");
lot.currentBid = rs.getDouble("current_bid");
lot.currency = rs.getString("currency");
String closing = rs.getString("closing_time");
lot.closingNotified = rs.getInt("closing_notified") != 0;
if (closing != null) {
lot.closingTime = LocalDateTime.parse(closing);
}
list.add(lot);
}
}
return list;
}
/**
* Updates the current bid of a lot after a bid refresh.
*/
synchronized void updateLotCurrentBid(Lot lot) throws SQLException {
try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(
"UPDATE lots SET current_bid = ? WHERE lot_id = ?")) {
ps.setDouble(1, lot.currentBid);
ps.setInt(2, lot.lotId);
ps.executeUpdate();
}
}
/**
* Updates the closingNotified flag of a lot (set to 1 when we have
* warned the user about its imminent closure).
*/
synchronized void updateLotNotificationFlags(Lot lot) throws SQLException {
try (Connection conn = DriverManager.getConnection(url); PreparedStatement ps = conn.prepareStatement(
"UPDATE lots SET closing_notified = ? WHERE lot_id = ?")) {
ps.setInt(1, lot.closingNotified ? 1 : 0);
ps.setInt(2, lot.lotId);
ps.executeUpdate();
}
}
}
/**
* Service for sending notifications via desktop notifications and/or email.
* Supports free notification methods:
* 1. Desktop notifications (Windows/Linux/macOS system tray)
* 2. Email via Gmail SMTP (free, requires app password)
*
* Configuration:
* - For email: Set notificationEmail to your Gmail address
* - Enable 2FA in Gmail and create an App Password
* - Use format "smtp:username:appPassword:toEmail" for credentials
* - Or use "desktop" for desktop-only notifications
*/
static class NotificationService {
private final boolean useDesktop;
private final boolean useEmail;
private final String smtpUsername;
private final String smtpPassword;
private final String toEmail;
/**
* Creates a notification service.
*
* @param config "desktop" for desktop only, or "smtp:username:password:toEmail" for email
* @param unusedParam Kept for compatibility (can pass empty string)
*/
NotificationService(String config, String unusedParam) {
if ("desktop".equalsIgnoreCase(config)) {
this.useDesktop = true;
this.useEmail = false;
this.smtpUsername = null;
this.smtpPassword = null;
this.toEmail = null;
} else if (config.startsWith("smtp:")) {
String[] parts = config.split(":", 4);
if (parts.length != 4) {
throw new IllegalArgumentException("Email config must be 'smtp:username:password:toEmail'");
}
this.useDesktop = true; // Always include desktop
this.useEmail = true;
this.smtpUsername = parts[1];
this.smtpPassword = parts[2];
this.toEmail = parts[3];
} else {
throw new IllegalArgumentException("Config must be 'desktop' or 'smtp:username:password:toEmail'");
}
}
/**
* Sends notification via configured channels.
*
* @param message The message body
* @param title Message title
* @param priority Priority level (0=normal, 1=high)
*/
void sendNotification(String message, String title, int priority) {
if (useDesktop) {
sendDesktopNotification(title, message, priority);
}
if (useEmail) {
sendEmailNotification(title, message, priority);
}
}
/**
* Sends a desktop notification using system tray.
* Works on Windows, macOS, and Linux with desktop environments.
*/
private void sendDesktopNotification(String title, String message, int priority) {
try {
if (java.awt.SystemTray.isSupported()) {
java.awt.SystemTray tray = java.awt.SystemTray.getSystemTray();
java.awt.Image image = java.awt.Toolkit.getDefaultToolkit()
.createImage(new byte[0]); // Empty image
java.awt.TrayIcon trayIcon = new java.awt.TrayIcon(image, "Troostwijk Scraper");
trayIcon.setImageAutoSize(true);
java.awt.TrayIcon.MessageType messageType = priority > 0
? java.awt.TrayIcon.MessageType.WARNING
: java.awt.TrayIcon.MessageType.INFO;
tray.add(trayIcon);
trayIcon.displayMessage(title, message, messageType);
// Remove icon after 2 seconds to avoid clutter
Thread.sleep(2000);
tray.remove(trayIcon);
System.out.println("Desktop notification sent: " + title);
} else {
System.out.println("Desktop notifications not supported, logging: " + title + " - " + message);
}
} catch (Exception e) {
System.err.println("Desktop notification failed: " + e.getMessage());
}
}
/**
* Sends email notification via Gmail SMTP (free).
* Uses Gmail's SMTP server with app password authentication.
*/
private void sendEmailNotification(String title, String message, int priority) {
try {
java.util.Properties props = new java.util.Properties();
props.put("mail.smtp.auth", "true");
props.put("mail.smtp.starttls.enable", "true");
props.put("mail.smtp.host", "smtp.gmail.com");
props.put("mail.smtp.port", "587");
props.put("mail.smtp.ssl.trust", "smtp.gmail.com");
javax.mail.Session session = javax.mail.Session.getInstance(props,
new javax.mail.Authenticator() {
protected javax.mail.PasswordAuthentication getPasswordAuthentication() {
return new javax.mail.PasswordAuthentication(smtpUsername, smtpPassword);
}
});
javax.mail.Message msg = new javax.mail.internet.MimeMessage(session);
msg.setFrom(new javax.mail.internet.InternetAddress(smtpUsername));
msg.setRecipients(javax.mail.Message.RecipientType.TO,
javax.mail.internet.InternetAddress.parse(toEmail));
msg.setSubject("[Troostwijk] " + title);
msg.setText(message);
msg.setSentDate(new java.util.Date());
if (priority > 0) {
msg.setHeader("X-Priority", "1");
msg.setHeader("Importance", "High");
}
javax.mail.Transport.send(msg);
System.out.println("Email notification sent: " + title);
} catch (Exception e) {
System.err.println("Email notification failed: " + e.getMessage());
}
}
}
/**
* Service for performing object detection on images using OpenCVs DNN
* module. The DNN module can load pretrained models from several
* frameworks (Darknet, TensorFlow, ONNX, etc.)【784097309529506†L209-L233】. Here
* we load a YOLO model (Darknet) by specifying the configuration and
* weights files. For each image we run a forward pass and return a
* list of detected class labels.
*/
static class ObjectDetectionService {
private final Net net;
private final List<String> classNames;
ObjectDetectionService(String cfgPath, String weightsPath, String classNamesPath) throws IOException {
// Load network
this.net = Dnn.readNetFromDarknet(cfgPath, weightsPath);
this.net.setPreferableBackend(DNN_BACKEND_OPENCV);
this.net.setPreferableTarget(DNN_TARGET_CPU);
// Load class names (one per line)
this.classNames = Files.readAllLines(Paths.get(classNamesPath));
}
/**
* Detects objects in the given image file and returns a list of
* humanreadable labels. Only detections above a confidence
* threshold are returned. For brevity this method omits drawing
* bounding boxes. See the OpenCV DNN documentation for details on
* postprocessing【784097309529506†L324-L344】.
*
* @param imagePath absolute path to the image
* @return list of detected class names
*/
List<String> detectObjects(String imagePath) {
List<String> labels = new ArrayList<>();
Mat image = Imgcodecs.imread(imagePath);
if (image.empty()) return labels;
// Create a 4D blob from the image
Mat blob = Dnn.blobFromImage(image, 1.0 / 255.0, new Size(416, 416), new Scalar(0, 0, 0), true, false);
net.setInput(blob);
List<Mat> outs = new ArrayList<>();
List<String> outNames = getOutputLayerNames(net);
net.forward(outs, outNames);
// Postprocess: for each detection compute score and choose class
float confThreshold = 0.5f;
for (Mat out : outs) {
for (int i = 0; i < out.rows(); i++) {
double[] data = out.get(i, 0);
if (data == null) continue;
// The first 5 numbers are bounding box, then class scores
double[] scores = new double[classNames.size()];
System.arraycopy(data, 5, scores, 0, scores.length);
int classId = argMax(scores);
double confidence = scores[classId];
if (confidence > confThreshold) {
String label = classNames.get(classId);
if (!labels.contains(label)) {
labels.add(label);
}
}
}
}
return labels;
}
/**
* Returns the indexes of the output layers in the network. YOLO
* automatically discovers its output layers; other models may require
* manually specifying them【784097309529506†L356-L365】.
*/
private List<String> getOutputLayerNames(Net net) {
List<String> names = new ArrayList<>();
List<Integer> outLayers = net.getUnconnectedOutLayers().toList();
List<String> layersNames = net.getLayerNames();
for (Integer i : outLayers) {
names.add(layersNames.get(i - 1));
}
return names;
}
/**
* Returns the index of the maximum value in the array.
*/
private int argMax(double[] array) {
int best = 0;
double max = array[0];
for (int i = 1; i < array.length; i++) {
if (array[i] > max) {
max = array[i];
best = i;
}
}
return best;
}
}
}

View File

@@ -0,0 +1,7 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
<rect width="100" height="100" fill="#2563eb" rx="15"/>
<path d="M25 40 L50 20 L75 40 L75 70 L25 70 Z" fill="#ffffff" stroke="#ffffff" stroke-width="2"/>
<circle cx="50" cy="45" r="8" fill="#2563eb"/>
<rect x="40" y="55" width="20" height="3" fill="#2563eb"/>
<text x="50" y="90" font-family="Arial" font-size="12" fill="#ffffff" text-anchor="middle" font-weight="bold">AUCTION</text>
</svg>

After

Width:  |  Height:  |  Size: 465 B

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,224 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Scrape-UI 1 - Enterprise</title>
<script src="https://cdn.tailwindcss.com"></script>
<style>
.gradient-bg {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
}
.card-hover {
transition: all 0.3s ease;
}
.card-hover:hover {
transform: translateY(-5px);
box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04);
}
</style>
</head>
<body class="bg-gray-50 min-h-screen">
<!-- Header -->
<header class="gradient-bg text-white py-8">
<div class="container mx-auto px-4">
<h1 class="text-4xl font-bold mb-2">Scrape-UI Enterprise</h1>
<p class="text-xl opacity-90">Powered by Quarkus + Modern Frontend</p>
</div>
</header>
<!-- Main Content -->
<main class="container mx-auto px-4 py-8">
<!-- API Status Card -->
<!-- API & Build Status Card -->
<div class="bg-white rounded-lg shadow-md p-6 mb-8 card-hover">
<h2 class="text-2xl font-bold mb-4 text-gray-800">Build & Runtime Status</h2>
<div id="api-status" class="space-y-4">
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
<!-- Build Information -->
<div class="bg-blue-50 p-4 rounded-lg">
<h3 class="font-semibold text-blue-800 mb-2">📦 Maven Build</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
<span class="text-gray-600">Group:</span>
<span class="font-mono font-medium" id="build-group">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Artifact:</span>
<span class="font-mono font-medium" id="build-artifact">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Version:</span>
<span class="font-mono font-medium px-2 py-1 bg-blue-100 rounded" id="build-version">-</span>
</div>
</div>
</div>
<!-- Runtime Information -->
<div class="bg-green-50 p-4 rounded-lg">
<h3 class="font-semibold text-green-800 mb-2">🚀 Runtime</h3>
<div class="space-y-2 text-sm">
<div class="flex justify-between">
<span class="text-gray-600">Status:</span>
<span class="px-2 py-1 rounded-full text-xs font-semibold bg-green-100 text-green-800" id="runtime-status">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Java:</span>
<span class="font-mono" id="java-version">-</span>
</div>
<div class="flex justify-between">
<span class="text-gray-600">Platform:</span>
<span class="font-mono" id="runtime-os">-</span>
</div>
</div>
</div>
</div>
<!-- Timestamp & Additional Info -->
<div class="pt-4 border-t">
<div class="flex justify-between items-center">
<div>
<p class="text-sm text-gray-500">Last Updated</p>
<p class="font-medium" id="last-updated">-</p>
</div>
<button onclick="fetchStatus()" class="bg-gray-100 hover:bg-gray-200 text-gray-700 px-4 py-2 rounded-lg text-sm transition-colors">
🔄 Refresh
</button>
</div>
</div>
</div>
</div>
<!-- API Response Card -->
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h2 class="text-2xl font-bold mb-4 text-gray-800">API Test</h2>
<button id="test-api" class="bg-blue-600 text-white px-6 py-2 rounded-lg hover:bg-blue-700 transition-colors mb-4">
Test Greeting API
</button>
<div id="api-response" class="bg-gray-100 p-4 rounded-lg">
<pre class="text-sm text-gray-700">Click the button to test the API</pre>
</div>
</div>
<!-- Features Grid -->
<div class="grid grid-cols-1 md:grid-cols-3 gap-6 mt-8">
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">⚡ Quarkus Backend</h3>
<p class="text-gray-600">Fast startup, low memory footprint, optimized for containers</p>
</div>
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">🚀 REST API</h3>
<p class="text-gray-600">RESTful endpoints with JSON responses</p>
</div>
<div class="bg-white rounded-lg shadow-md p-6 card-hover">
<h3 class="text-xl font-semibold mb-2 text-gray-800">🎨 Modern UI</h3>
<p class="text-gray-600">Responsive design with Tailwind CSS</p>
</div>
</div>
</main>
<script>
// Fetch API status on load
async function fetchStatus() {
try {
const response = await fetch('/api/status')
if (!response.ok) {
throw new Error(`HTTP ${ response.status }: ${ response.statusText }`)
}
const data = await response.json()
// Update Build Information
document.getElementById('build-group').textContent = data.groupId || 'N/A'
document.getElementById('build-artifact').textContent = data.artifactId || data.name || 'N/A'
document.getElementById('build-version').textContent = data.version || 'N/A'
// Update Runtime Information
document.getElementById('runtime-status').textContent = data.status || 'unknown'
document.getElementById('java-version').textContent = data.javaVersion || System.getProperty?.('java.version') || 'N/A'
document.getElementById('runtime-os').textContent = data.os || 'N/A'
// Update Timestamp
const timestamp = data.timestamp ? new Date(data.timestamp).toLocaleString() : 'N/A'
document.getElementById('last-updated').textContent = timestamp
// Update status badge color based on status
const statusBadge = document.getElementById('runtime-status')
if (data.status?.toLowerCase() === 'running') {
statusBadge.className = 'px-2 py-1 rounded-full text-xs font-semibold bg-green-100 text-green-800'
} else {
statusBadge.className = 'px-2 py-1 rounded-full text-xs font-semibold bg-yellow-100 text-yellow-800'
}
} catch (error) {
console.error('Error fetching status:', error)
document.getElementById('api-status').innerHTML = `
<div class="bg-red-50 border-l-4 border-red-500 p-4">
<div class="flex">
<div class="flex-shrink-0">
<svg class="h-5 w-5 text-red-400" viewBox="0 0 20 20" fill="currentColor">
<path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clip-rule="evenodd" />
</svg>
</div>
<div class="ml-3">
<p class="text-sm text-red-700">Failed to load status: ${ error.message }</p>
<button onclick="fetchStatus()" class="mt-2 text-sm text-red-700 hover:text-red-600 font-medium">
Retry →
</button>
</div>
</div>
</div>
`
}
}
// Fetch API status on load
async function fetchStatus3() {
try {
const response = await fetch('/api/status')
const data = await response.json()
document.getElementById('api-status').innerHTML = `
<p><strong>Application:</strong> ${ data.application }</p>
<p><strong>Status:</strong> <span class="text-green-600 font-semibold">${ data.status }</span></p>
<p><strong>Version:</strong> ${ data.version }</p>
<p><strong>Timestamp:</strong> ${ data.timestamp }</p>
`
} catch (error) {
document.getElementById('api-status').innerHTML = `
<p class="text-red-600">Error loading status: ${ error.message }</p>
`
}
}
// Test greeting API
document.getElementById('test-api').addEventListener('click', async () => {
try {
const response = await fetch('/api/hello')
const data = await response.json()
document.getElementById('api-response').innerHTML = `
<pre class="text-sm text-gray-700">${ JSON.stringify(data, null, 2) }</pre>
`
} catch (error) {
document.getElementById('api-response').innerHTML = `
<pre class="text-sm text-red-600">Error: ${ error.message }</pre>
`
}
})
// Auto-refresh every 30 seconds
let refreshInterval = setInterval(fetchStatus, 30000);
// Stop auto-refresh when page loses focus (optional)
document.addEventListener('visibilitychange', function() {
if (document.hidden) {
clearInterval(refreshInterval);
} else {
refreshInterval = setInterval(fetchStatus, 30000);
fetchStatus(); // Refresh immediately when returning to tab
}
});
// Load status on page load
fetchStatus()
</script>
</body>
</html>

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,75 @@
# Application Configuration
# Values will be injected from pom.xml during build
quarkus.application.name=${project.artifactId}
quarkus.application.version=${project.version}
# Custom properties for groupId if needed
application.groupId=${project.groupId}
application.artifactId=${project.artifactId}
application.version=${project.version}
# HTTP Configuration
quarkus.http.port=8081
# ========== DEVELOPMENT (quarkus:dev) ==========
%dev.quarkus.http.host=127.0.0.1
# ========== PRODUCTION (Docker/JAR) ==========
%prod.quarkus.http.host=0.0.0.0
# ========== TEST PROFILE ==========
%test.quarkus.http.host=localhost
# Enable CORS for frontend development
quarkus.http.cors=true
quarkus.http.cors.origins=*
quarkus.http.cors.methods=GET,POST,PUT,DELETE,OPTIONS
quarkus.http.cors.headers=accept,authorization,content-type,x-requested-with
# Logging Configuration
quarkus.log.console.format=%d{HH:mm:ss} %-5p [%c{2.}] (%t) %s%e%n
quarkus.log.console.level=INFO
# Development mode settings
%dev.quarkus.log.console.level=DEBUG
%dev.quarkus.live-reload.instrumentation=true
# JVM Arguments for native access (Jansi, OpenCV, etc.)
quarkus.native.additional-build-args=--enable-native-access=ALL-UNNAMED
# Production optimizations
%prod.quarkus.package.type=fast-jar
%prod.quarkus.http.enable-compression=true
# Static resources
quarkus.http.enable-compression=true
quarkus.rest.path=/
quarkus.http.root-path=/
# Auction Monitor Configuration
auction.database.path=/mnt/okcomputer/output/cache.db
auction.images.path=/mnt/okcomputer/output/images
# auction.notification.config=desktop
# Format: smtp:username:password:recipient_email
auction.notification.config=smtp:michael.bakker1986@gmail.com:agrepolhlnvhipkv:michael.bakker1986@gmail.com
auction.yolo.config=/mnt/okcomputer/output/models/yolov4.cfg
auction.yolo.weights=/mnt/okcomputer/output/models/yolov4.weights
auction.yolo.classes=/mnt/okcomputer/output/models/coco.names
# Scheduler Configuration
quarkus.scheduler.enabled=true
quarkus.scheduler.start-halted=false
# Workflow Schedules
auction.workflow.scraper-import.cron=0 */30 * * * ?
auction.workflow.image-processing.cron=0 0 * * * ?
auction.workflow.bid-monitoring.cron=0 */15 * * * ?
auction.workflow.closing-alerts.cron=0 */5 * * * ?
# HTTP Rate Limiting Configuration
# Prevents overloading external services and getting blocked
auction.http.rate-limit.default-max-rps=2
auction.http.rate-limit.troostwijk-max-rps=1
auction.http.timeout-seconds=30
# Health Check Configuration
quarkus.smallrye-health.root-path=/health

View File

@@ -0,0 +1,20 @@
# SLF4J Simple Logger Configuration
# Set default log level (trace, debug, info, warn, error, off)
org.slf4j.simpleLogger.defaultLogLevel=warn
# Show date/time in logs
org.slf4j.simpleLogger.showDateTime=true
org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss
# Show thread name
org.slf4j.simpleLogger.showThreadName=false
# Show log name (logger name)
org.slf4j.simpleLogger.showLogName=false
# Show short log name
org.slf4j.simpleLogger.showShortLogName=true
# Set specific logger levels
org.slf4j.simpleLogger.log.com.microsoft.playwright=warn
org.slf4j.simpleLogger.log.org.sqlite=warn

View File

@@ -0,0 +1,138 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.time.LocalDateTime;
import static org.junit.jupiter.api.Assertions.*;
/**
* Tests for closing time calculations that power the UI
* Tests the minutesUntilClose() logic used in dashboard and alerts
*/
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@DisplayName("Closing Time Calculation Tests")
class ClosingTimeCalculationTest {
@Test
@Order(1)
@DisplayName("Should calculate minutes until close for lot closing in 15 minutes")
void testMinutesUntilClose15Minutes() {
var lot = createLot(LocalDateTime.now().plusMinutes(15));
long minutes = lot.minutesUntilClose();
assertTrue(minutes >= 14 && minutes <= 16,
"Should be approximately 15 minutes, was: " + minutes);
}
@Test
@Order(2)
@DisplayName("Should calculate minutes until close for lot closing in 2 hours")
void testMinutesUntilClose2Hours() {
var lot = createLot(LocalDateTime.now().plusHours(2));
long minutes = lot.minutesUntilClose();
assertTrue(minutes >= 119 && minutes <= 121,
"Should be approximately 120 minutes, was: " + minutes);
}
@Test
@Order(3)
@DisplayName("Should return negative value for already closed lot")
void testMinutesUntilCloseNegative() {
var lot = createLot(LocalDateTime.now().minusHours(1));
long minutes = lot.minutesUntilClose();
assertTrue(minutes < 0,
"Should be negative for closed lots, was: " + minutes);
}
@Test
@Order(4)
@DisplayName("Should return MAX_VALUE when lot has no closing time")
void testMinutesUntilCloseNoTime() {
var lot = Lot.basic(100, 1001, "No closing time", "", "", "", 0, "General",
100.0, "EUR", "http://test.com/1001", null, false);
long minutes = lot.minutesUntilClose();
assertEquals(Long.MAX_VALUE, minutes,
"Should return MAX_VALUE when no closing time set");
}
@Test
@Order(5)
@DisplayName("Should identify lots closing within 5 minutes (critical threshold)")
void testCriticalClosingThreshold() {
var closing4Min = createLot(LocalDateTime.now().plusMinutes(4));
var closing5Min = createLot(LocalDateTime.now().plusMinutes(5));
var closing6Min = createLot(LocalDateTime.now().plusMinutes(6));
assertTrue(closing4Min.minutesUntilClose() < 5,
"Lot closing in 4 min should be < 5 minutes");
assertTrue(closing5Min.minutesUntilClose() >= 5,
"Lot closing in 5 min should be >= 5 minutes");
assertTrue(closing6Min.minutesUntilClose() > 5,
"Lot closing in 6 min should be > 5 minutes");
}
@Test
@Order(6)
@DisplayName("Should identify lots closing within 30 minutes (dashboard threshold)")
void testDashboardClosingThreshold() {
var closing20Min = createLot(LocalDateTime.now().plusMinutes(20));
var closing31Min = createLot(LocalDateTime.now().plusMinutes(31)); // Use 31 to avoid boundary timing issue
var closing40Min = createLot(LocalDateTime.now().plusMinutes(40));
assertTrue(closing20Min.minutesUntilClose() < 30,
"Lot closing in 20 min should be < 30 minutes");
assertTrue(closing31Min.minutesUntilClose() >= 30,
"Lot closing in 31 min should be >= 30 minutes");
assertTrue(closing40Min.minutesUntilClose() > 30,
"Lot closing in 40 min should be > 30 minutes");
}
@Test
@Order(7)
@DisplayName("Should calculate correctly for lots closing soon (boundary cases)")
void testBoundaryCases() {
// Just closed (< 1 minute ago)
var justClosed = createLot(LocalDateTime.now().minusSeconds(30));
assertTrue(justClosed.minutesUntilClose() <= 0, "Just closed should be <= 0");
// Closing very soon (< 1 minute)
var closingVerySoon = createLot(LocalDateTime.now().plusSeconds(30));
assertTrue(closingVerySoon.minutesUntilClose() < 1, "Closing in 30 sec should be < 1 minute");
// Closing in exactly 1 hour
var closing1Hour = createLot(LocalDateTime.now().plusHours(1));
long minutes1Hour = closing1Hour.minutesUntilClose();
assertTrue(minutes1Hour >= 59 && minutes1Hour <= 61,
"Closing in 1 hour should be ~60 minutes, was: " + minutes1Hour);
}
@Test
@Order(8)
@DisplayName("Multiple lots should sort correctly by urgency")
void testSortingByUrgency() {
var lot5Min = createLot(LocalDateTime.now().plusMinutes(5));
var lot30Min = createLot(LocalDateTime.now().plusMinutes(30));
var lot1Hour = createLot(LocalDateTime.now().plusHours(1));
var lot3Hours = createLot(LocalDateTime.now().plusHours(3));
var lots = java.util.List.of(lot3Hours, lot30Min, lot5Min, lot1Hour);
var sorted = lots.stream()
.sorted((a, b) -> Long.compare(a.minutesUntilClose(), b.minutesUntilClose()))
.toList();
assertEquals(lot5Min, sorted.get(0), "Most urgent should be first");
assertEquals(lot30Min, sorted.get(1), "Second most urgent");
assertEquals(lot1Hour, sorted.get(2), "Third most urgent");
assertEquals(lot3Hours, sorted.get(3), "Least urgent should be last");
}
// Helper method
private Lot createLot(LocalDateTime closingTime) {
return Lot.basic(100, 1001, "Test Item", "", "", "", 0, "General",
100.0, "EUR", "http://test.com/1001", closingTime, false);
}
}

View File

@@ -0,0 +1,390 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* Test cases for DatabaseService.
* Tests database operations including schema creation, CRUD operations, and data retrieval.
*/
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class DatabaseServiceTest {
private DatabaseService db;
private String testDbPath;
@BeforeAll
void setUp() throws SQLException {
// Load SQLite JDBC driver
try {
Class.forName("org.sqlite.JDBC");
} catch (ClassNotFoundException e) {
throw new SQLException("SQLite JDBC driver not found", e);
}
testDbPath = "test_database_" + System.currentTimeMillis() + ".db";
db = new DatabaseService(testDbPath);
db.ensureSchema();
}
@AfterAll
void tearDown() throws Exception {
// Clean up test database
Files.deleteIfExists(Paths.get(testDbPath));
}
@Test
@DisplayName("Should create database schema successfully")
void testEnsureSchema() {
assertDoesNotThrow(() -> db.ensureSchema());
}
@Test
@DisplayName("Should insert and retrieve auction")
void testUpsertAndGetAuction() throws SQLException {
var auction = new AuctionInfo(
12345,
"Test Auction",
"Amsterdam, NL",
"Amsterdam",
"NL",
"https://example.com/auction/12345",
"A7",
50,
LocalDateTime.of(2025, 12, 15, 14, 30)
);
db.upsertAuction(auction);
var auctions = db.getAllAuctions();
assertFalse(auctions.isEmpty());
var retrieved = auctions.stream()
.filter(a -> a.auctionId() == 12345)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertEquals("Test Auction", retrieved.title());
assertEquals("Amsterdam", retrieved.city());
assertEquals("NL", retrieved.country());
assertEquals(50, retrieved.lotCount());
}
@Test
@DisplayName("Should update existing auction on conflict")
void testUpsertAuctionUpdate() throws SQLException {
var auction1 = new AuctionInfo(
99999,
"Original Title",
"Rotterdam, NL",
"Rotterdam",
"NL",
"https://example.com/auction/99999",
"A1",
10,
null
);
db.upsertAuction(auction1);
// Update with same ID
var auction2 = new AuctionInfo(
99999,
"Updated Title",
"Rotterdam, NL",
"Rotterdam",
"NL",
"https://example.com/auction/99999",
"A1",
20,
null
);
db.upsertAuction(auction2);
var auctions = db.getAllAuctions();
var retrieved = auctions.stream()
.filter(a -> a.auctionId() == 99999)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertEquals("Updated Title", retrieved.title());
assertEquals(20, retrieved.lotCount());
}
@Test
@DisplayName("Should retrieve auctions by country code")
void testGetAuctionsByCountry() throws SQLException {
// Insert auctions from different countries
db.upsertAuction(new AuctionInfo(
10001, "Dutch Auction", "Amsterdam, NL", "Amsterdam", "NL",
"https://example.com/10001", "A1", 10, null
));
db.upsertAuction(new AuctionInfo(
10002, "Romanian Auction", "Cluj, RO", "Cluj", "RO",
"https://example.com/10002", "A2", 15, null
));
db.upsertAuction(new AuctionInfo(
10003, "Another Dutch", "Utrecht, NL", "Utrecht", "NL",
"https://example.com/10003", "A3", 20, null
));
var nlAuctions = db.getAuctionsByCountry("NL");
assertEquals(2, nlAuctions.stream().filter(a -> a.auctionId() >= 10001 && a.auctionId() <= 10003).count());
var roAuctions = db.getAuctionsByCountry("RO");
assertEquals(1, roAuctions.stream().filter(a -> a.auctionId() == 10002).count());
}
@Test
@DisplayName("Should insert and retrieve lot")
void testUpsertAndGetLot() throws SQLException {
var lot = Lot.basic(
12345, // saleId
67890, // lotId
"Forklift",
"Electric forklift in good condition",
"Toyota",
"Electric",
2018,
"Machinery",
1500.00,
"EUR",
"https://example.com/lot/67890",
LocalDateTime.of(2025, 12, 20, 16, 0),
false
);
db.upsertLot(lot);
var lots = db.getAllLots();
assertFalse(lots.isEmpty());
var retrieved = lots.stream()
.filter(l -> l.lotId() == 67890)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertEquals("Forklift", retrieved.title());
assertEquals("Toyota", retrieved.manufacturer());
assertEquals(2018, retrieved.year());
assertEquals(1500.00, retrieved.currentBid(), 0.01);
assertFalse(retrieved.closingNotified());
}
@Test
@DisplayName("Should update lot current bid")
void testUpdateLotCurrentBid() throws SQLException {
var lot = Lot.basic(
11111, 22222, "Test Item", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/22222", null, false
);
db.upsertLot(lot);
// Update bid
var updatedLot = Lot.basic(
11111, 22222, "Test Item", "Description", "", "", 0, "Category",
250.00, "EUR", "https://example.com/lot/22222", null, false
);
db.updateLotCurrentBid(updatedLot);
var lots = db.getAllLots();
var retrieved = lots.stream()
.filter(l -> l.lotId() == 22222)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertEquals(250.00, retrieved.currentBid(), 0.01);
}
@Test
@DisplayName("Should update lot notification flags")
void testUpdateLotNotificationFlags() throws SQLException {
var lot = Lot.basic(
33333, 44444, "Test Item", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/44444", null, false
);
db.upsertLot(lot);
// Update notification flag
var updatedLot = Lot.basic(
33333, 44444, "Test Item", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/44444", null, true
);
db.updateLotNotificationFlags(updatedLot);
var lots = db.getAllLots();
var retrieved = lots.stream()
.filter(l -> l.lotId() == 44444)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertTrue(retrieved.closingNotified());
}
@Test
@DisplayName("Should insert and retrieve image records")
void testInsertAndGetImages() throws SQLException {
// First create a lot
var lot = Lot.basic(
55555, 66666, "Test Lot", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/66666", null, false
);
db.upsertLot(lot);
// Insert images
db.insertImage(66666, "https://example.com/img1.jpg",
"C:/images/66666/img1.jpg", List.of("car", "vehicle"));
db.insertImage(66666, "https://example.com/img2.jpg",
"C:/images/66666/img2.jpg", List.of("truck"));
var images = db.getImagesForLot(66666);
assertEquals(2, images.size());
var img1 = images.stream()
.filter(i -> i.url().contains("img1.jpg"))
.findFirst()
.orElse(null);
assertNotNull(img1);
assertEquals("car,vehicle", img1.labels());
}
@Test
@DisplayName("Should count total images")
void testGetImageCount() throws SQLException {
int initialCount = db.getImageCount();
// Add a lot and image
var lot = Lot.basic(
77777, 88888, "Test Lot", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/88888", null, false
);
db.upsertLot(lot);
db.insertImage(88888, "https://example.com/test.jpg",
"C:/images/88888/test.jpg", List.of("object"));
int newCount = db.getImageCount();
assertTrue(newCount > initialCount);
}
@Test
@DisplayName("Should handle empty database gracefully")
void testEmptyDatabase() throws SQLException, IOException {
DatabaseService emptyDb = new DatabaseService("empty_test_" + System.currentTimeMillis() + ".db");
emptyDb.ensureSchema();
var auctions = emptyDb.getAllAuctions();
var lots = emptyDb.getAllLots();
int imageCount = emptyDb.getImageCount();
assertNotNull(auctions);
assertNotNull(lots);
assertTrue(auctions.isEmpty());
assertTrue(lots.isEmpty());
assertEquals(0, imageCount);
// Clean up
Files.deleteIfExists(Paths.get("empty_test_" + System.currentTimeMillis() + ".db"));
}
@Test
@DisplayName("Should handle lots with null closing time")
void testLotWithNullClosingTime() throws SQLException {
var lot = Lot.basic(
98765, 12340, "Test Item", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/12340", null, false
);
assertDoesNotThrow(() -> db.upsertLot(lot));
var retrieved = db.getAllLots().stream()
.filter(l -> l.lotId() == 12340)
.findFirst()
.orElse(null);
assertNotNull(retrieved);
assertNull(retrieved.closingTime());
}
@Test
@DisplayName("Should retrieve active lots only")
void testGetActiveLots() throws SQLException {
var activeLot = Lot.basic(
11111, 55551, "Active Lot", "Description", "", "", 0, "Category",
100.00, "EUR", "https://example.com/lot/55551",
LocalDateTime.now().plusDays(1), false
);
db.upsertLot(activeLot);
var activeLots = db.getActiveLots();
assertFalse(activeLots.isEmpty());
var found = activeLots.stream()
.anyMatch(l -> l.lotId() == 55551);
assertTrue(found);
}
@Test
@DisplayName("Should handle concurrent upserts")
void testConcurrentUpserts() throws InterruptedException, SQLException {
Thread t1 = new Thread(() -> {
try {
for (int i = 0; i < 10; i++) {
db.upsertLot(Lot.basic(
99000 + i, 99100 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
100.0, "EUR", "https://example.com/" + i, null, false
));
}
} catch (Exception e) {
fail("Thread 1 failed: " + e.getMessage());
}
});
Thread t2 = new Thread(() -> {
try {
for (int i = 10; i < 20; i++) {
db.upsertLot(Lot.basic(
99000 + i, 99100 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
200.0, "EUR", "https://example.com/" + i, null, false
));
}
} catch (Exception e) {
fail("Thread 2 failed: " + e.getMessage());
}
});
t1.start();
t2.start();
t1.join();
t2.join();
var lots = db.getAllLots();
long concurrentLots = lots.stream()
.filter(l -> l.lotId() >= 99100 && l.lotId() < 99120)
.count();
assertTrue(concurrentLots >= 20);
}
}

View File

@@ -0,0 +1,186 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.sql.SQLException;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
/**
* Test cases for ImageProcessingService.
* Tests object detection integration and database label updates.
*
* NOTE: Image downloading is now handled by the scraper, so these tests
* focus only on object detection and label storage.
*/
class ImageProcessingServiceTest {
private DatabaseService mockDb;
private ObjectDetectionService mockDetector;
private ImageProcessingService service;
private java.io.File testImage;
@BeforeEach
void setUp() throws Exception {
mockDb = mock(DatabaseService.class);
mockDetector = mock(ObjectDetectionService.class);
service = new ImageProcessingService(mockDb, mockDetector);
// Create a temporary test image file
testImage = java.io.File.createTempFile("test_image_", ".jpg");
testImage.deleteOnExit();
// Write minimal JPEG header to make it a valid file
try (var out = new java.io.FileOutputStream(testImage)) {
out.write(new byte[]{(byte)0xFF, (byte)0xD8, (byte)0xFF, (byte)0xE0});
}
}
@Test
@DisplayName("Should process single image and update labels")
void testProcessImage() throws SQLException {
// Normalize path (convert backslashes to forward slashes)
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
// Mock object detection with normalized path
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("car", "vehicle"));
// Process image
boolean result = service.processImage(1, testImage.getAbsolutePath(), 12345);
// Verify success
assertTrue(result);
verify(mockDetector).detectObjects(normalizedPath);
verify(mockDb).updateImageLabels(1, List.of("car", "vehicle"));
}
@Test
@DisplayName("Should handle empty detection results")
void testProcessImageWithNoDetections() throws SQLException {
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of());
boolean result = service.processImage(2, testImage.getAbsolutePath(), 12346);
assertTrue(result);
verify(mockDb).updateImageLabels(2, List.of());
}
@Test
@DisplayName("Should handle database error gracefully")
void testProcessImageDatabaseError() {
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("object"));
doThrow(new RuntimeException("Database error"))
.when(mockDb).updateImageLabels(anyInt(), anyList());
// Should return false on error
boolean result = service.processImage(3, testImage.getAbsolutePath(), 12347);
assertFalse(result);
}
@Test
@DisplayName("Should handle object detection error gracefully")
void testProcessImageDetectionError() {
when(mockDetector.detectObjects(anyString()))
.thenThrow(new RuntimeException("Detection failed"));
// Should return false on error
boolean result = service.processImage(4, "/path/to/image4.jpg", 12348);
assertFalse(result);
}
@Test
@DisplayName("Should process pending images batch")
void testProcessPendingImages() throws SQLException {
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
// Mock pending images from database - use real test image path
when(mockDb.getImagesNeedingDetection()).thenReturn(List.of(
new DatabaseService.ImageDetectionRecord(1, 100L, testImage.getAbsolutePath()),
new DatabaseService.ImageDetectionRecord(2, 101L, testImage.getAbsolutePath())
));
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("item1"))
.thenReturn(List.of("item2"));
when(mockDb.getImageLabels(anyInt()))
.thenReturn(List.of("item1"))
.thenReturn(List.of("item2"));
// Process batch
service.processPendingImages();
// Verify all images were processed
verify(mockDb).getImagesNeedingDetection();
verify(mockDetector, times(2)).detectObjects(normalizedPath);
verify(mockDb, times(2)).updateImageLabels(anyInt(), anyList());
}
@Test
@DisplayName("Should handle empty pending images list")
void testProcessPendingImagesEmpty() throws SQLException {
when(mockDb.getImagesNeedingDetection()).thenReturn(List.of());
service.processPendingImages();
verify(mockDb).getImagesNeedingDetection();
verify(mockDetector, never()).detectObjects(anyString());
}
@Test
@DisplayName("Should continue processing after single image failure")
void testProcessPendingImagesWithFailure() throws SQLException {
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDb.getImagesNeedingDetection()).thenReturn(List.of(
new DatabaseService.ImageDetectionRecord(1, 100L, testImage.getAbsolutePath()),
new DatabaseService.ImageDetectionRecord(2, 101L, testImage.getAbsolutePath())
));
// First image fails, second succeeds
when(mockDetector.detectObjects(normalizedPath))
.thenThrow(new RuntimeException("Detection error"))
.thenReturn(List.of("item"));
when(mockDb.getImageLabels(2))
.thenReturn(List.of("item"));
service.processPendingImages();
// Verify second image was still processed
verify(mockDetector, times(2)).detectObjects(normalizedPath);
}
@Test
@DisplayName("Should handle database query error in batch processing")
void testProcessPendingImagesDatabaseError() {
when(mockDb.getImagesNeedingDetection())
.thenThrow(new RuntimeException("Database connection failed"));
// Should not throw exception
assertDoesNotThrow(() -> service.processPendingImages());
}
@Test
@DisplayName("Should process images with multiple detected objects")
void testProcessImageMultipleDetections() throws SQLException {
String normalizedPath = testImage.getAbsolutePath().replace('\\', '/');
when(mockDetector.detectObjects(normalizedPath))
.thenReturn(List.of("car", "truck", "vehicle", "road"));
boolean result = service.processImage(5, testImage.getAbsolutePath(), 12349);
assertTrue(result);
verify(mockDb).updateImageLabels(5, List.of("car", "truck", "vehicle", "road"));
}
}

View File

@@ -0,0 +1,461 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.time.LocalDateTime;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* Integration test for complete workflow.
* Tests end-to-end scenarios including:
* 1. Scraper data import
* 2. Data transformation
* 3. Image processing
* 4. Object detection
* 5. Bid monitoring
* 6. Notifications
*/
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
class IntegrationTest {
private String testDbPath;
private DatabaseService db;
private NotificationService notifier;
private ObjectDetectionService detector;
private ImageProcessingService imageProcessor;
private TroostwijkMonitor monitor;
@BeforeAll
void setUp() throws SQLException, IOException {
testDbPath = "test_integration_" + System.currentTimeMillis() + ".db";
// Initialize all services
db = new DatabaseService(testDbPath);
db.ensureSchema();
notifier = new NotificationService("desktop");
detector = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
imageProcessor = new ImageProcessingService(db, detector);
monitor = new TroostwijkMonitor(
testDbPath,
"desktop",
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
}
@AfterAll
void tearDown() throws Exception {
Files.deleteIfExists(Paths.get(testDbPath));
}
@Test
@Order(1)
@DisplayName("Integration: Complete scraper data import workflow")
void testCompleteScraperImportWorkflow() throws SQLException {
// Step 1: Import auction from scraper format
var auction = new AuctionInfo(
12345,
"Industrial Equipment Auction",
"Rotterdam, NL",
"Rotterdam",
"NL",
"https://example.com/auction/12345",
"A7",
25,
LocalDateTime.now().plusDays(3)
);
db.upsertAuction(auction);
// Step 2: Import lots for this auction
var lot1 = Lot.basic(
12345, 10001,
"Toyota Forklift 2.5T",
"Electric forklift in excellent condition",
"Toyota",
"Electric",
2018,
"Machinery",
1500.00,
"EUR",
"https://example.com/lot/10001",
LocalDateTime.now().plusDays(3),
false
);
var lot2 = Lot.basic(
12345, 10002,
"Office Furniture Set",
"Desks, chairs, and cabinets",
"",
"",
0,
"Furniture",
500.00,
"EUR",
"https://example.com/lot/10002",
LocalDateTime.now().plusDays(3),
false
);
db.upsertLot(lot1);
db.upsertLot(lot2);
// Verify import
var auctions = db.getAllAuctions();
var lots = db.getAllLots();
assertTrue(auctions.stream().anyMatch(a -> a.auctionId() == 12345));
assertEquals(2, lots.stream().filter(l -> l.saleId() == 12345).count());
}
@Test
@Order(2)
@DisplayName("Integration: Image processing and detection workflow")
void testImageProcessingWorkflow() throws SQLException {
// Add images for a lot
db.insertImage(10001, "https://example.com/img1.jpg",
"C:/images/10001/img1.jpg", List.of("truck", "vehicle"));
db.insertImage(10001, "https://example.com/img2.jpg",
"C:/images/10001/img2.jpg", List.of("forklift", "machinery"));
// Verify images were saved
var images = db.getImagesForLot(10001);
assertEquals(2, images.size());
var labels = images.stream()
.flatMap(img -> List.of(img.labels().split(",")).stream())
.distinct()
.toList();
assertTrue(labels.contains("truck") || labels.contains("forklift"));
}
@Test
@Order(3)
@DisplayName("Integration: Bid monitoring and notification workflow")
void testBidMonitoringWorkflow() throws SQLException {
// Simulate bid change
var lot = db.getAllLots().stream()
.filter(l -> l.lotId() == 10001)
.findFirst()
.orElseThrow();
// Update bid
var updatedLot = Lot.basic(
lot.saleId(), lot.lotId(), lot.title(), lot.description(),
lot.manufacturer(), lot.type(), lot.year(), lot.category(),
2000.00, // Increased from 1500.00
lot.currency(), lot.url(), lot.closingTime(), lot.closingNotified()
);
db.updateLotCurrentBid(updatedLot);
// Send notification
var message = String.format("Nieuw bod op kavel %d: €%.2f (was €%.2f)",
lot.lotId(), 2000.00, 1500.00);
assertDoesNotThrow(() ->
notifier.sendNotification(message, "Kavel bieding update", 0)
);
// Verify bid was updated
var refreshed = db.getAllLots().stream()
.filter(l -> l.lotId() == 10001)
.findFirst()
.orElseThrow();
assertEquals(2000.00, refreshed.currentBid(), 0.01);
}
@Test
@Order(4)
@DisplayName("Integration: Closing alert workflow")
void testClosingAlertWorkflow() throws SQLException {
// Create lot closing soon
var closingSoon = Lot.basic(
12345, 20001,
"Closing Soon Item",
"Description",
"",
"",
0,
"Category",
750.00,
"EUR",
"https://example.com/lot/20001",
LocalDateTime.now().plusMinutes(4),
false
);
db.upsertLot(closingSoon);
// Check if lot is closing soon
assertTrue(closingSoon.minutesUntilClose() < 5);
// Send high-priority notification
var message = "Kavel " + closingSoon.lotId() + " sluit binnen 5 min.";
assertDoesNotThrow(() ->
notifier.sendNotification(message, "Lot nearing closure", 1)
);
// Mark as notified
var notified = Lot.basic(
closingSoon.saleId(), closingSoon.lotId(), closingSoon.title(),
closingSoon.description(), closingSoon.manufacturer(), closingSoon.type(),
closingSoon.year(), closingSoon.category(), closingSoon.currentBid(),
closingSoon.currency(), closingSoon.url(), closingSoon.closingTime(),
true
);
db.updateLotNotificationFlags(notified);
// Verify notification flag
var updated = db.getAllLots().stream()
.filter(l -> l.lotId() == 20001)
.findFirst()
.orElseThrow();
assertTrue(updated.closingNotified());
}
@Test
@Order(5)
@DisplayName("Integration: Multi-country auction filtering")
void testMultiCountryFiltering() throws SQLException {
// Add auctions from different countries
db.upsertAuction(new AuctionInfo(
30001, "Dutch Auction", "Amsterdam, NL", "Amsterdam", "NL",
"https://example.com/30001", "A1", 10, null
));
db.upsertAuction(new AuctionInfo(
30002, "Romanian Auction", "Cluj, RO", "Cluj", "RO",
"https://example.com/30002", "A2", 15, null
));
db.upsertAuction(new AuctionInfo(
30003, "Belgian Auction", "Brussels, BE", "Brussels", "BE",
"https://example.com/30003", "A3", 20, null
));
// Filter by country
var nlAuctions = db.getAuctionsByCountry("NL");
var roAuctions = db.getAuctionsByCountry("RO");
var beAuctions = db.getAuctionsByCountry("BE");
assertTrue(nlAuctions.stream().anyMatch(a -> a.auctionId() == 30001));
assertTrue(roAuctions.stream().anyMatch(a -> a.auctionId() == 30002));
assertTrue(beAuctions.stream().anyMatch(a -> a.auctionId() == 30003));
}
@Test
@Order(6)
@DisplayName("Integration: Complete monitoring cycle")
void testCompleteMonitoringCycle() throws SQLException {
// Monitor should handle all lots
monitor.printDatabaseStats();
var activeLots = db.getActiveLots();
assertFalse(activeLots.isEmpty());
// Process pending images
assertDoesNotThrow(() -> monitor.processPendingImages());
// Verify database integrity
var imageCount = db.getImageCount();
assertTrue(imageCount >= 0);
}
@Test
@Order(7)
@DisplayName("Integration: Data consistency across services")
void testDataConsistency() throws SQLException {
// Verify all auctions have valid data
var auctions = db.getAllAuctions();
for (var auction : auctions) {
assertNotNull(auction.auctionId());
assertNotNull(auction.title());
assertNotNull(auction.url());
}
// Verify all lots have valid data
var lots = db.getAllLots();
for (var lot : lots) {
assertNotNull(lot.lotId());
assertNotNull(lot.title());
assertTrue(lot.currentBid() >= 0);
}
}
@Test
@Order(8)
@DisplayName("Integration: Object detection value estimation workflow")
void testValueEstimationWorkflow() throws SQLException {
// Create lot with detected objects
var lot = Lot.basic(
40000, 50000,
"Construction Equipment",
"Heavy machinery for construction",
"Caterpillar",
"Excavator",
2015,
"Machinery",
25000.00,
"EUR",
"https://example.com/lot/50000",
LocalDateTime.now().plusDays(5),
false
);
db.upsertLot(lot);
// Add images with detected objects
db.insertImage(50000, "https://example.com/excavator1.jpg",
"C:/images/50000/1.jpg", List.of("truck", "excavator", "machinery"));
db.insertImage(50000, "https://example.com/excavator2.jpg",
"C:/images/50000/2.jpg", List.of("excavator", "construction"));
// Retrieve and analyze
var images = db.getImagesForLot(50000);
assertFalse(images.isEmpty());
// Count unique objects
var allLabels = images.stream()
.flatMap(img -> List.of(img.labels().split(",")).stream())
.distinct()
.toList();
assertTrue(allLabels.contains("excavator") || allLabels.contains("machinery"));
// Simulate value estimation notification
var message = String.format(
"Lot contains: %s\nEstimated value: €%,.2f",
String.join(", ", allLabels),
lot.currentBid()
);
assertDoesNotThrow(() ->
notifier.sendNotification(message, "Object Detected", 0)
);
}
@Test
@Order(9)
@DisplayName("Integration: Handle rapid concurrent updates")
void testConcurrentOperations() throws InterruptedException, SQLException {
var auctionThread = new Thread(() -> {
try {
for (var i = 0; i < 10; i++) {
db.upsertAuction(new AuctionInfo(
60000 + i, "Concurrent Auction " + i, "Test, NL", "Test", "NL",
"https://example.com/60" + i, "A1", 5, null
));
}
} catch (Exception e) {
fail("Auction thread failed: " + e.getMessage());
}
});
var lotThread = new Thread(() -> {
try {
for (var i = 0; i < 10; i++) {
db.upsertLot(Lot.basic(
60000 + i, 70000 + i, "Concurrent Lot " + i, "Desc", "", "", 0, "Cat",
100.0 * i, "EUR", "https://example.com/70" + i, null, false
));
}
} catch (Exception e) {
fail("Lot thread failed: " + e.getMessage());
}
});
auctionThread.start();
lotThread.start();
auctionThread.join();
lotThread.join();
// Verify all were inserted
var auctions = db.getAllAuctions();
var lots = db.getAllLots();
var auctionCount = auctions.stream()
.filter(a -> a.auctionId() >= 60000 && a.auctionId() < 60010)
.count();
var lotCount = lots.stream()
.filter(l -> l.lotId() >= 70000 && l.lotId() < 70010)
.count();
assertEquals(10, auctionCount);
assertEquals(10, lotCount);
}
@Test
@Order(10)
@DisplayName("Integration: End-to-end notification scenarios")
void testAllNotificationScenarios() {
// 1. Bid change notification
assertDoesNotThrow(() ->
notifier.sendNotification(
"Nieuw bod op kavel 12345: €150.00 (was €125.00)",
"Kavel bieding update",
0
)
);
// 2. Closing alert
assertDoesNotThrow(() ->
notifier.sendNotification(
"Kavel 67890 sluit binnen 5 min.",
"Lot nearing closure",
1
)
);
// 3. Object detection
assertDoesNotThrow(() ->
notifier.sendNotification(
"Detected: car, truck, machinery",
"Object Detected",
0
)
);
// 4. Value estimate
assertDoesNotThrow(() ->
notifier.sendNotification(
"Geschatte waarde: €5,000 - €7,500",
"Value Estimate",
0
)
);
// 5. Viewing day reminder
assertDoesNotThrow(() ->
notifier.sendNotification(
"Bezichtiging op 15-12-2025 om 14:00",
"Viewing Day Reminder",
0
)
);
}
}

View File

@@ -0,0 +1,243 @@
package auctiora;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
/**
* Test cases for NotificationService.
* Tests desktop and email notification configuration and delivery.
*/
class NotificationServiceTest {
@Test
@DisplayName("Should initialize with desktop-only configuration")
void testDesktopOnlyConfiguration() {
NotificationService service = new NotificationService("desktop");
assertNotNull(service);
}
@Test
@DisplayName("Should initialize with SMTP configuration")
void testSMTPConfiguration() {
NotificationService service = new NotificationService(
"smtp:test@gmail.com:app_password:recipient@example.com"
);
assertNotNull(service);
}
@Test
@DisplayName("Should reject invalid SMTP configuration format")
void testInvalidSMTPConfiguration() {
// Missing parts (only 2 parts total)
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:incomplete")
);
// Wrong format (only 3 parts total, needs 4)
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:only:two")
);
}
@Test
@DisplayName("Should reject unknown configuration type")
void testUnknownConfiguration() {
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("unknown_type")
);
}
@Test
@DisplayName("Should send desktop notification without error")
void testDesktopNotification() {
NotificationService service = new NotificationService("desktop");
// Should not throw exception even if system tray not available
assertDoesNotThrow(() ->
service.sendNotification("Test message", "Test title", 0)
);
}
@Test
@DisplayName("Should send high priority notification")
void testHighPriorityNotification() {
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("Urgent message", "High Priority", 1)
);
}
@Test
@DisplayName("Should send normal priority notification")
void testNormalPriorityNotification() {
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("Regular message", "Normal Priority", 0)
);
}
@Test
@DisplayName("Should handle notification when system tray not supported")
void testNoSystemTraySupport() {
NotificationService service = new NotificationService("desktop");
// Should gracefully handle missing system tray
assertDoesNotThrow(() ->
service.sendNotification("Test", "Test", 0)
);
}
@Test
@DisplayName("Should send email notification with valid SMTP config")
void testEmailNotificationWithValidConfig() {
// Note: This won't actually send email without valid credentials
// But it should initialize properly
NotificationService service = new NotificationService(
"smtp:test@gmail.com:fake_password:test@example.com"
);
// Should not throw during initialization
assertNotNull(service);
// Sending will fail with fake credentials, but shouldn't crash
assertDoesNotThrow(() ->
service.sendNotification("Test email", "Email Test", 0)
);
}
@Test
@DisplayName("Should include both desktop and email when SMTP configured")
void testBothNotificationChannels() {
NotificationService service = new NotificationService(
"smtp:user@gmail.com:password:recipient@example.com"
);
// Both desktop and email should be attempted
assertDoesNotThrow(() ->
service.sendNotification("Dual channel test", "Test", 0)
);
}
@Test
@DisplayName("Should handle empty message gracefully")
void testEmptyMessage() {
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification("", "", 0)
);
}
@Test
@DisplayName("Should handle very long message")
void testLongMessage() {
NotificationService service = new NotificationService("desktop");
String longMessage = "A".repeat(1000);
assertDoesNotThrow(() ->
service.sendNotification(longMessage, "Long Message Test", 0)
);
}
@Test
@DisplayName("Should handle special characters in message")
void testSpecialCharactersInMessage() {
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() ->
service.sendNotification(
"€123.45 - Kavel sluit binnen 5 min! ⚠️",
"Special Chars Test",
1
)
);
}
@Test
@DisplayName("Should accept case-insensitive desktop config")
void testCaseInsensitiveDesktopConfig() {
assertDoesNotThrow(() -> {
new NotificationService("DESKTOP");
new NotificationService("Desktop");
new NotificationService("desktop");
});
}
@Test
@DisplayName("Should validate SMTP config parts count")
void testSMTPConfigPartsValidation() {
// Too few parts
assertThrows(IllegalArgumentException.class, () ->
new NotificationService("smtp:user:pass")
);
// Too many parts should work (extras ignored in split)
assertDoesNotThrow(() ->
new NotificationService("smtp:user:pass:email:extra")
);
}
@Test
@DisplayName("Should handle multiple rapid notifications")
void testRapidNotifications() {
NotificationService service = new NotificationService("desktop");
assertDoesNotThrow(() -> {
for (int i = 0; i < 5; i++) {
service.sendNotification("Notification " + i, "Rapid Test", 0);
}
});
}
@Test
@DisplayName("Should handle notification with null config parameter")
void testNullConfigParameter() {
// Second parameter can be empty string (kept for compatibility)
assertDoesNotThrow(() ->
new NotificationService("desktop")
);
}
@Test
@DisplayName("Should send bid change notification format")
void testBidChangeNotificationFormat() {
NotificationService service = new NotificationService("desktop");
String message = "Nieuw bod op kavel 12345: €150.00 (was €125.00)";
String title = "Kavel bieding update";
assertDoesNotThrow(() ->
service.sendNotification(message, title, 0)
);
}
@Test
@DisplayName("Should send closing alert notification format")
void testClosingAlertNotificationFormat() {
NotificationService service = new NotificationService("desktop");
String message = "Kavel 12345 sluit binnen 5 min.";
String title = "Lot nearing closure";
assertDoesNotThrow(() ->
service.sendNotification(message, title, 1)
);
}
@Test
@DisplayName("Should send object detection notification format")
void testObjectDetectionNotificationFormat() {
NotificationService service = new NotificationService("desktop");
String message = "Lot contains: car, truck, machinery\nEstimated value: €5000";
String title = "Object Detected";
assertDoesNotThrow(() ->
service.sendNotification(message, title, 0)
);
}
}

View File

@@ -0,0 +1,185 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import static org.junit.jupiter.api.Assertions.*;
/**
* Test cases for ObjectDetectionService.
* Tests YOLO model loading and object detection functionality.
*/
class ObjectDetectionServiceTest {
private static final String TEST_CFG = "test_yolo.cfg";
private static final String TEST_WEIGHTS = "test_yolo.weights";
private static final String TEST_CLASSES = "test_classes.txt";
@Test
@DisplayName("Should initialize with missing YOLO models (disabled mode)")
void testInitializeWithoutModels() throws IOException {
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
assertNotNull(service);
}
@Test
@DisplayName("Should return empty list when detection is disabled")
void testDetectObjectsWhenDisabled() throws IOException {
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
var result = service.detectObjects("any_image.jpg");
assertNotNull(result);
assertTrue(result.isEmpty());
}
@Test
@DisplayName("Should handle invalid image path gracefully")
void testInvalidImagePath() throws IOException {
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
var result = service.detectObjects("completely_invalid_path.jpg");
assertNotNull(result);
assertTrue(result.isEmpty());
}
@Test
@DisplayName("Should handle empty image file")
void testEmptyImageFile() throws IOException {
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
// Create empty test file
var tempFile = Files.createTempFile("test_image", ".jpg");
try {
var result = service.detectObjects(tempFile.toString());
assertNotNull(result);
assertTrue(result.isEmpty());
} finally {
Files.deleteIfExists(tempFile);
}
}
@Test
@DisplayName("Should gracefully handle when model files exist but OpenCV fails to load")
void testInitializeWithValidModels() throws IOException {
var cfgPath = Paths.get(TEST_CFG);
var weightsPath = Paths.get(TEST_WEIGHTS);
var classesPath = Paths.get(TEST_CLASSES);
try {
Files.writeString(cfgPath, "[net]\nwidth=416\nheight=416\n");
Files.write(weightsPath, new byte[]{0, 1, 2, 3});
Files.writeString(classesPath, "person\ncar\ntruck\n");
// When files exist but OpenCV native library isn't loaded,
// service should construct successfully but be disabled (handled in @PostConstruct)
var service = new ObjectDetectionService(TEST_CFG, TEST_WEIGHTS, TEST_CLASSES);
// Service is created, but init() handles failures gracefully
// detectObjects should return empty list when disabled
assertNotNull(service);
} finally {
Files.deleteIfExists(cfgPath);
Files.deleteIfExists(weightsPath);
Files.deleteIfExists(classesPath);
}
}
@Test
@DisplayName("Should handle missing class names file")
void testMissingClassNamesFile() throws IOException {
// When model files don't exist, service initializes in disabled mode (no exception)
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
assertNotNull(service);
// Verify it returns empty results when disabled
assertTrue(service.detectObjects("test.jpg").isEmpty());
}
@Test
@DisplayName("Should detect when model files are missing")
void testDetectMissingModelFiles() throws IOException {
// Should initialize in disabled mode
ObjectDetectionService service = new ObjectDetectionService(
"missing.cfg",
"missing.weights",
"missing.names"
);
// Should return empty results when disabled
var results = service.detectObjects("test.jpg");
assertTrue(results.isEmpty());
}
@Test
@DisplayName("Should return unique labels only")
void testUniqueLabels() throws IOException {
// When disabled, returns empty list (unique by default)
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
var result = service.detectObjects("test.jpg");
assertNotNull(result);
assertEquals(0, result.size());
}
@Test
@DisplayName("Should handle multiple detections in same image")
void testMultipleDetections() throws IOException {
// Test structure for when detection works
// With actual YOLO models, this would return multiple objects
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
var result = service.detectObjects("test_image.jpg");
assertNotNull(result);
// When disabled, returns empty list
assertTrue(result.isEmpty());
}
@Test
@DisplayName("Should respect confidence threshold")
void testConfidenceThreshold() throws IOException {
// The service uses 0.5 confidence threshold
// This test documents that behavior
ObjectDetectionService service = new ObjectDetectionService(
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
// Low confidence detections should be filtered out
// (when detection is working)
var result = service.detectObjects("test.jpg");
assertNotNull(result);
}
}

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,255 @@
package auctiora;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.DisplayName;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDateTime;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
/**
* Test cases for ScraperDataAdapter.
* Tests conversion from external scraper schema to monitor schema.
*/
class ScraperDataAdapterTest {
@Test
@DisplayName("Should extract numeric ID from text format auction ID")
void testExtractNumericIdFromAuctionId() {
assertEquals(39813, ScraperDataAdapter.extractNumericId("A7-39813"));
assertEquals(12345, ScraperDataAdapter.extractNumericId("A1-12345"));
assertEquals(0, ScraperDataAdapter.extractNumericId(null));
assertEquals(0, ScraperDataAdapter.extractNumericId(""));
assertEquals(0, ScraperDataAdapter.extractNumericId("ABC"));
}
@Test
@DisplayName("Should extract numeric ID from text format lot ID")
void testExtractNumericIdFromLotId() {
// "A1-28505-5" → 285055 (concatenates all digits)
assertEquals(285055, ScraperDataAdapter.extractNumericId("A1-28505-5"));
assertEquals(123456, ScraperDataAdapter.extractNumericId("A7-1234-56"));
}
@Test
@DisplayName("Should return 0 for IDs that exceed Long.MAX_VALUE")
void testExtractNumericIdTooLarge() {
// These IDs are too large for a long (> 19 digits or > Long.MAX_VALUE)
assertEquals(0, ScraperDataAdapter.extractNumericId("856462986966260305674"));
assertEquals(0, ScraperDataAdapter.extractNumericId("28492384530402679688"));
assertEquals(0, ScraperDataAdapter.extractNumericId("A7-856462986966260305674"));
}
@Test
@DisplayName("Should convert scraper auction format to AuctionInfo")
void testFromScraperAuction() throws SQLException {
// Mock ResultSet with scraper format data
ResultSet rs = mock(ResultSet.class);
when(rs.getString("auction_id")).thenReturn("A7-39813");
when(rs.getString("title")).thenReturn("Industrial Equipment Auction");
when(rs.getString("location")).thenReturn("Cluj-Napoca, RO");
when(rs.getString("url")).thenReturn("https://example.com/auction/A7-39813");
when(rs.getInt("lots_count")).thenReturn(150);
when(rs.getString("first_lot_closing_time")).thenReturn("2025-12-15T14:30:00");
AuctionInfo result = ScraperDataAdapter.fromScraperAuction(rs);
assertNotNull(result);
assertEquals(39813, result.auctionId());
assertEquals("Industrial Equipment Auction", result.title());
assertEquals("Cluj-Napoca, RO", result.location());
assertEquals("Cluj-Napoca", result.city());
assertEquals("RO", result.country());
assertEquals("https://example.com/auction/A7-39813", result.url());
assertEquals("A7", result.typePrefix());
assertEquals(150, result.lotCount());
assertNotNull(result.firstLotClosingTime());
}
@Test
@DisplayName("Should handle auction with simple location without country")
void testFromScraperAuctionSimpleLocation() throws SQLException {
ResultSet rs = mock(ResultSet.class);
when(rs.getString("auction_id")).thenReturn("A1-12345");
when(rs.getString("title")).thenReturn("Test Auction");
when(rs.getString("location")).thenReturn("Amsterdam");
when(rs.getString("url")).thenReturn("https://example.com/auction/A1-12345");
when(rs.getInt("lots_count")).thenReturn(50);
when(rs.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo result = ScraperDataAdapter.fromScraperAuction(rs);
assertEquals("Amsterdam", result.city());
assertEquals("", result.country());
assertNull(result.firstLotClosingTime());
}
@Test
@DisplayName("Should convert scraper lot format to Lot")
void testFromScraperLot() throws SQLException {
ResultSet rs = mock(ResultSet.class);
when(rs.getString("lot_id")).thenReturn("A1-28505-5");
when(rs.getString("auction_id")).thenReturn("A7-39813");
when(rs.getString("title")).thenReturn("Forklift Toyota");
when(rs.getString("description")).thenReturn("Electric forklift in good condition");
when(rs.getString("category")).thenReturn("Machinery");
when(rs.getString("current_bid")).thenReturn("€1250.50");
when(rs.getString("closing_time")).thenReturn("2025-12-15T14:30:00");
when(rs.getString("url")).thenReturn("https://example.com/lot/A1-28505-5");
Lot result = ScraperDataAdapter.fromScraperLot(rs);
assertNotNull(result);
assertEquals(285055, result.lotId());
assertEquals(39813, result.saleId());
assertEquals("Forklift Toyota", result.title());
assertEquals("Electric forklift in good condition", result.description());
assertEquals("Machinery", result.category());
assertEquals(1250.50, result.currentBid(), 0.01);
assertEquals("EUR", result.currency());
assertEquals("https://example.com/lot/A1-28505-5", result.url());
assertNotNull(result.closingTime());
assertFalse(result.closingNotified());
}
@Test
@DisplayName("Should parse bid amount from various formats")
void testParseBidAmount() throws SQLException {
// Test €123.45 format
ResultSet rs1 = createLotResultSet("€123.45");
Lot lot1 = ScraperDataAdapter.fromScraperLot(rs1);
assertEquals(123.45, lot1.currentBid(), 0.01);
assertEquals("EUR", lot1.currency());
// Test $50.00 format
ResultSet rs2 = createLotResultSet("$50.00");
Lot lot2 = ScraperDataAdapter.fromScraperLot(rs2);
assertEquals(50.00, lot2.currentBid(), 0.01);
assertEquals("USD", lot2.currency());
// Test "No bids" format
ResultSet rs3 = createLotResultSet("No bids");
Lot lot3 = ScraperDataAdapter.fromScraperLot(rs3);
assertEquals(0.0, lot3.currentBid(), 0.01);
// Test plain number
ResultSet rs4 = createLotResultSet("999.99");
Lot lot4 = ScraperDataAdapter.fromScraperLot(rs4);
assertEquals(999.99, lot4.currentBid(), 0.01);
}
@Test
@DisplayName("Should handle missing or null fields gracefully")
void testHandleNullFields() throws SQLException {
ResultSet rs = mock(ResultSet.class);
when(rs.getString("lot_id")).thenReturn("A1-12345-1");
when(rs.getString("auction_id")).thenReturn("A7-99999");
when(rs.getString("title")).thenReturn("Test Lot");
when(rs.getString("description")).thenReturn(null);
when(rs.getString("category")).thenReturn(null);
when(rs.getString("current_bid")).thenReturn(null);
when(rs.getString("closing_time")).thenReturn(null);
when(rs.getString("url")).thenReturn("https://example.com/lot");
Lot result = ScraperDataAdapter.fromScraperLot(rs);
assertNotNull(result);
assertEquals("", result.description());
assertEquals("", result.category());
assertEquals(0.0, result.currentBid());
assertNull(result.closingTime());
}
@Test
@DisplayName("Should parse various timestamp formats")
void testTimestampParsing() throws SQLException {
// ISO local date time
ResultSet rs1 = mock(ResultSet.class);
setupBasicLotMock(rs1);
when(rs1.getString("closing_time")).thenReturn("2025-12-15T14:30:00");
Lot lot1 = ScraperDataAdapter.fromScraperLot(rs1);
assertNotNull(lot1.closingTime());
assertEquals(LocalDateTime.of(2025, 12, 15, 14, 30, 0), lot1.closingTime());
// SQL timestamp format
ResultSet rs2 = mock(ResultSet.class);
setupBasicLotMock(rs2);
when(rs2.getString("closing_time")).thenReturn("2025-12-15 14:30:00");
Lot lot2 = ScraperDataAdapter.fromScraperLot(rs2);
assertNotNull(lot2.closingTime());
}
@Test
@DisplayName("Should handle invalid timestamp gracefully")
void testInvalidTimestamp() throws SQLException {
ResultSet rs = mock(ResultSet.class);
setupBasicLotMock(rs);
when(rs.getString("closing_time")).thenReturn("invalid-date");
Lot result = ScraperDataAdapter.fromScraperLot(rs);
assertNull(result.closingTime());
}
@Test
@DisplayName("Should extract type prefix from auction ID")
void testTypeExtraction() throws SQLException {
ResultSet rs1 = mock(ResultSet.class);
when(rs1.getString("auction_id")).thenReturn("A7-39813");
when(rs1.getString("title")).thenReturn("Test");
when(rs1.getString("location")).thenReturn("Test, NL");
when(rs1.getString("url")).thenReturn("http://test.com");
when(rs1.getInt("lots_count")).thenReturn(10);
when(rs1.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo auction1 = ScraperDataAdapter.fromScraperAuction(rs1);
assertEquals("A7", auction1.typePrefix());
ResultSet rs2 = mock(ResultSet.class);
when(rs2.getString("auction_id")).thenReturn("B1-12345");
when(rs2.getString("title")).thenReturn("Test");
when(rs2.getString("location")).thenReturn("Test, NL");
when(rs2.getString("url")).thenReturn("http://test.com");
when(rs2.getInt("lots_count")).thenReturn(10);
when(rs2.getString("first_lot_closing_time")).thenReturn(null);
AuctionInfo auction2 = ScraperDataAdapter.fromScraperAuction(rs2);
assertEquals("B1", auction2.typePrefix());
}
@Test
@DisplayName("Should handle GBP currency symbol")
void testGBPCurrency() throws SQLException {
ResultSet rs = createLotResultSet("£75.00");
Lot lot = ScraperDataAdapter.fromScraperLot(rs);
assertEquals(75.00, lot.currentBid(), 0.01);
assertEquals("GBP", lot.currency());
}
// Helper methods
private ResultSet createLotResultSet(String bidAmount) throws SQLException {
ResultSet rs = mock(ResultSet.class);
when(rs.getString("lot_id")).thenReturn("A1-12345-1");
when(rs.getString("auction_id")).thenReturn("A7-99999");
when(rs.getString("title")).thenReturn("Test Lot");
when(rs.getString("description")).thenReturn("Test description");
when(rs.getString("category")).thenReturn("Test");
when(rs.getString("current_bid")).thenReturn(bidAmount);
when(rs.getString("closing_time")).thenReturn("2025-12-15T14:30:00");
when(rs.getString("url")).thenReturn("https://example.com/lot");
return rs;
}
private void setupBasicLotMock(ResultSet rs) throws SQLException {
when(rs.getString("lot_id")).thenReturn("A1-12345-1");
when(rs.getString("auction_id")).thenReturn("A7-99999");
when(rs.getString("title")).thenReturn("Test Lot");
when(rs.getString("description")).thenReturn("Test");
when(rs.getString("category")).thenReturn("Test");
when(rs.getString("current_bid")).thenReturn("€100.00");
when(rs.getString("url")).thenReturn("https://example.com/lot");
}
}

View File

@@ -0,0 +1,380 @@
package auctiora;
import org.junit.jupiter.api.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.time.LocalDateTime;
import static org.junit.jupiter.api.Assertions.*;
/**
* Test cases for TroostwijkMonitor.
* Tests monitoring orchestration, bid tracking, and notification triggers.
*/
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TroostwijkMonitorTest {
private String testDbPath;
private TroostwijkMonitor monitor;
@BeforeAll
void setUp() throws SQLException, IOException {
testDbPath = "test_monitor_" + System.currentTimeMillis() + ".db";
monitor = new TroostwijkMonitor(
testDbPath,
"desktop",
"non_existent.cfg",
"non_existent.weights",
"non_existent.txt"
);
}
@AfterAll
void tearDown() throws Exception {
Files.deleteIfExists(Paths.get(testDbPath));
}
@Test
@DisplayName("Should initialize monitor successfully")
void testMonitorInitialization() {
assertNotNull(monitor);
assertNotNull(monitor.getDb());
}
@Test
@DisplayName("Should print database stats without error")
void testPrintDatabaseStats() {
assertDoesNotThrow(() -> monitor.printDatabaseStats());
}
@Test
@DisplayName("Should process pending images without error")
void testProcessPendingImages() {
assertDoesNotThrow(() -> monitor.processPendingImages());
}
@Test
@DisplayName("Should handle empty database gracefully")
void testEmptyDatabaseHandling() throws SQLException {
var auctions = monitor.getDb().getAllAuctions();
var lots = monitor.getDb().getAllLots();
assertNotNull(auctions);
assertNotNull(lots);
assertTrue(auctions.isEmpty() || auctions.size() >= 0);
}
@Test
@DisplayName("Should track lots in database")
void testLotTracking() throws SQLException {
// Insert test lot
var lot = Lot.basic(
11111, 22222,
"Test Forklift",
"Electric forklift in good condition",
"Toyota",
"Electric",
2020,
"Machinery",
1500.00,
"EUR",
"https://example.com/lot/22222",
LocalDateTime.now().plusDays(1),
false
);
monitor.getDb().upsertLot(lot);
var lots = monitor.getDb().getAllLots();
assertTrue(lots.stream().anyMatch(l -> l.lotId() == 22222));
}
@Test
@DisplayName("Should monitor lots closing soon")
void testClosingSoonMonitoring() throws SQLException {
// Insert lot closing in 4 minutes
var closingSoon = Lot.basic(
33333, 44444,
"Closing Soon Item",
"Description",
"",
"",
0,
"Category",
100.00,
"EUR",
"https://example.com/lot/44444",
LocalDateTime.now().plusMinutes(4),
false
);
monitor.getDb().upsertLot(closingSoon);
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 44444)
.findFirst()
.orElse(null);
assertNotNull(found);
assertTrue(found.minutesUntilClose() < 30);
}
@Test
@DisplayName("Should identify lots with time remaining")
void testTimeRemainingCalculation() throws SQLException {
var futureLot = Lot.basic(
55555, 66666,
"Future Lot",
"Description",
"",
"",
0,
"Category",
200.00,
"EUR",
"https://example.com/lot/66666",
LocalDateTime.now().plusHours(2),
false
);
monitor.getDb().upsertLot(futureLot);
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 66666)
.findFirst()
.orElse(null);
assertNotNull(found);
assertTrue(found.minutesUntilClose() > 60);
}
@Test
@DisplayName("Should handle lots without closing time")
void testLotsWithoutClosingTime() throws SQLException {
var noClosing = Lot.basic(
77777, 88888,
"No Closing Time",
"Description",
"",
"",
0,
"Category",
150.00,
"EUR",
"https://example.com/lot/88888",
null,
false
);
monitor.getDb().upsertLot(noClosing);
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 88888)
.findFirst()
.orElse(null);
assertNotNull(found);
assertNull(found.closingTime());
}
@Test
@DisplayName("Should track notification status")
void testNotificationStatusTracking() throws SQLException {
var lot = Lot.basic(
99999, 11110,
"Test Notification",
"Description",
"",
"",
0,
"Category",
100.00,
"EUR",
"https://example.com/lot/11110",
LocalDateTime.now().plusMinutes(3),
false
);
monitor.getDb().upsertLot(lot);
// Update notification flag
var notified = Lot.basic(
99999, 11110,
"Test Notification",
"Description",
"",
"",
0,
"Category",
100.00,
"EUR",
"https://example.com/lot/11110",
LocalDateTime.now().plusMinutes(3),
true
);
monitor.getDb().updateLotNotificationFlags(notified);
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 11110)
.findFirst()
.orElse(null);
assertNotNull(found);
assertTrue(found.closingNotified());
}
@Test
@DisplayName("Should update bid amounts")
void testBidAmountUpdates() throws SQLException {
var lot = Lot.basic(
12121, 13131,
"Bid Update Test",
"Description",
"",
"",
0,
"Category",
100.00,
"EUR",
"https://example.com/lot/13131",
LocalDateTime.now().plusDays(1),
false
);
monitor.getDb().upsertLot(lot);
// Simulate bid increase
var higherBid = Lot.basic(
12121, 13131,
"Bid Update Test",
"Description",
"",
"",
0,
"Category",
250.00,
"EUR",
"https://example.com/lot/13131",
LocalDateTime.now().plusDays(1),
false
);
monitor.getDb().updateLotCurrentBid(higherBid);
var lots = monitor.getDb().getActiveLots();
var found = lots.stream()
.filter(l -> l.lotId() == 13131)
.findFirst()
.orElse(null);
assertNotNull(found);
assertEquals(250.00, found.currentBid(), 0.01);
}
@Test
@DisplayName("Should handle multiple concurrent lot updates")
void testConcurrentLotUpdates() throws InterruptedException, SQLException {
Thread t1 = new Thread(() -> {
try {
for (int i = 0; i < 5; i++) {
monitor.getDb().upsertLot(Lot.basic(
20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
100.0, "EUR", "https://example.com/" + i, null, false
));
}
} catch (Exception e) {
fail("Thread 1 failed: " + e.getMessage());
}
});
Thread t2 = new Thread(() -> {
try {
for (int i = 5; i < 10; i++) {
monitor.getDb().upsertLot(Lot.basic(
20000 + i, 30000 + i, "Concurrent " + i, "Desc", "", "", 0, "Cat",
200.0, "EUR", "https://example.com/" + i, null, false
));
}
} catch (Exception e) {
fail("Thread 2 failed: " + e.getMessage());
}
});
t1.start();
t2.start();
t1.join();
t2.join();
var lots = monitor.getDb().getActiveLots();
long count = lots.stream()
.filter(l -> l.lotId() >= 30000 && l.lotId() < 30010)
.count();
assertTrue(count >= 10);
}
@Test
@DisplayName("Should schedule monitoring without error")
void testScheduleMonitoring() {
// This just tests that scheduling doesn't throw
// Actual monitoring would run in background
assertDoesNotThrow(() -> {
// Don't actually start monitoring in test
// Just verify monitor is ready
assertNotNull(monitor);
});
}
@Test
@DisplayName("Should handle database with auctions and lots")
void testDatabaseWithData() throws SQLException {
// Insert auction
var auction = new AuctionInfo(
40000,
"Test Auction",
"Amsterdam, NL",
"Amsterdam",
"NL",
"https://example.com/auction/40000",
"A7",
10,
LocalDateTime.now().plusDays(2)
);
monitor.getDb().upsertAuction(auction);
// Insert related lot
var lot = Lot.basic(
40000, 50000,
"Test Lot",
"Description",
"",
"",
0,
"Category",
500.00,
"EUR",
"https://example.com/lot/50000",
LocalDateTime.now().plusDays(2),
false
);
monitor.getDb().upsertLot(lot);
// Verify
var auctions = monitor.getDb().getAllAuctions();
var lots = monitor.getDb().getAllLots();
assertTrue(auctions.stream().anyMatch(a -> a.auctionId() == 40000));
assertTrue(lots.stream().anyMatch(l -> l.lotId() == 50000));
}
}

20
workflows/maven.yml Normal file
View File

@@ -0,0 +1,20 @@
name: Publish to Gitea Package Registry
on:
push:
tags:
- 'v*'
jobs:
publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up JDK 25
uses: actions/setup-java@v4
with:
java-version: '25'
distribution: 'temurin'
- name: Publish with Maven
run: mvn --batch-mode clean deploy
env:
GITEA_TOKEN: ${{ secrets.EA_PUBLISH_TOKEN }}