Fix mock tests
This commit is contained in:
Tour
@@ -1,9 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<extensions>
|
||||
<!-- Override Maven's internal Guice with a Java 25 compatible version -->
|
||||
<extension>
|
||||
<groupId>com.google.inject</groupId>
|
||||
<artifactId>guice</artifactId>
|
||||
<version>7.0.0</version>
|
||||
</extension>
|
||||
</extensions>
|
||||
@@ -1,5 +0,0 @@
|
||||
--add-opens=java.base/java.lang=ALL-UNNAMED
|
||||
--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED
|
||||
--add-exports=java.base/sun.nio.ch=ALL-UNNAMED
|
||||
-Djdk.module.illegalAccess=deny
|
||||
-XX:+IgnoreUnrecognizedVMOptions
|
||||
@@ -1,117 +0,0 @@
|
||||
/*
|
||||
* Copyright 2007-present the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import java.net.*;
|
||||
import java.io.*;
|
||||
import java.nio.channels.*;
|
||||
import java.util.Properties;
|
||||
|
||||
public class MavenWrapperDownloader {
|
||||
|
||||
private static final String WRAPPER_VERSION = "0.5.6";
|
||||
/**
|
||||
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
|
||||
*/
|
||||
private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
|
||||
+ WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
|
||||
|
||||
/**
|
||||
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
|
||||
* use instead of the default one.
|
||||
*/
|
||||
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
|
||||
".mvn/wrapper/maven-wrapper.properties";
|
||||
|
||||
/**
|
||||
* Path where the maven-wrapper.jar will be saved to.
|
||||
*/
|
||||
private static final String MAVEN_WRAPPER_JAR_PATH =
|
||||
".mvn/wrapper/maven-wrapper.jar";
|
||||
|
||||
/**
|
||||
* Name of the property which should be used to override the default download url for the wrapper.
|
||||
*/
|
||||
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
|
||||
|
||||
public static void main(String args[]) {
|
||||
System.out.println("- Downloader started");
|
||||
File baseDirectory = new File(args[0]);
|
||||
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
|
||||
|
||||
// If the maven-wrapper.properties exists, read it and check if it contains a custom
|
||||
// wrapperUrl parameter.
|
||||
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
|
||||
String url = DEFAULT_DOWNLOAD_URL;
|
||||
if(mavenWrapperPropertyFile.exists()) {
|
||||
FileInputStream mavenWrapperPropertyFileInputStream = null;
|
||||
try {
|
||||
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
|
||||
Properties mavenWrapperProperties = new Properties();
|
||||
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
|
||||
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
|
||||
} catch (IOException e) {
|
||||
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
|
||||
} finally {
|
||||
try {
|
||||
if(mavenWrapperPropertyFileInputStream != null) {
|
||||
mavenWrapperPropertyFileInputStream.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// Ignore ...
|
||||
}
|
||||
}
|
||||
}
|
||||
System.out.println("- Downloading from: " + url);
|
||||
|
||||
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
|
||||
if(!outputFile.getParentFile().exists()) {
|
||||
if(!outputFile.getParentFile().mkdirs()) {
|
||||
System.out.println(
|
||||
"- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
|
||||
}
|
||||
}
|
||||
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
|
||||
try {
|
||||
downloadFileFromURL(url, outputFile);
|
||||
System.out.println("Done");
|
||||
System.exit(0);
|
||||
} catch (Throwable e) {
|
||||
System.out.println("- Error downloading");
|
||||
e.printStackTrace();
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
|
||||
if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
|
||||
String username = System.getenv("MVNW_USERNAME");
|
||||
char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
|
||||
Authenticator.setDefault(new Authenticator() {
|
||||
@Override
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(username, password);
|
||||
}
|
||||
});
|
||||
}
|
||||
URL website = new URL(urlString);
|
||||
ReadableByteChannel rbc;
|
||||
rbc = Channels.newChannel(website.openStream());
|
||||
FileOutputStream fos = new FileOutputStream(destination);
|
||||
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
|
||||
fos.close();
|
||||
rbc.close();
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
@@ -1,2 +0,0 @@
|
||||
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3/apache-maven-3-bin.zip
|
||||
wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar
|
||||
192
docs/DATABASE_CLEANUP_GUIDE.md
Normal file
192
docs/DATABASE_CLEANUP_GUIDE.md
Normal file
@@ -0,0 +1,192 @@
|
||||
# Database Cleanup Guide
|
||||
|
||||
## Problem: Mixed Data Formats
|
||||
|
||||
Your production database (`cache.db`) contains data from two different scrapers:
|
||||
|
||||
### Valid Data (99.92%)
|
||||
- **Format**: `A1-34732-49` (lot_id) + `c1f44ec2-ad6e-4c98-b0e2-cb1d8ccddcab` (auction_id UUID)
|
||||
- **Count**: 16,794 lots
|
||||
- **Source**: Current GraphQL-based scraper
|
||||
- **Status**: ✅ Clean, with proper auction_id
|
||||
|
||||
### Invalid Data (0.08%)
|
||||
- **Format**: `bmw-550i-4-4-v8-high-executive-...` (slug as lot_id) + `""` (empty auction_id)
|
||||
- **Count**: 13 lots
|
||||
- **Source**: Old legacy scraper
|
||||
- **Status**: ❌ Missing auction_id, causes issues
|
||||
|
||||
## Impact
|
||||
|
||||
These 13 invalid entries:
|
||||
- Cause `NullPointerException` in analytics when grouping by country
|
||||
- Cannot be properly linked to auctions
|
||||
- Skew statistics slightly
|
||||
- May cause issues with intelligence features that rely on auction_id
|
||||
|
||||
## Solution 1: Clean Sync (Recommended)
|
||||
|
||||
The updated sync script now **automatically removes old local data** before syncing:
|
||||
|
||||
```bash
|
||||
# Windows PowerShell
|
||||
.\scripts\Sync-ProductionData.ps1
|
||||
|
||||
# Linux/Mac
|
||||
./scripts/sync-production-data.sh --db-only
|
||||
```
|
||||
|
||||
**What it does**:
|
||||
1. Backs up existing database to `cache.db.backup-YYYYMMDD-HHMMSS`
|
||||
2. **Removes old local database completely**
|
||||
3. Downloads fresh copy from production
|
||||
4. Shows data quality report
|
||||
|
||||
**Output includes**:
|
||||
```
|
||||
Database statistics:
|
||||
┌─────────────┬────────┐
|
||||
│ table_name │ count │
|
||||
├─────────────┼────────┤
|
||||
│ auctions │ 526 │
|
||||
│ lots │ 16807 │
|
||||
│ images │ 536502 │
|
||||
│ cache │ 2134 │
|
||||
└─────────────┴────────┘
|
||||
|
||||
Data quality:
|
||||
┌────────────────────────────────────┬────────┬────────────┐
|
||||
│ metric │ count │ percentage │
|
||||
├────────────────────────────────────┼────────┼────────────┤
|
||||
│ Valid lots │ 16794 │ 99.92% │
|
||||
│ Invalid lots (missing auction_id) │ 13 │ 0.08% │
|
||||
│ Lots with intelligence fields │ 0 │ 0.00% │
|
||||
└────────────────────────────────────┴────────┴────────────┘
|
||||
```
|
||||
|
||||
## Solution 2: Manual Cleanup
|
||||
|
||||
If you want to clean your existing local database without re-downloading:
|
||||
|
||||
```bash
|
||||
# Dry run (see what would be deleted)
|
||||
./scripts/cleanup-database.sh --dry-run
|
||||
|
||||
# Actual cleanup
|
||||
./scripts/cleanup-database.sh
|
||||
```
|
||||
|
||||
**What it does**:
|
||||
1. Creates backup before cleanup
|
||||
2. Deletes lots with missing auction_id
|
||||
3. Deletes orphaned images (images without matching lots)
|
||||
4. Compacts database (VACUUM) to reclaim space
|
||||
5. Shows before/after statistics
|
||||
|
||||
**Example output**:
|
||||
```
|
||||
Current database state:
|
||||
┌──────────────────────────────────┬────────┐
|
||||
│ metric │ count │
|
||||
├──────────────────────────────────┼────────┤
|
||||
│ Total lots │ 16807 │
|
||||
│ Valid lots (with auction_id) │ 16794 │
|
||||
│ Invalid lots (missing auction_id) │ 13 │
|
||||
└──────────────────────────────────┴────────┘
|
||||
|
||||
Analyzing data to clean up...
|
||||
→ Invalid lots to delete: 13
|
||||
→ Orphaned images to delete: 0
|
||||
|
||||
This will permanently delete the above records.
|
||||
Continue? (y/N) y
|
||||
|
||||
Cleaning up database...
|
||||
[1/2] Deleting invalid lots...
|
||||
✓ Deleted 13 invalid lots
|
||||
[2/2] Deleting orphaned images...
|
||||
✓ Deleted 0 orphaned images
|
||||
[3/3] Compacting database...
|
||||
✓ Database compacted
|
||||
|
||||
Final database state:
|
||||
┌───────────────┬────────┐
|
||||
│ metric │ count │
|
||||
├───────────────┼────────┤
|
||||
│ Total lots │ 16794 │
|
||||
│ Total images │ 536502 │
|
||||
└───────────────┴────────┘
|
||||
|
||||
Database size: 8.9G
|
||||
```
|
||||
|
||||
## Solution 3: SQL Manual Cleanup
|
||||
|
||||
If you prefer to manually clean using SQL:
|
||||
|
||||
```sql
|
||||
-- Backup first!
|
||||
-- cp cache.db cache.db.backup
|
||||
|
||||
-- Check invalid entries
|
||||
SELECT COUNT(*), 'Invalid' as type
|
||||
FROM lots
|
||||
WHERE auction_id IS NULL OR auction_id = ''
|
||||
UNION ALL
|
||||
SELECT COUNT(*), 'Valid'
|
||||
FROM lots
|
||||
WHERE auction_id IS NOT NULL AND auction_id != '';
|
||||
|
||||
-- Delete invalid lots
|
||||
DELETE FROM lots
|
||||
WHERE auction_id IS NULL OR auction_id = '';
|
||||
|
||||
-- Delete orphaned images
|
||||
DELETE FROM images
|
||||
WHERE lot_id NOT IN (SELECT lot_id FROM lots);
|
||||
|
||||
-- Compact database
|
||||
VACUUM;
|
||||
```
|
||||
|
||||
## Prevention: Production Database Cleanup
|
||||
|
||||
To prevent these invalid entries from accumulating on production, you can:
|
||||
|
||||
1. **Clean production database** (one-time):
|
||||
```bash
|
||||
ssh tour@athena.lan
|
||||
docker run --rm -v shared-auction-data:/data alpine sqlite3 /data/cache.db "DELETE FROM lots WHERE auction_id IS NULL OR auction_id = '';"
|
||||
```
|
||||
|
||||
2. **Update scraper** to ensure all lots have auction_id
|
||||
3. **Add validation** in scraper to reject lots without auction_id
|
||||
|
||||
## When to Clean
|
||||
|
||||
### Immediately if:
|
||||
- ❌ Seeing `NullPointerException` in analytics
|
||||
- ❌ Dashboard insights failing
|
||||
- ❌ Country distribution not working
|
||||
|
||||
### Periodically:
|
||||
- 🔄 After syncing from production (if production has invalid data)
|
||||
- 🔄 Weekly/monthly maintenance
|
||||
- 🔄 Before major testing or demos
|
||||
|
||||
## Recommendation
|
||||
|
||||
**Use Solution 1 (Clean Sync)** for simplicity:
|
||||
- ✅ Guarantees clean state
|
||||
- ✅ No manual SQL needed
|
||||
- ✅ Shows data quality report
|
||||
- ✅ Safe (automatic backup)
|
||||
|
||||
The 13 invalid entries are from an old scraper and represent only 0.08% of data, so cleaning them up has minimal impact but prevents future errors.
|
||||
|
||||
---
|
||||
|
||||
**Related Documentation**:
|
||||
- [Sync Scripts README](../scripts/README.md)
|
||||
- [Data Sync Setup](DATA_SYNC_SETUP.md)
|
||||
- [Database Architecture](../wiki/DATABASE_ARCHITECTURE.md)
|
||||
@@ -1,20 +0,0 @@
|
||||
@echo off
|
||||
REM ============================================================================
|
||||
REM Troostwijk Auction Monitor - Status Check (Windows)
|
||||
REM ============================================================================
|
||||
REM
|
||||
REM This script shows the current status and exits.
|
||||
REM
|
||||
REM Usage:
|
||||
REM check-status.bat
|
||||
REM
|
||||
REM ============================================================================
|
||||
|
||||
REM Set configuration
|
||||
set DATABASE_FILE=C:\mnt\okcomputer\output\cache.db
|
||||
set NOTIFICATION_CONFIG=desktop
|
||||
|
||||
REM Check status
|
||||
java -jar target\auctiora-1.0-SNAPSHOT-jar-with-dependencies.jar status
|
||||
|
||||
pause
|
||||
@@ -1,30 +0,0 @@
|
||||
@echo off
|
||||
if "%1"=="" (
|
||||
echo Error: Commit message is required
|
||||
echo Usage: persist "Your commit message"
|
||||
echo Example: persist "Fixed login bug"
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Adding all changes...
|
||||
git add .
|
||||
if errorlevel 1 (
|
||||
echo Error: Failed to add changes
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Committing with message: "%*"
|
||||
git commit -a -m "%*"
|
||||
if errorlevel 1 (
|
||||
echo Error: Failed to commit
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo Pushing to remote...
|
||||
git push
|
||||
if errorlevel 1 (
|
||||
echo Error: Failed to push
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
echo All operations completed successfully!
|
||||
@@ -1,27 +0,0 @@
|
||||
@echo off
|
||||
REM ============================================================================
|
||||
REM Troostwijk Auction Monitor - Run Once (Windows)
|
||||
REM ============================================================================
|
||||
REM
|
||||
REM This script runs the complete workflow once and exits.
|
||||
REM Perfect for Windows Task Scheduler.
|
||||
REM
|
||||
REM Usage:
|
||||
REM run-once.bat
|
||||
REM
|
||||
REM Schedule in Task Scheduler:
|
||||
REM - Every 30 minutes: Data import
|
||||
REM - Every 1 hour: Image processing
|
||||
REM - Every 15 minutes: Bid monitoring
|
||||
REM
|
||||
REM ============================================================================
|
||||
|
||||
REM Set configuration
|
||||
set DATABASE_FILE=C:\mnt\okcomputer\output\cache.db
|
||||
set NOTIFICATION_CONFIG=desktop
|
||||
|
||||
REM Run the application once
|
||||
java -jar target\troostwijk-scraper-1.0-SNAPSHOT-jar-with-dependencies.jar once
|
||||
|
||||
REM Exit code for Task Scheduler
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -1,27 +0,0 @@
|
||||
@echo off
|
||||
REM ============================================================================
|
||||
REM Troostwijk Auction Monitor - Workflow Runner (Windows)
|
||||
REM ============================================================================
|
||||
REM
|
||||
REM This script runs the auction monitor in workflow mode (continuous operation)
|
||||
REM with all scheduled tasks running automatically.
|
||||
REM
|
||||
REM Usage:
|
||||
REM run-workflow.bat
|
||||
REM
|
||||
REM ============================================================================
|
||||
|
||||
echo Starting Troostwijk Auction Monitor - Workflow Mode...
|
||||
echo.
|
||||
|
||||
REM Set configuration
|
||||
set DATABASE_FILE=C:\mnt\okcomputer\output\cache.db
|
||||
set NOTIFICATION_CONFIG=desktop
|
||||
|
||||
REM Optional: Set for email notifications
|
||||
REM set NOTIFICATION_CONFIG=smtp:your@gmail.com:app_password:your@gmail.com
|
||||
|
||||
REM Run the application
|
||||
java -jar target\auctiora-1.0-SNAPSHOT-jar-with-dependencies.jar workflow
|
||||
|
||||
pause
|
||||
@@ -1,71 +0,0 @@
|
||||
# ============================================================================
|
||||
# Troostwijk Auction Monitor - Windows Task Scheduler Setup
|
||||
# ============================================================================
|
||||
#
|
||||
# This PowerShell script creates scheduled tasks in Windows Task Scheduler
|
||||
# to run the auction monitor automatically.
|
||||
#
|
||||
# Usage:
|
||||
# Run PowerShell as Administrator, then:
|
||||
# .\setup-windows-task.ps1
|
||||
#
|
||||
# ============================================================================
|
||||
|
||||
Write-Host "=== Auctiora Monitor - Task Scheduler Setup ===" -ForegroundColor Cyan
|
||||
Write-Host ""
|
||||
|
||||
# Configuration
|
||||
$scriptPath = $PSScriptRoot
|
||||
$jarPath = Join-Path $scriptPath "target\auctiora-1.0-SNAPSHOT-jar-with-dependencies.jar"
|
||||
$javaExe = "java"
|
||||
|
||||
# Check if JAR exists
|
||||
if (-not (Test-Path $jarPath)) {
|
||||
Write-Host "ERROR: JAR file not found at: $jarPath" -ForegroundColor Red
|
||||
Write-Host "Please run 'mvn clean package' first." -ForegroundColor Yellow
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host "Creating scheduled tasks..." -ForegroundColor Green
|
||||
Write-Host ""
|
||||
|
||||
# Task 1: Complete Workflow - Every 30 minutes
|
||||
$task1Name = "TroostwijkMonitor-Workflow"
|
||||
$task1Description = "Runs complete auction monitoring workflow every 30 minutes"
|
||||
$task1Action = New-ScheduledTaskAction -Execute $javaExe -Argument "-jar `"$jarPath`" once" -WorkingDirectory $scriptPath
|
||||
$task1Trigger = New-ScheduledTaskTrigger -Once -At (Get-Date) -RepetitionInterval (New-TimeSpan -Minutes 30) -RepetitionDuration ([TimeSpan]::MaxValue)
|
||||
$task1Settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable
|
||||
|
||||
try {
|
||||
Register-ScheduledTask -TaskName $task1Name -Action $task1Action -Trigger $task1Trigger -Settings $task1Settings -Description $task1Description -Force
|
||||
Write-Host "[✓] Created task: $task1Name (every 30 min)" -ForegroundColor Green
|
||||
} catch {
|
||||
Write-Host "[✗] Failed to create task: $task1Name" -ForegroundColor Red
|
||||
Write-Host " Error: $_" -ForegroundColor Yellow
|
||||
}
|
||||
|
||||
# Task 2: Status Check - Every 6 hours
|
||||
$task2Name = "TroostwijkMonitor-StatusCheck"
|
||||
$task2Description = "Checks auction monitoring status every 6 hours"
|
||||
$task2Action = New-ScheduledTaskAction -Execute $javaExe -Argument "-jar `"$jarPath`" status" -WorkingDirectory $scriptPath
|
||||
$task2Trigger = New-ScheduledTaskTrigger -Once -At (Get-Date) -RepetitionInterval (New-TimeSpan -Hours 6) -RepetitionDuration ([TimeSpan]::MaxValue)
|
||||
$task2Settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable
|
||||
|
||||
try {
|
||||
Register-ScheduledTask -TaskName $task2Name -Action $task2Action -Trigger $task2Trigger -Settings $task2Settings -Description $task2Description -Force
|
||||
Write-Host "[✓] Created task: $task2Name (every 6 hours)" -ForegroundColor Green
|
||||
} catch {
|
||||
Write-Host "[✗] Failed to create task: $task2Name" -ForegroundColor Red
|
||||
Write-Host " Error: $_" -ForegroundColor Yellow
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "=== Setup Complete ===" -ForegroundColor Cyan
|
||||
Write-Host ""
|
||||
Write-Host "Created tasks:" -ForegroundColor White
|
||||
Write-Host " 1. $task1Name - Runs every 30 minutes" -ForegroundColor Gray
|
||||
Write-Host " 2. $task2Name - Runs every 6 hours" -ForegroundColor Gray
|
||||
Write-Host ""
|
||||
Write-Host "To view tasks: Open Task Scheduler and look for 'TroostwijkMonitor-*'" -ForegroundColor Yellow
|
||||
Write-Host "To remove tasks: Run 'Unregister-ScheduledTask -TaskName TroostwijkMonitor-*'" -ForegroundColor Yellow
|
||||
Write-Host ""
|
||||
160
scripts/cleanup-database.sh
Normal file
160
scripts/cleanup-database.sh
Normal file
@@ -0,0 +1,160 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Database Cleanup Utility
|
||||
#
|
||||
# Removes invalid/old data from the local database
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/cleanup-database.sh [--dry-run]
|
||||
#
|
||||
# Options:
|
||||
# --dry-run Show what would be deleted without actually deleting
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
# Configuration
|
||||
LOCAL_DB_PATH="${1:-c:/mnt/okcomputer/cache.db}"
|
||||
DRY_RUN=false
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Parse arguments
|
||||
if [ "$1" = "--dry-run" ] || [ "$2" = "--dry-run" ]; then
|
||||
DRY_RUN=true
|
||||
fi
|
||||
|
||||
if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then
|
||||
grep '^#' "$0" | sed 's/^# \?//'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}╔════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${BLUE}║ Database Cleanup - Auctiora Monitor ║${NC}"
|
||||
echo -e "${BLUE}╚════════════════════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
|
||||
if [ ! -f "${LOCAL_DB_PATH}" ]; then
|
||||
echo -e "${RED}Error: Database not found at ${LOCAL_DB_PATH}${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Backup database before cleanup
|
||||
if [ "$DRY_RUN" = false ]; then
|
||||
BACKUP_PATH="${LOCAL_DB_PATH}.backup-before-cleanup-$(date +%Y%m%d-%H%M%S)"
|
||||
echo -e "${YELLOW}Creating backup: ${BACKUP_PATH}${NC}"
|
||||
cp "${LOCAL_DB_PATH}" "${BACKUP_PATH}"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Show current state
|
||||
echo -e "${BLUE}Current database state:${NC}"
|
||||
sqlite3 "${LOCAL_DB_PATH}" <<EOF
|
||||
.mode box
|
||||
SELECT
|
||||
'Total lots' as metric,
|
||||
COUNT(*) as count
|
||||
FROM lots
|
||||
UNION ALL
|
||||
SELECT
|
||||
'Valid lots (with auction_id)',
|
||||
COUNT(*)
|
||||
FROM lots
|
||||
WHERE auction_id IS NOT NULL AND auction_id != ''
|
||||
UNION ALL
|
||||
SELECT
|
||||
'Invalid lots (missing auction_id)',
|
||||
COUNT(*)
|
||||
FROM lots
|
||||
WHERE auction_id IS NULL OR auction_id = '';
|
||||
EOF
|
||||
echo ""
|
||||
|
||||
# Count items to be deleted
|
||||
echo -e "${YELLOW}Analyzing data to clean up...${NC}"
|
||||
|
||||
INVALID_LOTS=$(sqlite3 "${LOCAL_DB_PATH}" "SELECT COUNT(*) FROM lots WHERE auction_id IS NULL OR auction_id = '';")
|
||||
ORPHANED_IMAGES=$(sqlite3 "${LOCAL_DB_PATH}" "SELECT COUNT(*) FROM images WHERE lot_id NOT IN (SELECT lot_id FROM lots);")
|
||||
|
||||
echo -e " ${RED}→ Invalid lots to delete: ${INVALID_LOTS}${NC}"
|
||||
echo -e " ${YELLOW}→ Orphaned images to delete: ${ORPHANED_IMAGES}${NC}"
|
||||
echo ""
|
||||
|
||||
if [ "$INVALID_LOTS" -eq 0 ] && [ "$ORPHANED_IMAGES" -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ Database is clean! No cleanup needed.${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$DRY_RUN" = true ]; then
|
||||
echo -e "${BLUE}DRY RUN MODE - No changes will be made${NC}"
|
||||
echo ""
|
||||
echo "Would delete:"
|
||||
echo " - $INVALID_LOTS invalid lots"
|
||||
echo " - $ORPHANED_IMAGES orphaned images"
|
||||
echo ""
|
||||
echo "Run without --dry-run to perform cleanup"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Confirm cleanup
|
||||
echo -e "${YELLOW}This will permanently delete the above records.${NC}"
|
||||
read -p "Continue? (y/N) " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
||||
echo "Cleanup cancelled"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Perform cleanup
|
||||
echo ""
|
||||
echo -e "${YELLOW}Cleaning up database...${NC}"
|
||||
|
||||
# Delete invalid lots
|
||||
if [ "$INVALID_LOTS" -gt 0 ]; then
|
||||
echo -e " ${BLUE}[1/2] Deleting invalid lots...${NC}"
|
||||
sqlite3 "${LOCAL_DB_PATH}" "DELETE FROM lots WHERE auction_id IS NULL OR auction_id = '';"
|
||||
echo -e " ${GREEN}✓ Deleted ${INVALID_LOTS} invalid lots${NC}"
|
||||
fi
|
||||
|
||||
# Delete orphaned images
|
||||
if [ "$ORPHANED_IMAGES" -gt 0 ]; then
|
||||
echo -e " ${BLUE}[2/2] Deleting orphaned images...${NC}"
|
||||
sqlite3 "${LOCAL_DB_PATH}" "DELETE FROM images WHERE lot_id NOT IN (SELECT lot_id FROM lots);"
|
||||
echo -e " ${GREEN}✓ Deleted ${ORPHANED_IMAGES} orphaned images${NC}"
|
||||
fi
|
||||
|
||||
# Vacuum database to reclaim space
|
||||
echo -e " ${BLUE}[3/3] Compacting database...${NC}"
|
||||
sqlite3 "${LOCAL_DB_PATH}" "VACUUM;"
|
||||
echo -e " ${GREEN}✓ Database compacted${NC}"
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}╔════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${GREEN}║ Cleanup completed successfully ║${NC}"
|
||||
echo -e "${GREEN}╚════════════════════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
|
||||
# Show final state
|
||||
echo -e "${BLUE}Final database state:${NC}"
|
||||
sqlite3 "${LOCAL_DB_PATH}" <<EOF
|
||||
.mode box
|
||||
SELECT
|
||||
'Total lots' as metric,
|
||||
COUNT(*) as count
|
||||
FROM lots
|
||||
UNION ALL
|
||||
SELECT
|
||||
'Total images',
|
||||
COUNT(*)
|
||||
FROM images;
|
||||
EOF
|
||||
|
||||
echo ""
|
||||
DB_SIZE=$(du -h "${LOCAL_DB_PATH}" | cut -f1)
|
||||
echo -e "${BLUE}Database size: ${DB_SIZE}${NC}"
|
||||
echo ""
|
||||
@@ -69,11 +69,14 @@ sync_database() {
|
||||
ssh ${REMOTE_HOST} "docker run --rm -v ${REMOTE_VOLUME}:/data -v ${REMOTE_TMP}:${REMOTE_TMP} alpine cp /data/cache.db ${REMOTE_TMP}/cache.db"
|
||||
|
||||
echo -e "${YELLOW}[2/3] Downloading database from remote server...${NC}"
|
||||
# Create backup of existing local database
|
||||
# Create backup and remove old local database
|
||||
if [ -f "${LOCAL_DB_PATH}" ]; then
|
||||
BACKUP_PATH="${LOCAL_DB_PATH}.backup-$(date +%Y%m%d-%H%M%S)"
|
||||
echo -e "${BLUE} Backing up existing database to: ${BACKUP_PATH}${NC}"
|
||||
cp "${LOCAL_DB_PATH}" "${BACKUP_PATH}"
|
||||
|
||||
echo -e "${BLUE} Removing old local database...${NC}"
|
||||
rm -f "${LOCAL_DB_PATH}"
|
||||
fi
|
||||
|
||||
# Download new database
|
||||
@@ -98,6 +101,32 @@ UNION ALL
|
||||
SELECT 'images', COUNT(*) FROM images
|
||||
UNION ALL
|
||||
SELECT 'cache', COUNT(*) FROM cache;
|
||||
EOF
|
||||
|
||||
# Show data quality report
|
||||
echo -e "${BLUE} Data quality:${NC}"
|
||||
sqlite3 "${LOCAL_DB_PATH}" <<EOF
|
||||
.mode box
|
||||
SELECT
|
||||
'Valid lots' as metric,
|
||||
COUNT(*) as count,
|
||||
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%' as percentage
|
||||
FROM lots
|
||||
WHERE auction_id IS NOT NULL AND auction_id != ''
|
||||
UNION ALL
|
||||
SELECT
|
||||
'Invalid lots (missing auction_id)',
|
||||
COUNT(*),
|
||||
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
|
||||
FROM lots
|
||||
WHERE auction_id IS NULL OR auction_id = ''
|
||||
UNION ALL
|
||||
SELECT
|
||||
'Lots with intelligence fields',
|
||||
COUNT(*),
|
||||
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM lots), 2) || '%'
|
||||
FROM lots
|
||||
WHERE followers_count IS NOT NULL OR estimated_min IS NOT NULL;
|
||||
EOF
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,20 @@ class ImageProcessingService {
|
||||
*/
|
||||
boolean processImage(int imageId, String localPath, long lotId) {
|
||||
try {
|
||||
// Check if file exists before processing
|
||||
var file = new java.io.File(localPath);
|
||||
if (!file.exists() || !file.canRead()) {
|
||||
log.warn(" Image file not accessible: {}", localPath);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check file size (skip very large files that might cause issues)
|
||||
long fileSizeBytes = file.length();
|
||||
if (fileSizeBytes > 50 * 1024 * 1024) { // 50 MB limit
|
||||
log.warn(" Image file too large ({}MB): {}", fileSizeBytes / (1024 * 1024), localPath);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Run object detection on the local file
|
||||
var labels = detector.detectObjects(localPath);
|
||||
|
||||
|
||||
@@ -104,7 +104,16 @@ public class ObjectDetectionService {
|
||||
for (var i = 0; i < out.rows(); i++) {
|
||||
var data = out.get(i, 0);
|
||||
if (data == null) continue;
|
||||
|
||||
// The first 5 numbers are bounding box, then class scores
|
||||
// Check if data has enough elements before copying
|
||||
int expectedLength = 5 + classNames.size();
|
||||
if (data.length < expectedLength) {
|
||||
log.warn("Detection data too short: expected {} elements, got {}. Skipping detection.",
|
||||
expectedLength, data.length);
|
||||
continue;
|
||||
}
|
||||
|
||||
var scores = new double[classNames.size()];
|
||||
System.arraycopy(data, 5, scores, 0, scores.length);
|
||||
var classId = argMax(scores);
|
||||
@@ -117,6 +126,13 @@ public class ObjectDetectionService {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Release resources
|
||||
image.release();
|
||||
blob.release();
|
||||
for (var out : outs) {
|
||||
out.release();
|
||||
}
|
||||
return labels;
|
||||
}
|
||||
/**
|
||||
|
||||
@@ -94,10 +94,20 @@ public class QuarkusWorkflowScheduler {
|
||||
return;
|
||||
}
|
||||
|
||||
LOG.infof(" → Processing %d images", pendingImages.size());
|
||||
// Limit batch size to prevent thread blocking (max 100 images per run)
|
||||
final int MAX_BATCH_SIZE = 100;
|
||||
int totalPending = pendingImages.size();
|
||||
if (totalPending > MAX_BATCH_SIZE) {
|
||||
LOG.infof(" → Found %d pending images, processing first %d (batch limit)",
|
||||
totalPending, MAX_BATCH_SIZE);
|
||||
pendingImages = pendingImages.subList(0, MAX_BATCH_SIZE);
|
||||
} else {
|
||||
LOG.infof(" → Processing %d images", totalPending);
|
||||
}
|
||||
|
||||
var processed = 0;
|
||||
var detected = 0;
|
||||
var failed = 0;
|
||||
|
||||
for (var image : pendingImages) {
|
||||
try {
|
||||
@@ -121,19 +131,26 @@ public class QuarkusWorkflowScheduler {
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
failed++;
|
||||
}
|
||||
|
||||
// Rate limiting (lighter since no network I/O)
|
||||
Thread.sleep(100);
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
failed++;
|
||||
LOG.warnf(" ⚠️ Failed to process image: %s", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
var duration = System.currentTimeMillis() - start;
|
||||
LOG.infof(" ✓ Processed %d images, detected objects in %d (%.1fs)",
|
||||
processed, detected, duration / 1000.0);
|
||||
LOG.infof(" ✓ Processed %d/%d images, detected objects in %d, failed %d (%.1fs)",
|
||||
processed, totalPending, detected, failed, duration / 1000.0);
|
||||
|
||||
if (totalPending > MAX_BATCH_SIZE) {
|
||||
LOG.infof(" → %d images remaining for next run", totalPending - MAX_BATCH_SIZE);
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
LOG.errorf(e, " ❌ Image processing failed: %s", e.getMessage());
|
||||
|
||||
7
src/main/resources/META-INF/resources/favicon.svg
Normal file
7
src/main/resources/META-INF/resources/favicon.svg
Normal file
@@ -0,0 +1,7 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
|
||||
<rect width="100" height="100" fill="#2563eb" rx="15"/>
|
||||
<path d="M25 40 L50 20 L75 40 L75 70 L25 70 Z" fill="#ffffff" stroke="#ffffff" stroke-width="2"/>
|
||||
<circle cx="50" cy="45" r="8" fill="#2563eb"/>
|
||||
<rect x="40" y="55" width="20" height="3" fill="#2563eb"/>
|
||||
<text x="50" y="90" font-family="Arial" font-size="12" fill="#ffffff" text-anchor="middle" font-weight="bold">AUCTION</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 465 B |
@@ -1355,11 +1355,32 @@ function updatePagination(total, perPage, currentPage) {
|
||||
|
||||
// Trigger workflow with progress tracking
|
||||
async function triggerWorkflow(workflow) {
|
||||
const statusId = `status-${workflow.split('-')[0]}`;
|
||||
const progressId = `progress-${workflow.split('-')[0]}`;
|
||||
// Map workflow names to DOM element IDs
|
||||
const idMap = {
|
||||
'scraper-import': 'scraper',
|
||||
'image-processing': 'images',
|
||||
'bid-monitoring': 'bids',
|
||||
'closing-alerts': 'alerts'
|
||||
};
|
||||
|
||||
const elementId = idMap[workflow];
|
||||
if (!elementId) {
|
||||
console.error('Unknown workflow:', workflow);
|
||||
showToast(`Unknown workflow: ${workflow}`, 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
const statusId = `status-${elementId}`;
|
||||
const progressId = `progress-${elementId}`;
|
||||
const statusEl = document.getElementById(statusId);
|
||||
const progressEl = document.getElementById(progressId);
|
||||
|
||||
if (!statusEl || !progressEl) {
|
||||
console.error('Workflow UI elements not found:', statusId, progressId);
|
||||
showToast(`Cannot trigger workflow: UI elements not found`, 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Update UI
|
||||
statusEl.innerHTML = '<span class="inline-block w-2 h-2 bg-blue-500 rounded-full mr-1 animate-pulse"></span>Running';
|
||||
statusEl.className = 'text-xs font-semibold px-2 py-1 rounded-full bg-blue-100 text-blue-800';
|
||||
|
||||
Reference in New Issue
Block a user