26 lines
828 B
Python
26 lines
828 B
Python
#!/usr/bin/env python3
|
|
"""
|
|
Configuration module for Scaev Auctions Scraper
|
|
"""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Require Python 3.10+
|
|
if sys.version_info < (3, 10):
|
|
print("ERROR: This script requires Python 3.10 or higher")
|
|
print(f"Current version: {sys.version}")
|
|
sys.exit(1)
|
|
|
|
# ==================== CONFIGURATION ====================
|
|
BASE_URL = "https://www.troostwijkauctions.com"
|
|
CACHE_DB = "/mnt/okcomputer/output/cache.db"
|
|
OUTPUT_DIR = "/mnt/okcomputer/output"
|
|
IMAGES_DIR = "/mnt/okcomputer/output/images"
|
|
RATE_LIMIT_SECONDS = 0.5 # EXACTLY 0.5 seconds between requests
|
|
MAX_PAGES = 50 # Number of listing pages to crawl
|
|
DOWNLOAD_IMAGES = False # Set to True to download images
|
|
|
|
# Setup directories
|
|
Path(OUTPUT_DIR).mkdir(parents=True, exist_ok=True)
|
|
Path(IMAGES_DIR).mkdir(parents=True, exist_ok=True) |