44 lines
1.2 KiB
Python
44 lines
1.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Test scraping a single live lot page"""
|
|
import asyncio
|
|
import sys
|
|
sys.path.insert(0, 'src')
|
|
|
|
from scraper import TroostwijkScraper
|
|
|
|
async def main():
|
|
scraper = TroostwijkScraper()
|
|
|
|
from playwright.async_api import async_playwright
|
|
|
|
async with async_playwright() as p:
|
|
browser = await p.chromium.launch(headless=True)
|
|
page = await browser.new_page()
|
|
|
|
# Get a lot URL from the database
|
|
import sqlite3
|
|
conn = sqlite3.connect('/mnt/okcomputer/output/cache.db')
|
|
cursor = conn.execute("SELECT url FROM lots LIMIT 1")
|
|
row = cursor.fetchone()
|
|
conn.close()
|
|
|
|
if not row:
|
|
print("No lots in database")
|
|
return
|
|
|
|
lot_url = row[0]
|
|
print(f"Fetching: {lot_url}\n")
|
|
|
|
result = await scraper.crawl_page(page, lot_url)
|
|
|
|
if result:
|
|
print(f"\nExtracted Data:")
|
|
print(f" current_bid: {result.get('current_bid')}")
|
|
print(f" bid_count: {result.get('bid_count')}")
|
|
print(f" closing_time: {result.get('closing_time')}")
|
|
|
|
await browser.close()
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|