GraphQL integrate, data correctness
This commit is contained in:
43
test_live_lot.py
Normal file
43
test_live_lot.py
Normal file
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test scraping a single live lot page"""
|
||||
import asyncio
|
||||
import sys
|
||||
sys.path.insert(0, 'src')
|
||||
|
||||
from scraper import TroostwijkScraper
|
||||
|
||||
async def main():
|
||||
scraper = TroostwijkScraper()
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(headless=True)
|
||||
page = await browser.new_page()
|
||||
|
||||
# Get a lot URL from the database
|
||||
import sqlite3
|
||||
conn = sqlite3.connect('/mnt/okcomputer/output/cache.db')
|
||||
cursor = conn.execute("SELECT url FROM lots LIMIT 1")
|
||||
row = cursor.fetchone()
|
||||
conn.close()
|
||||
|
||||
if not row:
|
||||
print("No lots in database")
|
||||
return
|
||||
|
||||
lot_url = row[0]
|
||||
print(f"Fetching: {lot_url}\n")
|
||||
|
||||
result = await scraper.crawl_page(page, lot_url)
|
||||
|
||||
if result:
|
||||
print(f"\nExtracted Data:")
|
||||
print(f" current_bid: {result.get('current_bid')}")
|
||||
print(f" bid_count: {result.get('bid_count')}")
|
||||
print(f" closing_time: {result.get('closing_time')}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user