Tour/aupi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added error handling on ovm response

Browse Source
This commit is contained in:
Computerboer
2024-01-07 00:23:18 +01:00
parent 50024cf1ca
commit bbd20f7915
5 changed files with 52 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Auctionviewer.api.pyproj
View File

@@ -31,6 +31,7 @@
<Compile Include="app.py" /> <Compile Include="app.py" />
<Compile Include="utils\auctionutils.py" /> <Compile Include="utils\auctionutils.py" />
<Compile Include="cache.py" /> <Compile Include="cache.py" />
<Compile Include="utils\helperutils.py" />
<Compile Include="utils\locationutils.py" /> <Compile Include="utils\locationutils.py" />
<Compile Include="models\location.py" /> <Compile Include="models\location.py" />
</ItemGroup> </ItemGroup>

7
app.py
View File

@@ -1,3 +1,4 @@
from utils.helperutils import log
from traceback import print_exc from traceback import print_exc
from flask import Flask, jsonify from flask import Flask, jsonify
from flask_cors import CORS, cross_origin from flask_cors import CORS, cross_origin
@@ -19,16 +20,16 @@ def gethome():
def getAllAuctions(countrycode): def getAllAuctions(countrycode):
try: try:
if countrycode not in ['NL', 'BE', 'DE']: if countrycode not in ['NL', 'BE', 'DE']:
print('country not available: ' + countrycode) log('country not available: ' + countrycode)
return jsonify('NOT AVAILABLE COUNTRY') return jsonify('NOT AVAILABLE COUNTRY')
log('incoming api request')
res = getAuctionlocations(countrycode) res = getAuctionlocations(countrycode)
#return json.dumps(res, sort_keys=True, default=str) #return json.dumps(res, sort_keys=True, default=str)
return JsonEncoder().encode(res) return JsonEncoder().encode(res)
except Exception as e: except Exception as e:
print('something went wrong ') log('something went wrong ')
print_exc(e) print_exc(e)
return 'internal server error', 500 return 'internal server error', 500

17
cache.py
View File

@@ -5,6 +5,7 @@ import time
import json import json
from models.location import JsonEncoder from models.location import JsonEncoder
from utils.helperutils import log
cache = {} cache = {}
@@ -16,14 +17,14 @@ class Cache():
if(not cache): if(not cache):
return None return None
if(cacheobj.isOlderThanHours(notOlderThanHours)): if(cacheobj.isOlderThanHours(notOlderThanHours)):
print('removing cacheobject ' + key) log('removing cacheobject ' + key)
del cache[key] del cache[key]
return None return None
print(str(datetime.now()) + ' returning cacheobject ' + key) log( 'returning cacheobject ' + key)
return cacheobj.obj return cacheobj.obj
def add(key, obj): def add(key, obj):
print(str(datetime.now()) + ' adding cacheobject ' + key) log('adding cacheobject ' + key)
cacheobj = CacheObj(key, obj) cacheobj = CacheObj(key, obj)
cache[key] = cacheobj cache[key] = cacheobj
@@ -35,7 +36,7 @@ class CacheObj:
self.time=datetime.now() self.time=datetime.now()
def isOlderThanHours(self, hours): def isOlderThanHours(self, hours):
print('checking time cacheobject ' + self.key + ': ' + str(self.time) + " < " + str(datetime.now() - timedelta(hours=hours))) log('checking time cacheobject ' + self.key + ': ' + str(self.time) + " < " + str(datetime.now() - timedelta(hours=hours)))
return self.time < datetime.now() - timedelta(hours=hours) return self.time < datetime.now() - timedelta(hours=hours)
@@ -47,21 +48,21 @@ class FileCache():
if cachefile.is_file(): if cachefile.is_file():
ti_m = os.path.getmtime(filepath) ti_m = os.path.getmtime(filepath)
#checks last modified age of file, and removes it if it is too old #checks last modified age of file, and removes it if it is too old
print('checking time cachefile ' + filepath + ': ' + str(ti_m) + " < " + str(time.time() - (3600 * notOlderThanHours))) log('checking time cachefile ' + filepath + ': ' + str(ti_m) + " < " + str(time.time() - (3600 * notOlderThanHours)))
if(ti_m < time.time() - (3600 * notOlderThanHours)): if(ti_m < time.time() - (3600 * notOlderThanHours)):
print() log('removing old filecache')
os.remove(filepath); os.remove(filepath);
return None; return None;
with open(filepath) as json_file: with open(filepath) as json_file:
json_data = json.load(json_file); json_data = json.load(json_file);
print('returning json data from cachefile: ' + key) log('returning json data from cachefile: ' + key)
return json_data; return json_data;
return None return None
def add(key, obj): def add(key, obj):
print(str(datetime.now()) + ' adding filecacheobject ' + key); log('adding filecacheobject ' + key);
json_data = JsonEncoder().encode(obj) json_data = JsonEncoder().encode(obj)
with open("./filecache/" + key + ".json", 'w') as f: with open("./filecache/" + key + ".json", 'w') as f:
f.write(json_data) f.write(json_data)

53
utils/auctionutils.py
View File

@@ -1,8 +1,9 @@
from distutils.command import build import imp
import requests import requests
from cache import Cache, FileCache from cache import Cache, FileCache
from models.location import Auction, Auctionbrand, Countrycode, Maplocation from models.location import Auction, Auctionbrand, Countrycode, Maplocation, JsonEncoder
from utils.locationutils import getGeoLocationByCity from utils.locationutils import getGeoLocationByCity
from utils.helperutils import log
from datetime import datetime from datetime import datetime
import re import re
import math import math
@@ -60,8 +61,8 @@ def getTWKUrl():
if(response.status_code ==200): if(response.status_code ==200):
buildid = re.search(r'"buildId":"([^"]*)', response.text, re.MULTILINE ) buildid = re.search(r'"buildId":"([^"]*)', response.text, re.MULTILINE )
twkDataUrl = 'https://www.troostwijkauctions.com/_next/data/' + str(buildid[1]) + '/nl/' twkDataUrl = 'https://www.troostwijkauctions.com/_next/data/' + str(buildid[1]) + '/nl/'
print('buildid: ' + str(buildid[1])) log('buildid: ' + str(buildid[1]))
print('twkDataUrl: ' + twkDataUrl) log('twkDataUrl: ' + twkDataUrl)
return twkDataUrl return twkDataUrl
return None return None
@@ -82,7 +83,7 @@ def getTwkAuctions(countrycode):
response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode) response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode)
if(response.status_code ==200): if(response.status_code ==200):
print('Got Twk Auctions') log('Got Twk Auctions')
data = response.json(); data = response.json();
auctions = []; auctions = [];
@@ -91,7 +92,7 @@ def getTwkAuctions(countrycode):
# for result in data['pageProps']['auctionList']: # for result in data['pageProps']['auctionList']:
for i in range(1,pages,1): for i in range(1,pages,1):
print("getting page " + str(i) + ' of ' + str(pages)) log("getting page " + str(i) + ' of ' + str(pages))
if(i > 1): if(i > 1):
response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode + "&page=" + str(i)); response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode + "&page=" + str(i));
data = response.json(); data = response.json();
@@ -108,7 +109,7 @@ def getTwkAuctions(countrycode):
def getTWKAuction(twkDataUrl, auctionurlslug): def getTWKAuction(twkDataUrl, auctionurlslug):
response = requests.get(twkDataUrl + "a/" + auctionurlslug + '.json') response = requests.get(twkDataUrl + "a/" + auctionurlslug + '.json')
if(response.status_code ==200): if(response.status_code == 200):
data = response.json(); data = response.json();
if(len(data['pageProps']['lots']['results']) ==0): if(len(data['pageProps']['lots']['results']) ==0):
return None; return None;
@@ -130,19 +131,31 @@ def getOVMAuctions():
if(res): if(res):
return res return res
response = requests.get("https://onlineveilingmeester.nl/rest/nl/veilingen?status=open&domein=ONLINEVEILINGMEESTER") try:
response = requests.get("https://onlineveilingmeester.nl/rest/nl/veilingen?status=open&domein=ONLINEVEILINGMEESTER")
except:
log("The OVM auctions call threw a error")
if(response is None):
return None
if(response.status_code ==200): if(response.status_code ==200):
print('Got Ovm Auctions') log('Got Ovm Auctions')
data = response.json() try:
auctions = [] data = response.json()
for result in data['veilingen']: auctions = []
cityname ="Nederland" if result['isBezorgVeiling'] else result['afgifteAdres']['plaats'] for result in data['veilingen']:
cityname = "Nederland" if cityname is None else cityname #there can be auctions where you have to make an appointment to retrieve the lots cityname ="Nederland" if result['isBezorgVeiling'] else result['afgifteAdres']['plaats']
startdatetime = result['openingsDatumISO'].replace("T", " ").replace("Z", "") cityname = "Nederland" if cityname is None else cityname #there can be auctions where you have to make an appointment to retrieve the lots
enddatetime = result['sluitingsDatumISO'].replace("T", " ").replace("Z", "") startdatetime = result['openingsDatumISO'].replace("T", " ").replace("Z", "")
a = Auction(Auctionbrand.OVM, cityname,result['land'], result['naam'],startdatetime, enddatetime, str(result['land']).lower() + '/veilingen/' + str(result['id']) + '/kavels', 'images/150x150/' + str(result['id']) + '/' + result['image'], result['totaalKavels'] ) enddatetime = result['sluitingsDatumISO'].replace("T", " ").replace("Z", "")
auctions.append(a) a = Auction(Auctionbrand.OVM, cityname,result['land'], result['naam'],startdatetime, enddatetime, str(result['land']).lower() + '/veilingen/' + str(result['id']) + '/kavels', 'images/150x150/' + str(result['id']) + '/' + result['image'], result['totaalKavels'] )
Cache.add(cachename, auctions) auctions.append(a)
return auctions Cache.add(cachename, auctions)
return auctions
except:
log('Something went wrong in the mapping of OVM auctions to auctionviewer objects. The reason was: ' + response.reason + '. The response was: ' + JsonEncoder().encode(response.json()))
else:
log("The OVM auctions call didn't gave a 200 response but a " + str(response.status_code) + ". With the reason: " + response.reason)
return None return None

5
utils/helperutils.py Normal file
View File

@@ -0,0 +1,5 @@
# from datetime import datetime, timedelta
def log(value):
# print(str(datetime.now()) + ' ' + str(value))
print( str(value))