diff --git a/README.md b/README.md index 920b88d..a816ab6 100644 --- a/README.md +++ b/README.md @@ -1 +1,4 @@ -# PythonAuctionviewer \ No newline at end of file +# PythonAuctionviewer + +## Run dev server +To run the development server run `python app.py` in the root of the project \ No newline at end of file diff --git a/cache.py b/cache.py index 8643b08..630f757 100644 --- a/cache.py +++ b/cache.py @@ -67,5 +67,6 @@ class FileCache(): def add(key, obj): log('adding filecacheobject ' + key) json_data = JsonEncoder().encode(obj) + # json_data = json.dumps(obj, cls=JsonEncoder, indent=2) with open("./filecache/" + key + ".json", 'w+') as f: f.write(json_data) \ No newline at end of file diff --git a/data/locationfiles/NL.txt b/data/locationfiles/NL.txt index 2304736..cc2e0c5 100644 --- a/data/locationfiles/NL.txt +++ b/data/locationfiles/NL.txt @@ -465,6 +465,7 @@ 2743947 Zevenbergen Zevenbergen Zevenbergen,ze fen bei heng,zefenberugen,Зевенберген,ゼーフェンベルゲン,澤芬貝亨 51.645 4.60417 P PPL NL 06 1709 0 5 Europe/Amsterdam 2017-10-17 2743948 Gemeente Zevenaar Gemeente Zevenaar Gemeente Zevenaar,Zevenaar 51.95376 6.07727 A ADM2 NL 03 0299 44096 9 Europe/Amsterdam 2023-01-31 2743949 Zevenaar Zevenaar Zevenar,Зевенар 51.93 6.07083 P PPL NL 03 0299 26063 12 Europe/Amsterdam 2017-10-17 +2743949 omgeving zevenaar omgeving zevenaar Zevenar,Зевенар 51.93 6.07083 P PPL NL 03 0299 26063 12 Europe/Amsterdam 2017-10-17 2743950 Zeumersche Beek Zeumersche Beek 52.16452 5.53953 H STM NL 03 0 5 Europe/Amsterdam 2011-04-19 2743951 Zeumeren Zeumeren 52.17256 5.60208 P PPL NL 03 0203 0 12 Europe/Amsterdam 2009-01-17 2743952 Zetten Zetten 51.92833 5.71389 P PPL NL 03 1734 2985 10 Europe/Amsterdam 2017-10-17 @@ -13230,6 +13231,7 @@ 2757247 Den Bramel Den Bramel 52.12176 6.3111 S CSTL NL 03 1876 0 15 Europe/Amsterdam 2011-06-04 2757248 Den Braam Den Braam 52.1375 6.84028 P PPL NL 15 0158 0 38 Europe/Amsterdam 2007-06-03 2757251 Den Bosch Den Bosch 52.07795 6.13621 S EST NL 03 0213 0 13 Europe/Amsterdam 2011-06-04 +2757251 's hertogenbosch 's hertogenbosch 52.07795 6.13621 S EST NL 03 0213 0 13 Europe/Amsterdam 2011-06-04 2757252 Polder Den Bommel Polder Den Bommel 51.70597 4.28369 T PLDR NL 11 0 -2 Europe/Amsterdam 2011-04-19 2757253 Den Bommel Den Bommel Bommel 51.71583 4.28472 P PPL NL 11 1924 0 -1 Europe/Amsterdam 2017-10-17 2757254 Polder Den Bol Polder Den Bol 51.71665 4.75829 T PLDR NL 06 0 -1 Europe/Amsterdam 2011-04-19 diff --git a/models/location.py b/models/location.py index c571a89..bf38ac3 100644 --- a/models/location.py +++ b/models/location.py @@ -1,5 +1,7 @@ from enum import Enum +import enum from json import JSONEncoder +import json class Countrycode(Enum): NL = "NL", @@ -10,6 +12,7 @@ class Auctionbrand(str, Enum): NONE = "NONE", TWK = "TWK" OVM = "OVM" + AP = "AP" class GeonameLocation: @@ -32,7 +35,7 @@ class Maplocation: self.auctions = auctions class Auction: - def __init__(self, auctionbrand: Auctionbrand = Auctionbrand.NONE, city = "", countrycode:Countrycode = Countrycode.NL, name = "", starttime = None, closingtime = None, url = "", imageurl = "", numberoflots = 0, geonamelocation: GeonameLocation = None): + def __init__(self, auctionbrand: Auctionbrand = Auctionbrand.NONE, city = "", countrycode:Countrycode = Countrycode.NL, name = "", starttime = None, closingtime = None, url = "", imageurl = "", numberoflots = 0, geonamelocation: GeonameLocation = None, multiplelocations = False): self.city = city self.countrycode = countrycode self.name = name @@ -43,7 +46,34 @@ class Auction: self.numberoflots = numberoflots self.geonamelocation = geonamelocation self.brand = auctionbrand + self.multiplelocations = multiplelocations class JsonEncoder(JSONEncoder): - def default(self, o): - return o.__dict__ \ No newline at end of file + # def default(self, o): + # return o.__dict__ + #try 2 + def default(self, obj): + # Only serialize public, instance-level attributes + if hasattr(obj, '__dict__'): + return { + key: self.serialize(value) + for key, value in obj.__dict__.items() + if not key.startswith('_') # skip private/protected + } + return super().default(obj) + + def serialize(self, value): + if isinstance(value, list): + return [self.serialize(item) for item in value] + elif isinstance(value, dict): + return {k: self.serialize(v) for k, v in value.items()} + elif isinstance(value, enum.Enum): + return value.name # or value.value + elif hasattr(value, '__dict__'): + return self.default(value) # dive into nested object + else: + try: + json.dumps(value) + return value + except (TypeError, OverflowError): + return str(value) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index fa4b5db..60823e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -pip==9.0.1 -setuptools==28.8.0 -flask===3.0.3 -werkzeug==3.0.0 -flask_cors===5.0.0 -requests===2.27.1 +pip>=23.3 +setuptools>=78.1.1 +flask===3.1.1 +werkzeug>=3.0.6 +flask_cors>=6.0.0 +requests>=2.32.4 pathlib===1.0.1 \ No newline at end of file diff --git a/utils/APutils.py b/utils/APutils.py new file mode 100644 index 0000000..bfdecb8 --- /dev/null +++ b/utils/APutils.py @@ -0,0 +1,77 @@ + + +from datetime import datetime +import json +import re +from time import ctime +from traceback import print_exc +import requests +from cache import Cache +from models.location import Auction, Auctionbrand, Countrycode, JsonEncoder +from utils.helperutils import log + + +def getAPAuctions(): + cachename = 'AuctionPort_' + res = Cache.get(cachename) + if(res): + return res + + try: + response = requests.get("https://api.auctionport.be/auctions/small?size=100&page=1") + except: + log("The Auctionport auctions call threw a error") + + if(response is None): + return [] + + if(response.status_code ==200): + log('Got AP Auctions') + try: + data = response.json() + pages = data['pages'] + auctions = [] + for i in range(0,pages-1,1): + log("getting page " + str(i) + ' of ' + str(pages)) + # if(i > 1): + response = requests.get("https://api.auctionport.be/auctions/small?size=100&page=" + str(i)) + if(response is None): continue + if(response.status_code != 200): continue + + data = response.json() + + for PAauction in data['data']: + #makes sure that the locations array is filled with at least one location + if PAauction['locations'] == []: + PAauction['locations'] = [PAauction['location']] + + #makes sure that the auction is still active + closingdate = datetime.fromisoformat(PAauction['closingDate']) + if(closingdate.date() < datetime.now().date() ): continue + if(PAauction['lotCount'] <= 0): continue + + multipleLocations = len(PAauction['locations']) > 1 + + for location in PAauction['locations']: + if not location.endswith('Nederland'): continue + loc = re.sub('Nederland', '', location) + # loc = location.split(",") + postalcodeRegex = r'(.*?)[1-9][0-9]{3} ?(?!sa|sd|ss)[a-zA-Z]{2}' + city = re.sub(postalcodeRegex , '', loc) #removes postalcode and everything in front of it + city = city.strip() #removes whitespace + city = city.strip(',') #removes trailing and leading , + city = city.split(',') #splits on , to overcome not matching regex + city = city[len(city)-1] + city = city.strip() + newauction = Auction(Auctionbrand.AP,city, Countrycode.NL, PAauction['title'], datetime.fromisoformat(PAauction['openDate']), datetime.fromisoformat(PAauction['closingDate']), '/auction/'+ str(PAauction['id']), PAauction['imageUrl'], PAauction['lotCount'] , None, multipleLocations) + + auctions.append(newauction) + Cache.add(cachename, auctions) + return auctions + except Exception as e: + log(e.__cause__ + '-- Something went wrong in the mapping of AP auctions to auctionviewer objects. The reason was: ' + response.reason + '. The response was: ' + JsonEncoder().encode(response.json())) + print_exc(e) + + else: + log("The AP auctions call didn't gave a 200 response but a " + str(response.status_code) + ". With the reason: " + response.reason) + return [] \ No newline at end of file diff --git a/utils/OVMutils.py b/utils/OVMutils.py new file mode 100644 index 0000000..1f00e60 --- /dev/null +++ b/utils/OVMutils.py @@ -0,0 +1,52 @@ +from traceback import print_exc +import requests +from cache import Cache +from models.location import Auction, Auctionbrand, JsonEncoder +from utils.helperutils import log + + +def getOVMAuctions(): + cachename = 'OnlineVeiling_' + res = Cache.get(cachename) + if(res): + return res + + try: + response = requests.get("https://onlineveilingmeester.nl/rest/nl/veilingen?status=open&domein=ONLINEVEILINGMEESTER") + except: + log("The OVM auctions call threw a error") + + if(response is None): + return [] + + if(response.status_code ==200): + log('Got Ovm Auctions') + try: + data = response.json() + auctions = [] + for result in data['veilingen']: + cityname ="Nederland" if result['isBezorgVeiling'] else result['afgifteAdres']['plaats'] + cityname = "Nederland" if cityname is None else cityname #there can be auctions where you have to make an appointment to retrieve the lots + startdatetime = result['openingsDatumISO'].replace("T", " ").replace("Z", "") + enddatetime = result['sluitingsDatumISO'].replace("T", " ").replace("Z", "") + image = "" + #if hasattr(result, 'image') : #result['image'] : + image = result.get('image', "") #['image'] + if image == "": + images = result.get('imageList') + if(len(images) >0): + image = images[0] + else: + log("No image found for OVM auction: " + result['naam']) + + a = Auction(Auctionbrand.OVM, cityname,result['land'], result['naam'],startdatetime, enddatetime, str(result['land']).lower() + '/veilingen/' + str(result['id']) + '/kavels', 'images/150x150/' + image, result['totaalKavels'] ) + auctions.append(a) + Cache.add(cachename, auctions) + return auctions + except Exception as e: + log(e.__cause__ + '-- Something went wrong in the mapping of OVM auctions to auctionviewer objects. The reason was: ' + response.reason + '. The response was: ' + JsonEncoder().encode(response.json())) + print_exc(e) + + else: + log("The OVM auctions call didn't gave a 200 response but a " + str(response.status_code) + ". With the reason: " + response.reason) + return [] \ No newline at end of file diff --git a/utils/TWKutils.py b/utils/TWKutils.py new file mode 100644 index 0000000..7f3ef9d --- /dev/null +++ b/utils/TWKutils.py @@ -0,0 +1,77 @@ +from datetime import datetime +import math +import re +import requests + +from cache import Cache +from models.location import Auction, Auctionbrand +from utils.helperutils import log + + +def getTWKUrl(): + response = requests.get('https://www.troostwijkauctions.com/') + if(response.status_code ==200): + buildid = re.search(r'"buildId":"([^"]*)', response.text, re.MULTILINE ) + twkDataUrl = 'https://www.troostwijkauctions.com/_next/data/' + str(buildid[1]) + '/nl/' + log('buildid: ' + str(buildid[1])) + log('twkDataUrl: ' + twkDataUrl) + return twkDataUrl + + return None + + +def getTwkAuctions(countrycode): + cachename = 'TwkAuctions_'+ countrycode + res = Cache.get(cachename) + if(res): + return res + + # buildidresponse = requests.get('https://www.troostwijkauctions.com/') + twkDataUrl = getTWKUrl() + + if(twkDataUrl is None): + return [] + + response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode) + + if(response.status_code ==200): + log('Got Twk Auctions') + data = response.json() + auctions = [] + + totalAuctionCount = data['pageProps']['totalSize']; + pages = math.ceil(totalAuctionCount / data['pageProps']['pageSize']) + # for result in data['pageProps']['auctionList']: + + for i in range(1,pages,1): + log("getting page " + str(i) + ' of ' + str(pages)) + if(i > 1): + response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode + "&page=" + str(i)) + data = response.json() + + for twka in data['pageProps']['listData']: + # print(twka['urlSlug']) + auction = getTWKAuction(twkDataUrl, twka['urlSlug']) + if(auction): + auctions.append(auction) + Cache.add(cachename, auctions) + + return auctions + return [] + +def getTWKAuction(twkDataUrl, auctionurlslug): + log("getting TWK auctiondetails:" + twkDataUrl + "a/" + auctionurlslug + ".json") + response = requests.get(twkDataUrl + "a/" + auctionurlslug + '.json') + if(response.status_code == 200): + data = response.json() + if(len(data['pageProps']['lots']['results']) ==0): + return None + + twka = data['pageProps']['auction'] + firstlot = data['pageProps']['lots']['results'][0] + city = "Nederland" if firstlot['location']['city'].lower() == 'online' or firstlot['location']['city'].lower() == "free delivery" else firstlot['location']['city'] + a = Auction(Auctionbrand.TWK, city, firstlot['location']['countryCode'].upper(), twka['name'], datetime.fromtimestamp(twka['startDate']), datetime.fromtimestamp(twka['minEndDate']), '/a/' + auctionurlslug, twka['image']['url'], twka['lotCount'] ) + # print(a); + return a + + return None \ No newline at end of file diff --git a/utils/auctionutils.py b/utils/auctionutils.py index 699eee5..98da347 100644 --- a/utils/auctionutils.py +++ b/utils/auctionutils.py @@ -2,11 +2,12 @@ import requests from traceback import print_exc from cache import Cache, FileCache from models.location import Auction, Auctionbrand, Countrycode, Maplocation, JsonEncoder +from utils.APutils import getAPAuctions +from utils.OVMutils import getOVMAuctions +from utils.TWKutils import getTwkAuctions from utils.locationutils import getGeoLocationByCity from utils.helperutils import log from datetime import datetime -import re -import math def getAuctionlocations(countrycode: Countrycode, clearcache:bool = False): cachename = 'allauctions_' + countrycode @@ -23,6 +24,7 @@ def getAuctionlocations(countrycode: Countrycode, clearcache:bool = False): twkauctions = [] ovmauctions = [] + apauctions = [] try: twkauctions = getTwkAuctions(countrycode) @@ -36,8 +38,14 @@ def getAuctionlocations(countrycode: Countrycode, clearcache:bool = False): log('something went wrong while running the OVM auctions request') print_exc(e) + try: + apauctions = getAPAuctions() + except Exception as e: + log('something went wrong while running the OVM auctions request') + print_exc(e) - auctions = [*twkauctions, *ovmauctions] + + auctions = [*twkauctions, *ovmauctions, *apauctions] for auction in auctions: auction.geonamelocation = getGeoLocationByCity(auction.city, countrycode) @@ -71,119 +79,6 @@ def get_geonameid(auction): return auction.geonamelocation.geonameid return None -# global twkDataUrl; # = ''; # 'https://www.troostwijkauctions.com/_next/data/' #e6-N0pLHv12LVGS0oYzx6/nl/' -def getTWKUrl(): - response = requests.get('https://www.troostwijkauctions.com/') - if(response.status_code ==200): - buildid = re.search(r'"buildId":"([^"]*)', response.text, re.MULTILINE ) - twkDataUrl = 'https://www.troostwijkauctions.com/_next/data/' + str(buildid[1]) + '/nl/' - log('buildid: ' + str(buildid[1])) - log('twkDataUrl: ' + twkDataUrl) - return twkDataUrl - return None - - -def getTwkAuctions(countrycode): - cachename = 'TwkAuctions_'+ countrycode - res = Cache.get(cachename) - if(res): - return res - - # buildidresponse = requests.get('https://www.troostwijkauctions.com/') - twkDataUrl = getTWKUrl() - - if(twkDataUrl is None): - return [] - - response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode) - - if(response.status_code ==200): - log('Got Twk Auctions') - data = response.json() - auctions = [] - - totalAuctionCount = data['pageProps']['totalSize']; - pages = math.ceil(totalAuctionCount / data['pageProps']['pageSize']) - # for result in data['pageProps']['auctionList']: - - for i in range(1,pages,1): - log("getting page " + str(i) + ' of ' + str(pages)) - if(i > 1): - response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode + "&page=" + str(i)); - data = response.json() - - for twka in data['pageProps']['listData']: - # print(twka['urlSlug']) - auction = getTWKAuction(twkDataUrl, twka['urlSlug']) - if(auction): - auctions.append(auction) - Cache.add(cachename, auctions) - - return auctions - return [] - -def getTWKAuction(twkDataUrl, auctionurlslug): - log("getting TWK auctiondetails:" + twkDataUrl + "a/" + auctionurlslug + ".json") - response = requests.get(twkDataUrl + "a/" + auctionurlslug + '.json') - if(response.status_code == 200): - data = response.json() - if(len(data['pageProps']['lots']['results']) ==0): - return None - - twka = data['pageProps']['auction'] - firstlot = data['pageProps']['lots']['results'][0] - city = "Nederland" if firstlot['location']['city'].lower() == 'online' or firstlot['location']['city'].lower() == "free delivery" else firstlot['location']['city'] - a = Auction(Auctionbrand.TWK, city, firstlot['location']['countryCode'].upper(), twka['name'], datetime.fromtimestamp(twka['startDate']), datetime.fromtimestamp(twka['minEndDate']), '/a/' + auctionurlslug, twka['image']['url'], twka['lotCount'] ) - # print(a); - return a - - return None - -def getOVMAuctions(): - cachename = 'OnlineVeiling_' - res = Cache.get(cachename) - if(res): - return res - - try: - response = requests.get("https://onlineveilingmeester.nl/rest/nl/veilingen?status=open&domein=ONLINEVEILINGMEESTER") - except: - log("The OVM auctions call threw a error") - - if(response is None): - return [] - - if(response.status_code ==200): - log('Got Ovm Auctions') - try: - data = response.json() - auctions = [] - for result in data['veilingen']: - cityname ="Nederland" if result['isBezorgVeiling'] else result['afgifteAdres']['plaats'] - cityname = "Nederland" if cityname is None else cityname #there can be auctions where you have to make an appointment to retrieve the lots - startdatetime = result['openingsDatumISO'].replace("T", " ").replace("Z", "") - enddatetime = result['sluitingsDatumISO'].replace("T", " ").replace("Z", "") - image = "" - #if hasattr(result, 'image') : #result['image'] : - image = result.get('image', "") #['image'] - if image == "": - images = result.get('imageList') - if(len(images) >0): - image = images[0] - else: - log("No image found for OVM auction: " + result['naam']) - - a = Auction(Auctionbrand.OVM, cityname,result['land'], result['naam'],startdatetime, enddatetime, str(result['land']).lower() + '/veilingen/' + str(result['id']) + '/kavels', 'images/150x150/' + image, result['totaalKavels'] ) - auctions.append(a) - Cache.add(cachename, auctions) - return auctions - except Exception as e: - log(e.__cause__ + '-- Something went wrong in the mapping of OVM auctions to auctionviewer objects. The reason was: ' + response.reason + '. The response was: ' + JsonEncoder().encode(response.json())) - print_exc(e) - - else: - log("The OVM auctions call didn't gave a 200 response but a " + str(response.status_code) + ". With the reason: " + response.reason) - return [] \ No newline at end of file