from distutils.command import build import requests from cache import Cache, FileCache from models.location import Auction, Auctionbrand, Countrycode, Maplocation from utils.locationutils import getGeoLocationByCity from datetime import datetime import re import math def getAuctionlocations(countrycode: Countrycode): cachename = 'allauctions_' + countrycode res = FileCache.get(cachename, 23) # res = Cache.get(cachename) if(res): return res twkauctions = getTwkAuctions(countrycode) ovmauctions = getOVMAuctions() auctions = [*twkauctions, *ovmauctions] for auction in auctions: auction.geonamelocation = getGeoLocationByCity(auction.city, countrycode) geonameids = map(get_geonameid, auctions) uniquegeonameids = (list(set(geonameids))) maplocations = [] #loops through the uniques geonameids for geoid in uniquegeonameids: #filters all auctions for this geonameid geoauctions = list(filter(lambda a: get_geonameid(a) == geoid , auctions)) if(geoauctions): #gets the location (if it has any) for the geolocation location = geoauctions[0].geonamelocation if(location): maplocation = Maplocation(location.latitude, location.longitude, len(geoauctions), location, geoauctions) maplocations.append(maplocation); for location in maplocations: del location.geonamelocation #removes object which is not used anymore for auction in location.auctions: del auction.geonamelocation #removes object to not have duplicate data send to the server FileCache.add(cachename, maplocations) return maplocations def get_geonameid(auction): if(auction.geonamelocation): return auction.geonamelocation.geonameid return None # global twkDataUrl; # = ''; # 'https://www.troostwijkauctions.com/_next/data/' #e6-N0pLHv12LVGS0oYzx6/nl/' def getTWKUrl(): response = requests.get('https://www.troostwijkauctions.com/') if(response.status_code ==200): buildid = re.search(r'"buildId":"([^"]*)', response.text, re.MULTILINE ) twkDataUrl = 'https://www.troostwijkauctions.com/_next/data/' + str(buildid[1]) + '/nl/' print('buildid: ' + str(buildid[1])) print('twkDataUrl: ' + twkDataUrl) return twkDataUrl return None def getTwkAuctions(countrycode): cachename = 'TwkAuctions_'+ countrycode res = Cache.get(cachename) if(res): return res # buildidresponse = requests.get('https://www.troostwijkauctions.com/') twkDataUrl = getTWKUrl(); if(twkDataUrl is None): return []; response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode) if(response.status_code ==200): print('Got Twk Auctions') data = response.json(); auctions = []; totalAuctionCount = data['pageProps']['auctionList']['totalSize']; pages = math.ceil(totalAuctionCount / len(data['pageProps']['auctionList']['results'])) # for result in data['pageProps']['auctionList']: for i in range(1,pages,1): print("getting page " + str(i) + ' of ' + str(pages)) if(i > 1): response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode + "&page=" + str(i)); data = response.json(); for twka in data['pageProps']['auctionList']['results']: # print(twka['urlSlug']) auction = getTWKAuction(twkDataUrl, twka['urlSlug']) if(auction): auctions.append(auction) Cache.add(cachename, auctions) return auctions return [] def getTWKAuction(twkDataUrl, auctionurlslug): response = requests.get(twkDataUrl + "a/" + auctionurlslug + '.json') if(response.status_code ==200): data = response.json(); if(len(data['pageProps']['lots']['results']) ==0): return None; twka = data['pageProps']['auction']; firstlot = data['pageProps']['lots']['results'][0] city = "Nederland" if firstlot['location']['city'].lower() == 'online' else firstlot['location']['city'] # if(firstlot['location']['city'].lower() != 'online'): # city = firstlot['location']['city']; a = Auction(Auctionbrand.TWK, city, firstlot['location']['countryCode'].upper(), twka['name'], datetime.fromtimestamp(twka['startDate']), datetime.fromtimestamp(twka['minEndDate']), '/a/' + auctionurlslug, twka['image']['url'], twka['lotCount'] ) # print(a); return a; return None def getOVMAuctions(): cachename = 'OnlineVeiling_' res = Cache.get(cachename) if(res): return res response = requests.get("https://onlineveilingmeester.nl/rest/nl/veilingen?status=open&domein=ONLINEVEILINGMEESTER") if(response.status_code ==200): print('Got Ovm Auctions') data = response.json() auctions = [] for result in data['veilingen']: cityname ="Nederland" if result['isBezorgVeiling'] else result['afgifteAdres']['plaats'] cityname = "Nederland" if cityname is None else cityname #there can be auctions where you have to make an appointment to retrieve the lots startdatetime = result['openingsDatumISO'].replace("T", " ").replace("Z", "") enddatetime = result['sluitingsDatumISO'].replace("T", " ").replace("Z", "") a = Auction(Auctionbrand.OVM, cityname,result['land'], result['naam'],startdatetime, enddatetime, str(result['land']).lower() + '/veilingen/' + str(result['id']) + '/kavels', 'images/150x150/' + str(result['id']) + '/' + result['image'], result['totaalKavels'] ) auctions.append(a) Cache.add(cachename, auctions) return auctions return None