Added auctionport auctions

README.md

@@ -1 +1,4 @@
 # PythonAuctionviewer
+
+## Run dev server
+To run the development server run `python app.py` in the root of the project

cache.py

@@ -67,5 +67,6 @@ class FileCache():
   def add(key, obj):
         log('adding filecacheobject ' + key)
         json_data = JsonEncoder().encode(obj)
+        # json_data = json.dumps(obj, cls=JsonEncoder, indent=2)
         with open("./filecache/" + key + ".json", 'w+') as f:
           f.write(json_data)

data/locationfiles/NL.txt

@@ -465,6 +465,7 @@
 2743947	Zevenbergen	Zevenbergen	Zevenbergen,ze fen bei heng,zefenberugen,Зевенберген,ゼーフェンベルゲン,澤芬貝亨	51.645	4.60417	P	PPL	NL		06	1709			0		5	Europe/Amsterdam	2017-10-17
 2743948	Gemeente Zevenaar	Gemeente Zevenaar	Gemeente Zevenaar,Zevenaar	51.95376	6.07727	A	ADM2	NL		03	0299			44096		9	Europe/Amsterdam	2023-01-31
 2743949	Zevenaar	Zevenaar	Zevenar,Зевенар	51.93	6.07083	P	PPL	NL		03	0299			26063		12	Europe/Amsterdam	2017-10-17
+2743949	omgeving zevenaar	omgeving zevenaar	Zevenar,Зевенар	51.93	6.07083	P	PPL	NL		03	0299			26063		12	Europe/Amsterdam	2017-10-17
 2743950	Zeumersche Beek	Zeumersche Beek		52.16452	5.53953	H	STM	NL		03				0		5	Europe/Amsterdam	2011-04-19
 2743951	Zeumeren	Zeumeren		52.17256	5.60208	P	PPL	NL		03	0203			0		12	Europe/Amsterdam	2009-01-17
 2743952	Zetten	Zetten		51.92833	5.71389	P	PPL	NL		03	1734			2985		10	Europe/Amsterdam	2017-10-17
@@ -13230,6 +13231,7 @@
 2757247	Den Bramel	Den Bramel		52.12176	6.3111	S	CSTL	NL		03	1876			0		15	Europe/Amsterdam	2011-06-04
 2757248	Den Braam	Den Braam		52.1375	6.84028	P	PPL	NL		15	0158			0		38	Europe/Amsterdam	2007-06-03
 2757251	Den Bosch	Den Bosch		52.07795	6.13621	S	EST	NL		03	0213			0		13	Europe/Amsterdam	2011-06-04
+2757251	's hertogenbosch	's hertogenbosch		52.07795	6.13621	S	EST	NL		03	0213			0		13	Europe/Amsterdam	2011-06-04
 2757252	Polder Den Bommel	Polder Den Bommel		51.70597	4.28369	T	PLDR	NL		11				0		-2	Europe/Amsterdam	2011-04-19
 2757253	Den Bommel	Den Bommel	Bommel	51.71583	4.28472	P	PPL	NL		11	1924			0		-1	Europe/Amsterdam	2017-10-17
 2757254	Polder Den Bol	Polder Den Bol		51.71665	4.75829	T	PLDR	NL		06				0		-1	Europe/Amsterdam	2011-04-19

models/location.py

@@ -1,5 +1,7 @@
 from enum import Enum
+import enum
 from json import JSONEncoder
+import json
 
 class Countrycode(Enum):
     NL = "NL",
@@ -10,6 +12,7 @@ class Auctionbrand(str, Enum):
     NONE = "NONE",
     TWK = "TWK"
     OVM = "OVM"
+    AP = "AP"
 
 
 class GeonameLocation:
@@ -32,7 +35,7 @@ class Maplocation:
         self.auctions = auctions
 
 class Auction: 
-    def __init__(self, auctionbrand: Auctionbrand = Auctionbrand.NONE, city = "", countrycode:Countrycode = Countrycode.NL, name = "", starttime = None, closingtime = None, url = "", imageurl = "", numberoflots = 0, geonamelocation: GeonameLocation =  None):
+    def __init__(self, auctionbrand: Auctionbrand = Auctionbrand.NONE, city = "", countrycode:Countrycode = Countrycode.NL, name = "", starttime = None, closingtime = None, url = "", imageurl = "", numberoflots = 0, geonamelocation: GeonameLocation =  None, multiplelocations = False):
         self.city = city
         self.countrycode = countrycode
         self.name = name
@@ -43,7 +46,34 @@ class Auction:
         self.numberoflots = numberoflots
         self.geonamelocation = geonamelocation
         self.brand = auctionbrand
+        self.multiplelocations = multiplelocations
 
 class JsonEncoder(JSONEncoder):
-    def default(self, o):
+    # def default(self, o):
-        return o.__dict__
+    #     return o.__dict__
+    #try 2
+    def default(self, obj):
+        # Only serialize public, instance-level attributes
+        if hasattr(obj, '__dict__'):
+            return {
+                key: self.serialize(value)
+                for key, value in obj.__dict__.items()
+                if not key.startswith('_')  # skip private/protected
+            }
+        return super().default(obj)
+
+    def serialize(self, value):
+        if isinstance(value, list):
+            return [self.serialize(item) for item in value]
+        elif isinstance(value, dict):
+            return {k: self.serialize(v) for k, v in value.items()}
+        elif isinstance(value, enum.Enum):
+            return value.name  # or value.value
+        elif hasattr(value, '__dict__'):
+            return self.default(value)  # dive into nested object
+        else:
+            try:
+                json.dumps(value)
+                return value
+            except (TypeError, OverflowError):
+                return str(value)

requirements.txt

@@ -1,7 +1,7 @@
-pip==9.0.1
+pip>=23.3
-setuptools==28.8.0
+setuptools>=78.1.1
-flask===3.0.3
+flask===3.1.1
-werkzeug==3.0.0
+werkzeug>=3.0.6
-flask_cors===5.0.0
+flask_cors>=6.0.0
-requests===2.27.1
+requests>=2.32.4
 pathlib===1.0.1

utils/APutils.py

@@ -0,0 +1,77 @@
+
+
+from datetime import datetime
+import json
+import re
+from time import ctime
+from traceback import print_exc
+import requests
+from cache import Cache
+from models.location import Auction, Auctionbrand, Countrycode, JsonEncoder
+from utils.helperutils import log
+
+
+def getAPAuctions():
+    cachename = 'AuctionPort_'
+    res = Cache.get(cachename)
+    if(res):
+      return res
+
+    try:
+      response = requests.get("https://api.auctionport.be/auctions/small?size=100&page=1")
+    except:
+        log("The Auctionport auctions call threw a error")
+
+    if(response is None):
+       return []
+      
+    if(response.status_code ==200):
+        log('Got AP Auctions')
+        try:
+            data = response.json()
+            pages = data['pages']
+            auctions = []
+            for i in range(0,pages-1,1):
+                log("getting page " + str(i) + ' of ' + str(pages))
+                # if(i > 1):
+                response = requests.get("https://api.auctionport.be/auctions/small?size=100&page=" + str(i))
+                if(response is None): continue
+                if(response.status_code != 200): continue
+
+                data = response.json()
+                
+                for PAauction in data['data']:
+                    #makes sure that the locations array is filled with at least one location
+                    if PAauction['locations'] == []: 
+                        PAauction['locations'] = [PAauction['location']]
+
+                    #makes sure that the auction is still active
+                    closingdate = datetime.fromisoformat(PAauction['closingDate'])
+                    if(closingdate.date() < datetime.now().date() ): continue
+                    if(PAauction['lotCount'] <= 0): continue
+
+                    multipleLocations = len(PAauction['locations']) > 1
+
+                    for location in PAauction['locations']:
+                        if not location.endswith('Nederland'): continue
+                        loc = re.sub('Nederland', '', location)
+                        # loc = location.split(",")
+                        postalcodeRegex = r'(.*?)[1-9][0-9]{3} ?(?!sa|sd|ss)[a-zA-Z]{2}'
+                        city = re.sub(postalcodeRegex  , '', loc) #removes postalcode and everything in front of it
+                        city = city.strip() #removes whitespace
+                        city = city.strip(',') #removes trailing and leading ,
+                        city = city.split(',') #splits on , to overcome not matching regex
+                        city = city[len(city)-1]
+                        city = city.strip()
+                        newauction = Auction(Auctionbrand.AP,city, Countrycode.NL, PAauction['title'], datetime.fromisoformat(PAauction['openDate']), datetime.fromisoformat(PAauction['closingDate']), '/auction/'+ str(PAauction['id']), PAauction['imageUrl'], PAauction['lotCount'] , None, multipleLocations)
+
+                        auctions.append(newauction)
+            Cache.add(cachename, auctions)
+            return auctions
+        except Exception as e:
+           log(e.__cause__ + '--  Something went wrong in the mapping of AP auctions to auctionviewer objects. The reason was: ' + response.reason  + '. The response was: ' + JsonEncoder().encode(response.json()))
+           print_exc(e)
+        
+    else:
+       log("The AP auctions call didn't gave a 200 response but a " + str(response.status_code) + ". With the reason: " + response.reason)
+    return []

utils/OVMutils.py

@@ -0,0 +1,52 @@
+from traceback import print_exc
+import requests
+from cache import Cache
+from models.location import Auction, Auctionbrand, JsonEncoder
+from utils.helperutils import log
+
+
+def getOVMAuctions():
+    cachename = 'OnlineVeiling_'
+    res = Cache.get(cachename)
+    if(res):
+      return res
+
+    try:
+      response = requests.get("https://onlineveilingmeester.nl/rest/nl/veilingen?status=open&domein=ONLINEVEILINGMEESTER")
+    except:
+        log("The OVM auctions call threw a error")
+
+    if(response is None):
+       return []
+      
+    if(response.status_code ==200):
+        log('Got Ovm Auctions')
+        try:
+           data = response.json()
+           auctions = []
+           for result in data['veilingen']:
+               cityname ="Nederland"  if result['isBezorgVeiling'] else result['afgifteAdres']['plaats'] 
+               cityname = "Nederland" if cityname is None else cityname #there can be auctions where you have to make an appointment to retrieve the lots
+               startdatetime = result['openingsDatumISO'].replace("T", " ").replace("Z", "")
+               enddatetime = result['sluitingsDatumISO'].replace("T", " ").replace("Z", "")
+               image = ""
+               #if hasattr(result, 'image') : #result['image'] :
+               image = result.get('image', "")  #['image']
+               if image == "":
+                  images = result.get('imageList')
+                  if(len(images) >0):
+                     image = images[0]
+                  else:
+                     log("No image found for OVM auction: " + result['naam'])
+               
+               a = Auction(Auctionbrand.OVM, cityname,result['land'], result['naam'],startdatetime, enddatetime, str(result['land']).lower() + '/veilingen/' + str(result['id']) + '/kavels', 'images/150x150/' + image, result['totaalKavels'] )
+               auctions.append(a)
+           Cache.add(cachename, auctions)
+           return auctions
+        except Exception as e:
+           log(e.__cause__ + '--  Something went wrong in the mapping of OVM auctions to auctionviewer objects. The reason was: ' + response.reason  + '. The response was: ' + JsonEncoder().encode(response.json()))
+           print_exc(e)
+        
+    else:
+       log("The OVM auctions call didn't gave a 200 response but a " + str(response.status_code) + ". With the reason: " + response.reason)
+    return []

utils/TWKutils.py

@@ -0,0 +1,77 @@
+from datetime import datetime
+import math
+import re
+import requests
+
+from cache import Cache
+from models.location import Auction, Auctionbrand
+from utils.helperutils import log
+
+
+def getTWKUrl():
+    response = requests.get('https://www.troostwijkauctions.com/')
+    if(response.status_code ==200):
+      buildid = re.search(r'"buildId":"([^"]*)', response.text, re.MULTILINE )
+      twkDataUrl = 'https://www.troostwijkauctions.com/_next/data/' + str(buildid[1]) + '/nl/'
+      log('buildid: ' + str(buildid[1]))
+      log('twkDataUrl: ' + twkDataUrl)
+      return twkDataUrl
+
+    return None
+
+
+def getTwkAuctions(countrycode):
+    cachename = 'TwkAuctions_'+ countrycode
+    res = Cache.get(cachename)
+    if(res):
+      return res
+
+    # buildidresponse = requests.get('https://www.troostwijkauctions.com/')
+    twkDataUrl = getTWKUrl()
+
+    if(twkDataUrl is None):
+        return []
+
+    response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode)
+
+    if(response.status_code ==200):
+        log('Got Twk Auctions')
+        data = response.json()
+        auctions = []
+        
+        totalAuctionCount = data['pageProps']['totalSize'];
+        pages = math.ceil(totalAuctionCount / data['pageProps']['pageSize'])
+        # for result in data['pageProps']['auctionList']:
+        
+        for i in range(1,pages,1):
+          log("getting page " + str(i) + ' of ' + str(pages))
+          if(i > 1):
+              response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode + "&page=" + str(i))
+              data = response.json()
+          
+          for twka in data['pageProps']['listData']:
+            # print(twka['urlSlug'])
+            auction = getTWKAuction(twkDataUrl, twka['urlSlug'])
+            if(auction):
+              auctions.append(auction)
+        Cache.add(cachename, auctions)
+
+        return auctions
+    return []
+
+def getTWKAuction(twkDataUrl, auctionurlslug):
+    log("getting TWK auctiondetails:" + twkDataUrl + "a/" + auctionurlslug + ".json")
+    response = requests.get(twkDataUrl + "a/" + auctionurlslug + '.json')
+    if(response.status_code == 200):
+        data = response.json()
+        if(len(data['pageProps']['lots']['results']) ==0):
+          return None
+    
+        twka = data['pageProps']['auction']
+        firstlot = data['pageProps']['lots']['results'][0]
+        city = "Nederland" if  firstlot['location']['city'].lower() == 'online' or firstlot['location']['city'].lower() == "free delivery" else firstlot['location']['city']
+        a = Auction(Auctionbrand.TWK, city, firstlot['location']['countryCode'].upper(), twka['name'], datetime.fromtimestamp(twka['startDate']), datetime.fromtimestamp(twka['minEndDate']), '/a/' + auctionurlslug, twka['image']['url'], twka['lotCount'] )
+        # print(a);
+        return a
+
+    return None

utils/auctionutils.py

@@ -2,11 +2,12 @@ import requests
 from traceback import print_exc
 from cache import Cache, FileCache
 from models.location import Auction, Auctionbrand, Countrycode, Maplocation, JsonEncoder
+from utils.APutils import getAPAuctions
+from utils.OVMutils import getOVMAuctions
+from utils.TWKutils import getTwkAuctions
 from utils.locationutils import getGeoLocationByCity
 from utils.helperutils import log
 from datetime import datetime
-import re
-import math
 
 def getAuctionlocations(countrycode: Countrycode, clearcache:bool = False):
     cachename = 'allauctions_' + countrycode
@@ -23,6 +24,7 @@ def getAuctionlocations(countrycode: Countrycode, clearcache:bool = False):
     
     twkauctions = []
     ovmauctions = []
+    apauctions = []
 
     try:
         twkauctions = getTwkAuctions(countrycode)
@@ -36,8 +38,14 @@ def getAuctionlocations(countrycode: Countrycode, clearcache:bool = False):
         log('something went wrong while running the OVM auctions request')
         print_exc(e)
 
+    try:
+         apauctions = getAPAuctions() 
+    except Exception as e: 
+        log('something went wrong while running the OVM auctions request')
+        print_exc(e)
 
-    auctions = [*twkauctions, *ovmauctions]
+
+    auctions = [*twkauctions, *ovmauctions, *apauctions]
 
     for auction in auctions:
         auction.geonamelocation = getGeoLocationByCity(auction.city, countrycode)
@@ -71,119 +79,6 @@ def get_geonameid(auction):
         return auction.geonamelocation.geonameid
     return None
 
-# global twkDataUrl; # = ''; #  'https://www.troostwijkauctions.com/_next/data/' #e6-N0pLHv12LVGS0oYzx6/nl/'
 
 
-def getTWKUrl():
-    response = requests.get('https://www.troostwijkauctions.com/')
-    if(response.status_code ==200):
-      buildid = re.search(r'"buildId":"([^"]*)', response.text, re.MULTILINE )
-      twkDataUrl = 'https://www.troostwijkauctions.com/_next/data/' + str(buildid[1]) + '/nl/'
-      log('buildid: ' + str(buildid[1]))
-      log('twkDataUrl: ' + twkDataUrl)
-      return twkDataUrl
 
-    return None
-
-
-def getTwkAuctions(countrycode):
-    cachename = 'TwkAuctions_'+ countrycode
-    res = Cache.get(cachename)
-    if(res):
-      return res
-
-    # buildidresponse = requests.get('https://www.troostwijkauctions.com/')
-    twkDataUrl = getTWKUrl()
-
-    if(twkDataUrl is None):
-        return []
-
-    response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode)
-
-    if(response.status_code ==200):
-        log('Got Twk Auctions')
-        data = response.json()
-        auctions = []
-        
-        totalAuctionCount = data['pageProps']['totalSize'];
-        pages = math.ceil(totalAuctionCount / data['pageProps']['pageSize'])
-        # for result in data['pageProps']['auctionList']:
-        
-        for i in range(1,pages,1):
-          log("getting page " + str(i) + ' of ' + str(pages))
-          if(i > 1):
-              response = requests.get(twkDataUrl + "auctions.json?countries=" + countrycode + "&page=" + str(i));
-              data = response.json()
-          
-          for twka in data['pageProps']['listData']:
-            # print(twka['urlSlug'])
-            auction = getTWKAuction(twkDataUrl, twka['urlSlug'])
-            if(auction):
-              auctions.append(auction)
-        Cache.add(cachename, auctions)
-
-        return auctions
-    return []
-
-def getTWKAuction(twkDataUrl, auctionurlslug):
-    log("getting TWK auctiondetails:" + twkDataUrl + "a/" + auctionurlslug + ".json")
-    response = requests.get(twkDataUrl + "a/" + auctionurlslug + '.json')
-    if(response.status_code == 200):
-        data = response.json()
-        if(len(data['pageProps']['lots']['results']) ==0):
-          return None
-    
-        twka = data['pageProps']['auction']
-        firstlot = data['pageProps']['lots']['results'][0]
-        city = "Nederland" if  firstlot['location']['city'].lower() == 'online' or firstlot['location']['city'].lower() == "free delivery" else firstlot['location']['city']
-        a = Auction(Auctionbrand.TWK, city, firstlot['location']['countryCode'].upper(), twka['name'], datetime.fromtimestamp(twka['startDate']), datetime.fromtimestamp(twka['minEndDate']), '/a/' + auctionurlslug, twka['image']['url'], twka['lotCount'] )
-        # print(a);
-        return a
-
-    return None
-
-def getOVMAuctions():
-    cachename = 'OnlineVeiling_'
-    res = Cache.get(cachename)
-    if(res):
-      return res
-
-    try:
-      response = requests.get("https://onlineveilingmeester.nl/rest/nl/veilingen?status=open&domein=ONLINEVEILINGMEESTER")
-    except:
-        log("The OVM auctions call threw a error")
-
-    if(response is None):
-       return []
-      
-    if(response.status_code ==200):
-        log('Got Ovm Auctions')
-        try:
-           data = response.json()
-           auctions = []
-           for result in data['veilingen']:
-               cityname ="Nederland"  if result['isBezorgVeiling'] else result['afgifteAdres']['plaats'] 
-               cityname = "Nederland" if cityname is None else cityname #there can be auctions where you have to make an appointment to retrieve the lots
-               startdatetime = result['openingsDatumISO'].replace("T", " ").replace("Z", "")
-               enddatetime = result['sluitingsDatumISO'].replace("T", " ").replace("Z", "")
-               image = ""
-               #if hasattr(result, 'image') : #result['image'] :
-               image = result.get('image', "")  #['image']
-               if image == "":
-                  images = result.get('imageList')
-                  if(len(images) >0):
-                     image = images[0]
-                  else:
-                     log("No image found for OVM auction: " + result['naam'])
-               
-               a = Auction(Auctionbrand.OVM, cityname,result['land'], result['naam'],startdatetime, enddatetime, str(result['land']).lower() + '/veilingen/' + str(result['id']) + '/kavels', 'images/150x150/' + image, result['totaalKavels'] )
-               auctions.append(a)
-           Cache.add(cachename, auctions)
-           return auctions
-        except Exception as e:
-           log(e.__cause__ + '--  Something went wrong in the mapping of OVM auctions to auctionviewer objects. The reason was: ' + response.reason  + '. The response was: ' + JsonEncoder().encode(response.json()))
-           print_exc(e)
-        
-    else:
-       log("The OVM auctions call didn't gave a 200 response but a " + str(response.status_code) + ". With the reason: " + response.reason)
-    return []