diff --git a/compare.py b/compare.py index d234014..435dd66 100644 --- a/compare.py +++ b/compare.py @@ -7,38 +7,46 @@ """ from bs4 import BeautifulSoup -import datetime +from datetime import datetime from tinydb import TinyDB, Query import urllib3 import sys import scrape_listing as sl import search_eBay as se - +import export_html as exhtml if __name__ == "__main__": seller_db = TinyDB("seller_db.json") database_lenght = len(seller_db.all()) + comparison_results = list() if database_lenght == 0: print("Unable to load seller shop database!") sys.exit(-1) print("Loaded seller shop database: " + str(database_lenght) + " listings") + compare_start_timestamp = datetime.now() #set start time for comparing for shop_listing_url in seller_db.all(): #print(shop_listing_url["epid"]) for retrieve_counter in range(5): try: - shop_listing_data = sl.scrape_listing(shop_listing_url["epid"]) + seller_listing_data = sl.scrape_listing(shop_listing_url["epid"]) break except: #pass print("Unable to retrieve seller listing data from: " + shop_listing_url["epid"]) - if shop_listing_data: - print("\n\nCompare: " + shop_listing_data["title"] + " | " + str(shop_listing_data["price"]) + "€ | " + shop_listing_url["epid"]) + if seller_listing_data: + print("\n\nCompare: " + seller_listing_data["title"] + " | " + str(seller_listing_data["price"]) + "€ | " + shop_listing_url["epid"]) - competitor_listings = se.search(shop_listing_data["title"], 1) - print("Found " + str(len(competitor_listings)) + " listings from competitors with term: " + shop_listing_data["title"]) + comparison_result = { + 'seller_listing': seller_listing_data, + 'competitor_listings': list(), + 'max_price_delta': float(0.0) + } + + competitor_listings = se.search(seller_listing_data["title"], 1) + print("Found " + str(len(competitor_listings)) + " listings from competitors with term: " + seller_listing_data["title"]) sys.stdout.flush() cheaper_listings = list() for competitor_listing_url in competitor_listings: @@ -46,7 +54,6 @@ if __name__ == "__main__": #print(shop_listing_url["epid"]) if seller_db.search(Query().epid == competitor_listing_url): - #if competitor_listing_url == shop_listing_url["epid"]: print("Found listing from sellers shop --> ignore " + competitor_listing_url) continue #else: @@ -63,16 +70,41 @@ if __name__ == "__main__": if competitor_listing_data: #print(competitor_listing_data["price"]) - if competitor_listing_data["price"] < shop_listing_data["price"]: - #print("found cheaper competitor: " + str(competitor_listing_data["price"]) + "€ instead: " + str(shop_listing_data["price"]) + "€ ---> " + competitor_listing_url) + if competitor_listing_data["price"] < seller_listing_data["price"]: + #print("found cheaper competitor: " + str(competitor_listing_data["price"]) + "€ instead: " + str(seller_listing_data["price"]) + "€ ---> " + competitor_listing_url) cheaper_listings.append({ 'title': competitor_listing_data["title"], 'price': competitor_listing_data["price"], + 'image': competitor_listing_data["image"], 'url': competitor_listing_url}) for cheaper_listing in sorted(cheaper_listings, key=lambda d: d['price']) : #print(cheaper_listing) - print("found cheaper competitor: " + str(cheaper_listing["price"]) + "€ instead: " + str(shop_listing_data["price"]) + "€ ---> " + cheaper_listing["url"]) - #break + print("found cheaper competitor: " + str(cheaper_listing["price"]) + "€ instead: " + str(seller_listing_data["price"]) + "€ ---> " + cheaper_listing["url"]) + comparison_result['competitor_listings'].append(cheaper_listing) + if comparison_result['max_price_delta'] == 0.0: + comparison_result['max_price_delta'] = seller_listing_data["price"] - cheaper_listing["price"] + + if cheaper_listings: + comparison_results.append(comparison_result) + + break + + now = datetime.now() # current date and time + + exp = exhtml.exporter("./html_out/") + + for comparison in sorted(comparison_results, key=lambda d: d['max_price_delta'], reverse=True): + exp.export_comparison(comparison['seller_listing'], comparison['competitor_listings']) + + exp.export_startpage(str(database_lenght), len(comparison), datetime.timestamp(compare_start_timestamp), now.strftime("%m/%d/%Y, %H:%M:%S")) + + + + + + + + \ No newline at end of file diff --git a/export_html.py b/export_html.py index d77420e..fd41c04 100644 --- a/export_html.py +++ b/export_html.py @@ -9,6 +9,7 @@ from datetime import datetime import os import template_html as thtml +import shutil class exporter: export_dir="" @@ -21,27 +22,53 @@ class exporter: os.mkdir(self.export_dir) except FileExistsError: pass - + try: + os.mkdir(os.path.join(self.export_dir,"compare/")) + except FileExistsError: + pass + + self.copy_static_export() + + def copy_static_export(self): + try: + os.mkdir(os.path.join(self.export_dir,"css/")) + except FileExistsError: + pass + try: + os.mkdir(os.path.join(self.export_dir,"data/")) + except FileExistsError: + pass + + shutil.copy("./html/css/w3.css", os.path.join(self.export_dir,"css/","w3.css")) + shutil.copy("./html/data/favicon.ico", os.path.join(self.export_dir,"data/","favicon.ico")) + shutil.copy("./html/data/icon.png", os.path.join(self.export_dir,"data/","icon.png")) + def export_comparison(self, seller_listing, competitor_listings): self.counter +=1 - f = open(os.path.join(self.export_dir, str(self.counter) + ".html"), "a") + f = open(os.path.join(self.export_dir, "compare/", str(self.counter) + ".html"), "a") f.write(thtml.html_comparison_head()) - - - - + f.write("") + f.write(thtml.html_comparison_navigation(self.counter)) + f.write(thtml.html_comparison_seller_listing(seller_listing)) + f.write(thtml.html_comparison_competitor_list_header()) + competitor_listing_counter = 0 + for competitor_listing in competitor_listings: + competitor_listing_counter +=1 + f.write(thtml.html_comparison_competitor_listing(competitor_listing, competitor_listing_counter)) + f.write(thtml.html_comparison_trailer()) f.close() + def export_startpage(self, seller_listings_count, cheaper_listings_count, compare_time, date): + duration_export = datetime.timestamp(self.tsStart) + print("Comparison needed: ", duration_export) - - - def export_startpage(): - duration = datetime.timestamp(self.tsStart) - print("Comparison needed: ", duration) - - - + f = open(os.path.join(self.export_dir, "index.html"), "a") + f.write(thtml.html_startpage_head()) + f.write(thtml.html_startpage_info(seller_listings_count, cheaper_listings_count, compare_time, duration_export, date)) + f.write(thtml.html_startpage_trailer()) + f.close() + if __name__ == "__main__": seller_listing_dummy = { diff --git a/html/index.html b/html/index.html index 4bca54f..e7e35b6 100644 --- a/html/index.html +++ b/html/index.html @@ -19,8 +19,21 @@

eBay competitor price compare

-
- +
+
+

Seller info

+

Number of seller listings:

+

Possibly cheaper listings:

+

Runtime info

+

Compare time needed:

+

Export time needed:

+

Date of creation:

+
+
+
+ +
+
diff --git a/scrape_listing.py b/scrape_listing.py index e621165..764bd1c 100644 --- a/scrape_listing.py +++ b/scrape_listing.py @@ -34,9 +34,9 @@ def scrape_listing(url): listing = { 'title': soup.find("div", class_="vim x-item-title").span.text, 'price': float(soup.find("span", id="prcIsum")["content"]), - 'image': soup.find("img", id="icImg")["src"] + 'image': soup.find("img", id="icImg")["src"], + 'url' : url } - return listing if __name__ == "__main__": diff --git a/template_html.py b/template_html.py index 66e4205..6bcac7a 100644 --- a/template_html.py +++ b/template_html.py @@ -12,10 +12,10 @@ def html_comparison_head(): - - + + eBay competitor price compare - + ''' @@ -66,6 +70,7 @@ def html_comparison_navigation(counter): back_link = str((str(counter-1)+".html") if ((counter-1) > 0) else "#") current = str(counter) + current_link = str(counter)+".html" next_link = str((str(counter+1)+".html")) #TODO: test if last one and replace with '#' return ''' @@ -76,7 +81,7 @@ def html_comparison_navigation(counter):
- +

Compare #''' + current + '''
@@ -86,4 +91,120 @@ def html_comparison_navigation(counter):
-''' \ No newline at end of file +''' + +def html_comparison_seller_listing(seller_listing): + return ''' +
+ +
+
+
+ image not found +
+
+

''' + seller_listing['title'] + '''

+

''' + str(seller_listing['price']) + ''' €

+
+
+ +
+ ''' + +def html_comparison_competitor_list_header(): + return ''' +

Competitor results:

+
+
+ ''' + +def html_comparison_competitor_listing(competitor_listing, competitor_listing_counter): + return ''' +
+ +
+
+
#''' + str(competitor_listing_counter) + '''
+
+ image not found +
+
+

''' + competitor_listing['title'] + '''

+

''' + str(competitor_listing['price']) + ''' €

+
+
+ +
+
+ ''' + +def html_comparison_trailer(): + return ''' +
+ + +''' + +def html_startpage_head(): + return ''' + + + + + + + + eBay competitor price compare + + + + ''' + +def html_startpage_info(seller_listings_count, cheaper_listings_count, compare_time, export_time, date): + + + return ''' +
+

eBay competitor price compare

+
+
+

Seller info

+

Number of seller listings: ''' + str(seller_listings_count) + '''

+

Possibly cheaper listings: ''' + str(cheaper_listings_count) + '''

+

Runtime info

+

Compare time needed: ''' + str(compare_time) + '''

+

Export time needed: ''' + str(export_time) + '''

+

Date of creation: ''' + str(cheaper_listings_count) + '''

+
+
+
+ +
+
+
+ ''' + +def html_startpage_trailer(): + return ''' +
+
+
+ +
mail@hendrikschutter.com
 + +
https://git.mosad.xyz/localhorst/eBayCompetitorPriceCompare
 +
+
+
+