diff --git a/eBayCompetitorPriceCompare.code-workspace b/eBayCompetitorPriceCompare.code-workspace new file mode 100644 index 0000000..876a149 --- /dev/null +++ b/eBayCompetitorPriceCompare.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file diff --git a/search_db.json b/search_db.json new file mode 100644 index 0000000..b9aa4cf --- /dev/null +++ b/search_db.json @@ -0,0 +1 @@ +{"_default": {"1": {"epid": "https://www.ebay.de/itm/325212174251"}, "2": {"epid": "https://www.ebay.de/itm/165509861026"}, "3": {"epid": "https://www.ebay.de/itm/234573771828"}, "4": {"epid": "https://www.ebay.de/itm/115404959911"}, "5": {"epid": "https://www.ebay.de/itm/265716024700"}, "6": {"epid": "https://www.ebay.de/itm/384921220404"}, "7": {"epid": "https://www.ebay.de/itm/125342657251"}, "8": {"epid": "https://www.ebay.de/itm/363857161698"}, "9": {"epid": "https://www.ebay.de/itm/363857160883"}, "10": {"epid": "https://www.ebay.de/itm/363857160692"}, "11": {"epid": "https://www.ebay.de/itm/363857160691"}, "12": {"epid": "https://www.ebay.de/itm/363857160656"}, "13": {"epid": "https://www.ebay.de/itm/363857160615"}, "14": {"epid": "https://www.ebay.de/itm/363857160535"}, "15": {"epid": "https://www.ebay.de/itm/195096955553"}, "16": {"epid": "https://www.ebay.de/itm/255561052674"}, "17": {"epid": "https://www.ebay.de/itm/125342557954"}, "18": {"epid": "https://www.ebay.de/itm/284836318883"}, "19": {"epid": "https://www.ebay.de/itm/265715690410"}, "20": {"epid": "https://www.ebay.de/itm/255560824944"}, "21": {"epid": "https://www.ebay.de/itm/255560824939"}, "22": {"epid": "https://www.ebay.de/itm/284836318795"}, "23": {"epid": "https://www.ebay.de/itm/284836318798"}, "24": {"epid": "https://www.ebay.de/itm/255560824898"}, "25": {"epid": "https://www.ebay.de/itm/265715690315"}, "26": {"epid": "https://www.ebay.de/itm/265715690300"}, "27": {"epid": "https://www.ebay.de/itm/284836318765"}, "28": {"epid": "https://www.ebay.de/itm/284836318768"}, "29": {"epid": "https://www.ebay.de/itm/255560824844"}, "30": {"epid": "https://www.ebay.de/itm/284836318731"}, "31": {"epid": "https://www.ebay.de/itm/284836318735"}, "32": {"epid": "https://www.ebay.de/itm/255560824812"}, "33": {"epid": "https://www.ebay.de/itm/255560824795"}, "34": {"epid": "https://www.ebay.de/itm/265715690244"}, "35": {"epid": "https://www.ebay.de/itm/325211742653"}, "36": {"epid": "https://www.ebay.de/itm/363856788994"}, "37": {"epid": "https://www.ebay.de/itm/363856786295"}, "38": {"epid": "https://www.ebay.de/itm/225010180205"}, "39": {"epid": "https://www.ebay.de/itm/144579823229"}, "40": {"epid": "https://www.ebay.de/itm/363856742499"}, "41": {"epid": "https://www.ebay.de/itm/275332072283"}, "42": {"epid": "https://www.ebay.de/itm/295011963973"}, "43": {"epid": "https://www.ebay.de/itm/195096316604"}, "44": {"epid": "https://www.ebay.de/itm/155016945757"}, "45": {"epid": "https://www.ebay.de/itm/155016945027"}, "46": {"epid": "https://www.ebay.de/itm/175301891815"}, "47": {"epid": "https://www.ebay.de/itm/195096313380"}, "48": {"epid": "https://www.ebay.de/itm/195096313262"}, "49": {"epid": "https://www.ebay.de/itm/195096313261"}, "50": {"epid": "https://www.ebay.de/itm/195096313251"}, "51": {"epid": "https://www.ebay.de/itm/284836140428"}, "52": {"epid": "https://www.ebay.de/itm/195096313195"}, "53": {"epid": "https://www.ebay.de/itm/195096313077"}, "54": {"epid": "https://www.ebay.de/itm/195096313075"}, "55": {"epid": "https://www.ebay.de/itm/195096313034"}, "56": {"epid": "https://www.ebay.de/itm/195096313012"}, "57": {"epid": "https://www.ebay.de/itm/195096312896"}, "58": {"epid": "https://www.ebay.de/itm/195096312897"}, "59": {"epid": "https://www.ebay.de/itm/195096312876"}, "60": {"epid": "https://www.ebay.de/itm/195096312669"}, "61": {"epid": "https://www.ebay.de/itm/195096312670"}, "62": {"epid": "https://www.ebay.de/itm/195096312659"}, "63": {"epid": "https://www.ebay.de/itm/234573371008"}, "64": {"epid": "https://www.ebay.de/itm/374105796206"}, "65": {"epid": "https://www.ebay.de/itm/195095921069"}, "66": {"epid": "https://www.ebay.de/itm/384920243964"}, "67": {"epid": "https://www.ebay.de/itm/195095911356"}, "68": {"epid": "https://www.ebay.de/itm/304508853701"}, "69": {"epid": "https://www.ebay.de/itm/284835817712"}, "70": {"epid": "https://www.ebay.de/itm/304508853587"}, "71": {"epid": "https://www.ebay.de/itm/255560300222"}, "72": {"epid": "https://www.ebay.de/itm/275331811942"}, "73": {"epid": "https://www.ebay.de/itm/155016612182"}, "74": {"epid": "https://www.ebay.de/itm/144579382956"}, "75": {"epid": "https://www.ebay.de/itm/195095851717"}, "76": {"epid": "https://www.ebay.de/itm/255560284177"}, "77": {"epid": "https://www.ebay.de/itm/255560280037"}, "78": {"epid": "https://www.ebay.de/itm/195095836290"}, "79": {"epid": "https://www.ebay.de/itm/255560273946"}, "80": {"epid": "https://www.ebay.de/itm/255560272916"}, "81": {"epid": "https://www.ebay.de/itm/325211269645"}, "82": {"epid": "https://www.ebay.de/itm/195095820038"}, "83": {"epid": "https://www.ebay.de/itm/284835777436"}, "84": {"epid": "https://www.ebay.de/itm/255560263188"}, "85": {"epid": "https://www.ebay.de/itm/203972687680"}, "86": {"epid": "https://www.ebay.de/itm/165508540955"}, "87": {"epid": "https://www.ebay.de/itm/225009719808"}, "88": {"epid": "https://www.ebay.de/itm/325211246449"}, "89": {"epid": "https://www.ebay.de/itm/284835761276"}, "90": {"epid": "https://www.ebay.de/itm/225009713824"}, "91": {"epid": "https://www.ebay.de/itm/334458608688"}, "92": {"epid": "https://www.ebay.de/itm/384919638560"}, "93": {"epid": "https://www.ebay.de/itm/325211213337"}, "94": {"epid": "https://www.ebay.de/itm/195095685700"}, "95": {"epid": "https://www.ebay.de/itm/125341768654"}, "96": {"epid": "https://www.ebay.de/itm/394096643078"}, "97": {"epid": "https://www.ebay.de/itm/225009692752"}, "98": {"epid": "https://www.ebay.de/itm/325211201161"}, "99": {"epid": "https://www.ebay.de/itm/175301511376"}, "100": {"epid": "https://www.ebay.de/itm/354085112978"}, "101": {"epid": "https://www.ebay.de/itm/325211190210"}, "102": {"epid": "https://www.ebay.de/itm/363856419835"}, "103": {"epid": "https://www.ebay.de/itm/363856419767"}, "104": {"epid": "https://www.ebay.de/itm/363856418585"}, "105": {"epid": "https://www.ebay.de/itm/363856418592"}, "106": {"epid": "https://www.ebay.de/itm/363856418181"}, "107": {"epid": "https://www.ebay.de/itm/255560191187"}, "108": {"epid": "https://www.ebay.de/itm/363856416838"}, "109": {"epid": "https://www.ebay.de/itm/234573030402"}, "110": {"epid": "https://www.ebay.de/itm/234573030209"}, "111": {"epid": "https://www.ebay.de/itm/234573030260"}, "112": {"epid": "https://www.ebay.de/itm/234573030181"}, "113": {"epid": "https://www.ebay.de/itm/255560190814"}, "114": {"epid": "https://www.ebay.de/itm/255560190831"}, "115": {"epid": "https://www.ebay.de/itm/255560190851"}, "116": {"epid": "https://www.ebay.de/itm/334458569880"}, "117": {"epid": "https://www.ebay.de/itm/234573030160"}, "118": {"epid": "https://www.ebay.de/itm/234573030167"}, "119": {"epid": "https://www.ebay.de/itm/255560190787"}, "120": {"epid": "https://www.ebay.de/itm/334458569814"}, "121": {"epid": "https://www.ebay.de/itm/334458569821"}, "122": {"epid": "https://www.ebay.de/itm/334458569838"}, "123": {"epid": "https://www.ebay.de/itm/234573030043"}, "124": {"epid": "https://www.ebay.de/itm/334458569743"}, "125": {"epid": "https://www.ebay.de/itm/334458569752"}, "126": {"epid": "https://www.ebay.de/itm/234573030006"}, "127": {"epid": "https://www.ebay.de/itm/234573030007"}, "128": {"epid": "https://www.ebay.de/itm/255560190680"}, "129": {"epid": "https://www.ebay.de/itm/255560190682"}, "130": {"epid": "https://www.ebay.de/itm/255560190684"}, "131": {"epid": "https://www.ebay.de/itm/334458569648"}, "132": {"epid": "https://www.ebay.de/itm/334458569706"}, "133": {"epid": "https://www.ebay.de/itm/334458569642"}, "134": {"epid": "https://www.ebay.de/itm/255560190544"}, "135": {"epid": "https://www.ebay.de/itm/234573029587"}, "136": {"epid": "https://www.ebay.de/itm/234573029573"}, "137": {"epid": "https://www.ebay.de/itm/255560190148"}, "138": {"epid": "https://www.ebay.de/itm/255560190150"}, "139": {"epid": "https://www.ebay.de/itm/334458569221"}, "140": {"epid": "https://www.ebay.de/itm/334458569257"}, "141": {"epid": "https://www.ebay.de/itm/234573029466"}, "142": {"epid": "https://www.ebay.de/itm/234573029472"}, "143": {"epid": "https://www.ebay.de/itm/234573029477"}, "144": {"epid": "https://www.ebay.de/itm/234573029505"}, "145": {"epid": "https://www.ebay.de/itm/255560190056"}, "146": {"epid": "https://www.ebay.de/itm/255560190066"}, "147": {"epid": "https://www.ebay.de/itm/334458569147"}, "148": {"epid": "https://www.ebay.de/itm/334458569159"}, "149": {"epid": "https://www.ebay.de/itm/334458569197"}, "150": {"epid": "https://www.ebay.de/itm/255560190054"}, "151": {"epid": "https://www.ebay.de/itm/234573029222"}, "152": {"epid": "https://www.ebay.de/itm/234573029224"}, "153": {"epid": "https://www.ebay.de/itm/255560189825"}, "154": {"epid": "https://www.ebay.de/itm/255560189839"}, "155": {"epid": "https://www.ebay.de/itm/334458568905"}, "156": {"epid": "https://www.ebay.de/itm/363856415600"}, "157": {"epid": "https://www.ebay.de/itm/363856415554"}, "158": {"epid": "https://www.ebay.de/itm/334458567444"}, "159": {"epid": "https://www.ebay.de/itm/255560188246"}, "160": {"epid": "https://www.ebay.de/itm/255560188258"}, "161": {"epid": "https://www.ebay.de/itm/334458567405"}, "162": {"epid": "https://www.ebay.de/itm/334458567419"}, "163": {"epid": "https://www.ebay.de/itm/334458567436"}, "164": {"epid": "https://www.ebay.de/itm/255560188171"}, "165": {"epid": "https://www.ebay.de/itm/255560188213"}, "166": {"epid": "https://www.ebay.de/itm/334458567352"}, "167": {"epid": "https://www.ebay.de/itm/334458567355"}, "168": {"epid": "https://www.ebay.de/itm/234573025522"}, "169": {"epid": "https://www.ebay.de/itm/334458566553"}, "170": {"epid": "https://www.ebay.de/itm/234573025445"}, "171": {"epid": "https://www.ebay.de/itm/334458566486"}, "172": {"epid": "https://www.ebay.de/itm/363856415241"}, "173": {"epid": "https://www.ebay.de/itm/234573025206"}, "174": {"epid": "https://www.ebay.de/itm/255560186950"}, "175": {"epid": "https://www.ebay.de/itm/255560186971"}, "176": {"epid": "https://www.ebay.de/itm/334458566197"}, "177": {"epid": "https://www.ebay.de/itm/255560186943"}, "178": {"epid": "https://www.ebay.de/itm/334458566074"}, "179": {"epid": "https://www.ebay.de/itm/234573024649"}, "180": {"epid": "https://www.ebay.de/itm/334458565531"}, "181": {"epid": "https://www.ebay.de/itm/334458565377"}, "182": {"epid": "https://www.ebay.de/itm/334458565401"}}} \ No newline at end of file diff --git a/search_listing.py b/search_listing.py new file mode 100644 index 0000000..b26c335 --- /dev/null +++ b/search_listing.py @@ -0,0 +1,60 @@ + +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" Author: Hendrik Schutter, mail@hendrikschutter.com + Date of creation: 2022/05/31 + Date of last modification: 2022/05/31 +""" + +from bs4 import BeautifulSoup +import datetime +from tinydb import TinyDB, Query +import urllib3 +import sys + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +def make_soup(url): + http = urllib3.PoolManager() + r = http.request("GET", url) + return BeautifulSoup(r.data,'lxml') + +def search_listing(search_term, db, max_pages): + #sort by newest listing + #display page 1 (first) + #EU only + #60 listings in one result page + + page_counter = 1 + last_result_page = False + + while not last_result_page: + result_page_added = 0 + url = 'https://www.ebay.de/sch/i.html?_from=R40&_nkw=' + search_term + '&_sop=10&_pgn='+ str(page_counter) + '&LH_PrefLoc=3&_ipg=60' + #print ("Web Page: ", url) + + soup = make_soup(url) + results = soup.find_all("li", class_="s-item s-item__pl-on-bottom s-item--watch-at-corner") + + for result in results: + try: + rec = { + 'epid': result.div.div.div.a['href'].split("?", 1)[0], + } + #check if listing is allready stored + if not db.search(Query().epid == rec["epid"]): + result_page_added += 1 + db.insert(rec) + + except (AttributeError, KeyError) as ex: + pass + if (result_page_added == 0) or (page_counter == max_pages): + last_result_page = True + page_counter += 1 + + +if __name__ == "__main__": + search_db = TinyDB("search_db.json") + search_listing("mainboard", search_db, max_pages = 4) + print(search_db.all()) +