UI improvments and backend prefetch
This commit is contained in:
@ -17,22 +17,34 @@ import time
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
import threading
|
||||
|
||||
app = Flask(__name__)
|
||||
CORS(app)
|
||||
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
# ZIP code cache file
|
||||
# Configuration
|
||||
CACHE_FILE = "zip_cache.json"
|
||||
zip_cache = {}
|
||||
|
||||
# Active scrape sessions
|
||||
scrape_sessions = {}
|
||||
|
||||
SESSION_TIMEOUT = 300 # seconds
|
||||
LISTINGS_PER_PAGE = 25
|
||||
|
||||
# Global state
|
||||
zip_cache = {}
|
||||
scrape_sessions = {}
|
||||
app_start_time = time.time()
|
||||
|
||||
# Metrics
|
||||
metrics = {
|
||||
"search_requests": 0,
|
||||
"scrape_requests": 0,
|
||||
"kleinanzeigen_response_codes": {},
|
||||
"nominatim_response_codes": {},
|
||||
}
|
||||
|
||||
|
||||
def cleanup_old_sessions():
|
||||
"""Remove sessions older than SESSION_TIMEOUT"""
|
||||
current_time = time.time()
|
||||
sessions_to_remove = []
|
||||
|
||||
@ -63,8 +75,22 @@ def make_soup(url):
|
||||
"""Fetch URL and return BeautifulSoup object"""
|
||||
user_agent = {"user-agent": get_random_user_agent()}
|
||||
http = urllib3.PoolManager(10, headers=user_agent)
|
||||
try:
|
||||
r = http.request("GET", url)
|
||||
# Track response code
|
||||
status_code = str(r.status)
|
||||
if "kleinanzeigen.de" in url:
|
||||
metrics["kleinanzeigen_response_codes"][status_code] = (
|
||||
metrics["kleinanzeigen_response_codes"].get(status_code, 0) + 1
|
||||
)
|
||||
return BeautifulSoup(r.data, "lxml")
|
||||
except Exception as e:
|
||||
print(f"Error fetching {url}: {e}")
|
||||
if "kleinanzeigen.de" in url:
|
||||
metrics["kleinanzeigen_response_codes"]["error"] = (
|
||||
metrics["kleinanzeigen_response_codes"].get("error", 0) + 1
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
def geocode_zip(zip_code):
|
||||
@ -88,6 +114,13 @@ def geocode_zip(zip_code):
|
||||
response = requests.get(
|
||||
url, params=params, headers={"user-agent": get_random_user_agent()}
|
||||
)
|
||||
|
||||
# Track response code
|
||||
status_code = str(response.status_code)
|
||||
metrics["nominatim_response_codes"][status_code] = (
|
||||
metrics["nominatim_response_codes"].get(status_code, 0) + 1
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
|
||||
if data:
|
||||
@ -102,6 +135,9 @@ def geocode_zip(zip_code):
|
||||
return coords
|
||||
except Exception as e:
|
||||
print(f"Geocoding error for {zip_code}: {e}")
|
||||
metrics["nominatim_response_codes"]["error"] = (
|
||||
metrics["nominatim_response_codes"].get("error", 0) + 1
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
@ -136,8 +172,8 @@ def search_listings(search_term, max_pages, min_price, max_price):
|
||||
|
||||
for result in results:
|
||||
try:
|
||||
listing_url = result.a["href"]
|
||||
found_listings.add(base_url + listing_url)
|
||||
listing_href = result.a["href"]
|
||||
found_listings.add(base_url + listing_href)
|
||||
except (AttributeError, KeyError):
|
||||
pass
|
||||
except Exception as e:
|
||||
@ -151,6 +187,7 @@ def scrape_listing(url):
|
||||
"""Scrape individual listing details"""
|
||||
try:
|
||||
soup = make_soup(url)
|
||||
metrics["scrape_requests"] += 1
|
||||
|
||||
title = soup.find("h1", class_="boxedarticle--title")
|
||||
if not title:
|
||||
@ -231,26 +268,61 @@ def scrape_listing(url):
|
||||
return None
|
||||
|
||||
|
||||
def prefetch_listings_thread(session_id):
|
||||
"""Background thread to prefetch all listings"""
|
||||
session = scrape_sessions.get(session_id)
|
||||
if not session:
|
||||
return
|
||||
|
||||
print(f"Starting prefetch for session {session_id}")
|
||||
|
||||
for i, url in enumerate(session["urls"]):
|
||||
# Check if session was cancelled or deleted
|
||||
if (
|
||||
session_id not in scrape_sessions
|
||||
or scrape_sessions[session_id]["cancelled"]
|
||||
):
|
||||
print(f"Prefetch stopped for session {session_id}")
|
||||
return
|
||||
|
||||
listing = scrape_listing(url)
|
||||
if listing:
|
||||
session["listings"].append(listing)
|
||||
session["scraped"] += 1
|
||||
time.sleep(0.3) # Rate limiting
|
||||
|
||||
print(
|
||||
f"Prefetch complete for session {session_id}: {len(session['listings'])} listings"
|
||||
)
|
||||
|
||||
|
||||
@app.route("/api/search", methods=["POST"])
|
||||
def api_search():
|
||||
"""API endpoint for searching listings - returns only count and URLs"""
|
||||
"""API endpoint for searching listings - returns count and starts prefetch"""
|
||||
data = request.json
|
||||
metrics["search_requests"] += 1
|
||||
|
||||
# Cleanup old sessions before creating new one
|
||||
cleanup_old_sessions()
|
||||
|
||||
search_term = data.get("search_term", "")
|
||||
max_pages = data.get("max_pages", 1)
|
||||
num_listings = data.get("num_listings", 25)
|
||||
min_price = data.get("min_price", 0)
|
||||
max_price = data.get("max_price", 10000)
|
||||
max_price = data.get("max_price", 1000000000)
|
||||
|
||||
if not search_term:
|
||||
return jsonify({"error": "Search term is required"}), 400
|
||||
|
||||
# Calculate pages needed
|
||||
max_pages = max(1, (num_listings + LISTINGS_PER_PAGE - 1) // LISTINGS_PER_PAGE)
|
||||
|
||||
try:
|
||||
# Search for listing URLs only
|
||||
listing_urls = search_listings(search_term, max_pages, min_price, max_price)
|
||||
|
||||
# Limit to requested number
|
||||
listing_urls = listing_urls[:num_listings]
|
||||
|
||||
# Create session ID
|
||||
session_id = str(uuid.uuid4())
|
||||
|
||||
@ -264,6 +336,12 @@ def api_search():
|
||||
"created_at": time.time(),
|
||||
}
|
||||
|
||||
# Start prefetch in background thread
|
||||
prefetch_thread = threading.Thread(
|
||||
target=prefetch_listings_thread, args=(session_id,), daemon=True
|
||||
)
|
||||
prefetch_thread.start()
|
||||
|
||||
return jsonify({"session_id": session_id, "total": len(listing_urls)})
|
||||
|
||||
except Exception as e:
|
||||
@ -272,8 +350,7 @@ def api_search():
|
||||
|
||||
@app.route("/api/scrape/<session_id>", methods=["GET"])
|
||||
def api_scrape(session_id):
|
||||
"""API endpoint for scraping next listing in session"""
|
||||
# Cleanup old sessions on each request
|
||||
"""API endpoint to get next scraped listing from session"""
|
||||
cleanup_old_sessions()
|
||||
|
||||
if session_id not in scrape_sessions:
|
||||
@ -284,21 +361,28 @@ def api_scrape(session_id):
|
||||
if session["cancelled"]:
|
||||
return jsonify({"cancelled": True}), 200
|
||||
|
||||
if session["scraped"] >= session["total"]:
|
||||
return jsonify({"complete": True, "listing": None})
|
||||
# Wait briefly if no listings are ready yet
|
||||
wait_count = 0
|
||||
while (
|
||||
len(session["listings"]) == 0
|
||||
and session["scraped"] < session["total"]
|
||||
and wait_count < 10
|
||||
):
|
||||
time.sleep(0.1)
|
||||
wait_count += 1
|
||||
|
||||
# Scrape next listing
|
||||
url = session["urls"][session["scraped"]]
|
||||
listing = scrape_listing(url)
|
||||
if len(session["listings"]) > 0:
|
||||
listing = session["listings"].pop(0)
|
||||
else:
|
||||
listing = None
|
||||
|
||||
if listing:
|
||||
session["listings"].append(listing)
|
||||
|
||||
session["scraped"] += 1
|
||||
is_complete = (
|
||||
session["scraped"] >= session["total"] and len(session["listings"]) == 0
|
||||
)
|
||||
|
||||
return jsonify(
|
||||
{
|
||||
"complete": session["scraped"] >= session["total"],
|
||||
"complete": is_complete,
|
||||
"listing": listing,
|
||||
"progress": {"current": session["scraped"], "total": session["total"]},
|
||||
}
|
||||
@ -307,51 +391,40 @@ def api_scrape(session_id):
|
||||
|
||||
@app.route("/api/scrape/<session_id>/cancel", methods=["POST"])
|
||||
def api_cancel_scrape(session_id):
|
||||
"""API endpoint to cancel scraping session"""
|
||||
"""API endpoint to cancel scraping session and delete cached listings"""
|
||||
cleanup_old_sessions()
|
||||
|
||||
if session_id not in scrape_sessions:
|
||||
return jsonify({"error": "Invalid session ID"}), 404
|
||||
|
||||
scrape_sessions[session_id]["cancelled"] = True
|
||||
# Delete session completely (including cached listings)
|
||||
del scrape_sessions[session_id]
|
||||
|
||||
return jsonify(
|
||||
{
|
||||
"cancelled": True,
|
||||
"listings": scrape_sessions[session_id]["listings"],
|
||||
"total_scraped": len(scrape_sessions[session_id]["listings"]),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@app.route("/api/scrape/<session_id>/results", methods=["GET"])
|
||||
def api_get_results(session_id):
|
||||
"""API endpoint to get all scraped results"""
|
||||
cleanup_old_sessions()
|
||||
|
||||
if session_id not in scrape_sessions:
|
||||
return jsonify({"error": "Invalid session ID"}), 404
|
||||
|
||||
session = scrape_sessions[session_id]
|
||||
|
||||
return jsonify(
|
||||
{
|
||||
"listings": session["listings"],
|
||||
"total": len(session["listings"]),
|
||||
"progress": {"current": session["scraped"], "total": session["total"]},
|
||||
}
|
||||
)
|
||||
return jsonify({"cancelled": True, "message": "Session deleted"})
|
||||
|
||||
|
||||
@app.route("/api/health", methods=["GET"])
|
||||
def health():
|
||||
"""Health check endpoint"""
|
||||
return jsonify({"status": "ok"})
|
||||
|
||||
|
||||
@app.route("/api/metrics", methods=["GET"])
|
||||
def api_metrics():
|
||||
"""Prometheus-style metrics endpoint"""
|
||||
cleanup_old_sessions()
|
||||
|
||||
uptime = time.time() - app_start_time
|
||||
|
||||
return jsonify(
|
||||
{
|
||||
"status": "ok",
|
||||
"cache_size": len(zip_cache),
|
||||
"search_requests_total": metrics["search_requests"],
|
||||
"scrape_requests_total": metrics["scrape_requests"],
|
||||
"uptime_seconds": uptime,
|
||||
"kleinanzeigen_response_codes": metrics["kleinanzeigen_response_codes"],
|
||||
"nominatim_response_codes": metrics["nominatim_response_codes"],
|
||||
"active_sessions": len(scrape_sessions),
|
||||
"cache_size": len(zip_cache),
|
||||
}
|
||||
)
|
||||
|
||||
@ -365,5 +438,4 @@ if __name__ == "__main__":
|
||||
zip_cache = json.load(f)
|
||||
|
||||
print(f"Loaded {len(zip_cache)} ZIP codes from cache")
|
||||
print("ZIP code cache loaded with", len(zip_cache), "entries")
|
||||
app.run(debug=True, host="0.0.0.0", port=5000)
|
||||
app.run(debug=True, host="0.0.0.0", port=5000, threaded=True)
|
||||
|
||||
@ -1370,5 +1370,233 @@
|
||||
"20457": {
|
||||
"lat": 53.5335376,
|
||||
"lon": 9.9806284
|
||||
},
|
||||
"39120": {
|
||||
"lat": 52.0855315,
|
||||
"lon": 11.6329414
|
||||
},
|
||||
"47137": {
|
||||
"lat": 51.4714791,
|
||||
"lon": 6.7669058
|
||||
},
|
||||
"69207": {
|
||||
"lat": 49.3421242,
|
||||
"lon": 8.6370166
|
||||
},
|
||||
"27616": {
|
||||
"lat": 53.4469731,
|
||||
"lon": 8.8063121
|
||||
},
|
||||
"10119": {
|
||||
"lat": 52.5301255,
|
||||
"lon": 13.4055082
|
||||
},
|
||||
"47443": {
|
||||
"lat": 51.4656893,
|
||||
"lon": 6.6524975
|
||||
},
|
||||
"10557": {
|
||||
"lat": 52.5256483,
|
||||
"lon": 13.3640508
|
||||
},
|
||||
"37671": {
|
||||
"lat": 51.7685153,
|
||||
"lon": 9.3310829
|
||||
},
|
||||
"16225": {
|
||||
"lat": 52.8292295,
|
||||
"lon": 13.8384643
|
||||
},
|
||||
"52146": {
|
||||
"lat": 50.8292627,
|
||||
"lon": 6.1519607
|
||||
},
|
||||
"48149": {
|
||||
"lat": 51.9638718,
|
||||
"lon": 7.6026944
|
||||
},
|
||||
"48653": {
|
||||
"lat": 51.9171483,
|
||||
"lon": 7.1606437
|
||||
},
|
||||
"71134": {
|
||||
"lat": 48.6820157,
|
||||
"lon": 8.8818568
|
||||
},
|
||||
"53111": {
|
||||
"lat": 50.7402492,
|
||||
"lon": 7.0985907
|
||||
},
|
||||
"01993": {
|
||||
"lat": 51.5082352,
|
||||
"lon": 13.8868418
|
||||
},
|
||||
"32105": {
|
||||
"lat": 52.0890558,
|
||||
"lon": 8.7396016
|
||||
},
|
||||
"82445": {
|
||||
"lat": 47.6253557,
|
||||
"lon": 11.1138077
|
||||
},
|
||||
"40217": {
|
||||
"lat": 51.2131204,
|
||||
"lon": 6.774469
|
||||
},
|
||||
"49401": {
|
||||
"lat": 52.5279348,
|
||||
"lon": 8.232843
|
||||
},
|
||||
"55268": {
|
||||
"lat": 49.90063,
|
||||
"lon": 8.203086
|
||||
},
|
||||
"24306": {
|
||||
"lat": 54.1620728,
|
||||
"lon": 10.4375556
|
||||
},
|
||||
"22763": {
|
||||
"lat": 53.550853,
|
||||
"lon": 9.9138756
|
||||
},
|
||||
"21339": {
|
||||
"lat": 53.2548482,
|
||||
"lon": 10.3911518
|
||||
},
|
||||
"56218": {
|
||||
"lat": 50.388183,
|
||||
"lon": 7.5046437
|
||||
},
|
||||
"86899": {
|
||||
"lat": 48.0336819,
|
||||
"lon": 10.8638784
|
||||
},
|
||||
"84034": {
|
||||
"lat": 48.5286909,
|
||||
"lon": 12.0999127
|
||||
},
|
||||
"82110": {
|
||||
"lat": 48.1321961,
|
||||
"lon": 11.3600169
|
||||
},
|
||||
"56626": {
|
||||
"lat": 50.431234,
|
||||
"lon": 7.3730436
|
||||
},
|
||||
"10315": {
|
||||
"lat": 52.5180707,
|
||||
"lon": 13.5144045
|
||||
},
|
||||
"52080": {
|
||||
"lat": 50.784851,
|
||||
"lon": 6.160716
|
||||
},
|
||||
"51688": {
|
||||
"lat": 51.1169225,
|
||||
"lon": 7.419399
|
||||
},
|
||||
"45127": {
|
||||
"lat": 51.4574619,
|
||||
"lon": 7.0103435
|
||||
},
|
||||
"48324": {
|
||||
"lat": 51.8545927,
|
||||
"lon": 7.7859503
|
||||
},
|
||||
"26386": {
|
||||
"lat": 53.553156,
|
||||
"lon": 8.1039435
|
||||
},
|
||||
"86356": {
|
||||
"lat": 48.392226,
|
||||
"lon": 10.8016665
|
||||
},
|
||||
"50939": {
|
||||
"lat": 50.9095331,
|
||||
"lon": 6.9259241
|
||||
},
|
||||
"14195": {
|
||||
"lat": 52.4585754,
|
||||
"lon": 13.2846329
|
||||
},
|
||||
"21680": {
|
||||
"lat": 53.5904569,
|
||||
"lon": 9.4760161
|
||||
},
|
||||
"01257": {
|
||||
"lat": 50.9983029,
|
||||
"lon": 13.8123958
|
||||
},
|
||||
"29410": {
|
||||
"lat": 52.8367097,
|
||||
"lon": 11.1224073
|
||||
},
|
||||
"38300": {
|
||||
"lat": 52.1513013,
|
||||
"lon": 10.56812
|
||||
},
|
||||
"01819": {
|
||||
"lat": 50.8809928,
|
||||
"lon": 13.9044785
|
||||
},
|
||||
"85238": {
|
||||
"lat": 48.4082502,
|
||||
"lon": 11.4634544
|
||||
},
|
||||
"33378": {
|
||||
"lat": 51.8441657,
|
||||
"lon": 8.317883
|
||||
},
|
||||
"99192": {
|
||||
"lat": 50.934868,
|
||||
"lon": 10.9138336
|
||||
},
|
||||
"60438": {
|
||||
"lat": 50.1786706,
|
||||
"lon": 8.6271811
|
||||
},
|
||||
"35075": {
|
||||
"lat": 50.7787074,
|
||||
"lon": 8.5791746
|
||||
},
|
||||
"10827": {
|
||||
"lat": 52.4836896,
|
||||
"lon": 13.3528221
|
||||
},
|
||||
"24392": {
|
||||
"lat": 54.6331185,
|
||||
"lon": 9.7771951
|
||||
},
|
||||
"78647": {
|
||||
"lat": 48.0717008,
|
||||
"lon": 8.6373591
|
||||
},
|
||||
"10627": {
|
||||
"lat": 52.5075196,
|
||||
"lon": 13.3031999
|
||||
},
|
||||
"22419": {
|
||||
"lat": 53.6662872,
|
||||
"lon": 10.0055952
|
||||
},
|
||||
"06388": {
|
||||
"lat": 51.6898413,
|
||||
"lon": 11.9119914
|
||||
},
|
||||
"67117": {
|
||||
"lat": 49.4122026,
|
||||
"lon": 8.3936093
|
||||
},
|
||||
"68219": {
|
||||
"lat": 49.435092,
|
||||
"lon": 8.5365013
|
||||
},
|
||||
"77866": {
|
||||
"lat": 48.6610036,
|
||||
"lon": 7.9359671
|
||||
},
|
||||
"53175": {
|
||||
"lat": 50.6989638,
|
||||
"lon": 7.1445107
|
||||
}
|
||||
}
|
||||
336
web/index.html
336
web/index.html
@ -1,9 +1,9 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<html lang="de">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Kleinanzeigen Map Search</title>
|
||||
<title>Kleinanzeigen Karten-Suche</title>
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/leaflet/1.9.4/leaflet.min.css" />
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/leaflet/1.9.4/leaflet.min.js"></script>
|
||||
<style>
|
||||
@ -17,93 +17,153 @@
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
height: 100vh;
|
||||
overflow: hidden;
|
||||
background: #1a1a1a;
|
||||
color: #e0e0e0;
|
||||
}
|
||||
|
||||
.container {
|
||||
display: grid;
|
||||
grid-template-columns: 350px 1fr;
|
||||
grid-template-columns: 380px 1fr;
|
||||
grid-template-rows: auto 1fr;
|
||||
height: 100vh;
|
||||
}
|
||||
|
||||
.search-bar {
|
||||
grid-column: 1 / -1;
|
||||
background: #fff;
|
||||
padding: 15px 20px;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||||
background: #242424;
|
||||
padding: 20px;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.3);
|
||||
display: grid;
|
||||
grid-template-columns: 1fr auto auto auto auto;
|
||||
gap: 12px;
|
||||
align-items: end;
|
||||
}
|
||||
|
||||
.form-group {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.form-group label {
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
color: #b0b0b0;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.search-bar input, .search-bar select {
|
||||
padding: 8px 12px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
padding: 12px;
|
||||
border: 1px solid #3a3a3a;
|
||||
border-radius: 6px;
|
||||
font-size: 14px;
|
||||
background: #2a2a2a;
|
||||
color: #e0e0e0;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.search-bar input[type="text"] {
|
||||
flex: 1;
|
||||
min-width: 200px;
|
||||
.search-bar input:focus, .search-bar select:focus {
|
||||
outline: none;
|
||||
border-color: #0ea5e9;
|
||||
box-shadow: 0 0 0 3px rgba(14, 165, 233, 0.1);
|
||||
}
|
||||
|
||||
.search-bar input[type="number"] {
|
||||
width: 100px;
|
||||
.price-inputs {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.price-inputs input {
|
||||
width: 120px;
|
||||
}
|
||||
|
||||
.price-separator {
|
||||
color: #666;
|
||||
font-weight: 600;
|
||||
padding: 0 4px;
|
||||
}
|
||||
|
||||
.search-bar button {
|
||||
padding: 8px 20px;
|
||||
background: #0066cc;
|
||||
padding: 12px 24px;
|
||||
background: linear-gradient(135deg, #0ea5e9 0%, #0284c7 100%);
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
font-weight: 600;
|
||||
font-size: 14px;
|
||||
transition: all 0.2s;
|
||||
box-shadow: 0 2px 8px rgba(14, 165, 233, 0.3);
|
||||
}
|
||||
|
||||
.search-bar button:hover:not(:disabled) {
|
||||
background: #0052a3;
|
||||
transform: translateY(-1px);
|
||||
box-shadow: 0 4px 12px rgba(14, 165, 233, 0.4);
|
||||
}
|
||||
|
||||
.search-bar button:disabled {
|
||||
background: #ccc;
|
||||
background: #3a3a3a;
|
||||
cursor: not-allowed;
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.search-bar button.cancel {
|
||||
background: #dc3545;
|
||||
background: linear-gradient(135deg, #ef4444 0%, #dc2626 100%);
|
||||
box-shadow: 0 2px 8px rgba(239, 68, 68, 0.3);
|
||||
}
|
||||
|
||||
.search-bar button.cancel:hover {
|
||||
background: #c82333;
|
||||
box-shadow: 0 4px 12px rgba(239, 68, 68, 0.4);
|
||||
}
|
||||
|
||||
.results-panel {
|
||||
background: #f8f9fa;
|
||||
background: #1e1e1e;
|
||||
overflow-y: auto;
|
||||
border-right: 1px solid #ddd;
|
||||
border-right: 1px solid #2a2a2a;
|
||||
}
|
||||
|
||||
.results-panel::-webkit-scrollbar {
|
||||
width: 8px;
|
||||
}
|
||||
|
||||
.results-panel::-webkit-scrollbar-track {
|
||||
background: #1a1a1a;
|
||||
}
|
||||
|
||||
.results-panel::-webkit-scrollbar-thumb {
|
||||
background: #3a3a3a;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.results-panel::-webkit-scrollbar-thumb:hover {
|
||||
background: #4a4a4a;
|
||||
}
|
||||
|
||||
.results-header {
|
||||
background: white;
|
||||
padding: 15px 20px;
|
||||
border-bottom: 1px solid #ddd;
|
||||
background: #242424;
|
||||
padding: 20px;
|
||||
border-bottom: 1px solid #2a2a2a;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 10;
|
||||
}
|
||||
|
||||
.results-count {
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
margin-bottom: 10px;
|
||||
font-weight: 700;
|
||||
color: #e0e0e0;
|
||||
margin-bottom: 12px;
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
.progress-info {
|
||||
background: #e3f2fd;
|
||||
padding: 12px;
|
||||
border-radius: 4px;
|
||||
margin-bottom: 10px;
|
||||
background: rgba(14, 165, 233, 0.1);
|
||||
padding: 14px;
|
||||
border-radius: 6px;
|
||||
margin-bottom: 12px;
|
||||
display: none;
|
||||
border: 1px solid rgba(14, 165, 233, 0.2);
|
||||
}
|
||||
|
||||
.progress-info.active {
|
||||
@ -112,99 +172,107 @@
|
||||
|
||||
.progress-bar {
|
||||
width: 100%;
|
||||
height: 8px;
|
||||
background: #e0e0e0;
|
||||
border-radius: 4px;
|
||||
height: 6px;
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
border-radius: 3px;
|
||||
overflow: hidden;
|
||||
margin-top: 8px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.progress-fill {
|
||||
height: 100%;
|
||||
background: #0066cc;
|
||||
background: linear-gradient(90deg, #0ea5e9 0%, #06b6d4 100%);
|
||||
width: 0%;
|
||||
transition: width 0.3s;
|
||||
box-shadow: 0 0 10px rgba(14, 165, 233, 0.5);
|
||||
}
|
||||
|
||||
.progress-text {
|
||||
font-size: 12px;
|
||||
color: #1565c0;
|
||||
margin-bottom: 4px;
|
||||
font-size: 13px;
|
||||
color: #0ea5e9;
|
||||
margin-bottom: 6px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.eta-text {
|
||||
font-size: 11px;
|
||||
color: #666;
|
||||
color: #888;
|
||||
margin-top: 6px;
|
||||
}
|
||||
|
||||
.sort-control {
|
||||
display: flex;
|
||||
gap: 5px;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.sort-control label {
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
font-size: 12px;
|
||||
color: #888;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.sort-control select {
|
||||
padding: 5px 8px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
padding: 8px 12px;
|
||||
border: 1px solid #3a3a3a;
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
background: #2a2a2a;
|
||||
color: #e0e0e0;
|
||||
}
|
||||
|
||||
.result-item {
|
||||
background: white;
|
||||
margin: 10px;
|
||||
background: #242424;
|
||||
margin: 12px;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
|
||||
border: 1px solid #2a2a2a;
|
||||
}
|
||||
|
||||
.result-item:hover {
|
||||
box-shadow: 0 4px 12px rgba(0,0,0,0.15);
|
||||
border-color: #0ea5e9;
|
||||
box-shadow: 0 4px 16px rgba(14, 165, 233, 0.2);
|
||||
transform: translateY(-2px);
|
||||
}
|
||||
|
||||
.result-item.selected {
|
||||
border: 2px solid #0066cc;
|
||||
border-color: #0ea5e9;
|
||||
box-shadow: 0 0 0 2px rgba(14, 165, 233, 0.3);
|
||||
}
|
||||
|
||||
.result-image {
|
||||
width: 100%;
|
||||
height: 180px;
|
||||
object-fit: cover;
|
||||
background: #e9ecef;
|
||||
background: #1a1a1a;
|
||||
}
|
||||
|
||||
.result-content {
|
||||
padding: 12px;
|
||||
padding: 14px;
|
||||
}
|
||||
|
||||
.result-title {
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
margin-bottom: 8px;
|
||||
color: #e0e0e0;
|
||||
margin-bottom: 10px;
|
||||
font-size: 14px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.result-price {
|
||||
color: #0066cc;
|
||||
color: #0ea5e9;
|
||||
font-weight: 700;
|
||||
font-size: 18px;
|
||||
margin-bottom: 8px;
|
||||
font-size: 20px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.result-meta {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
font-size: 12px;
|
||||
color: #666;
|
||||
color: #888;
|
||||
}
|
||||
|
||||
.result-location {
|
||||
@ -220,24 +288,27 @@
|
||||
.map-container {
|
||||
position: relative;
|
||||
height: 100%;
|
||||
background: #1a1a1a;
|
||||
}
|
||||
|
||||
#map {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
filter: brightness(0.9) contrast(1.1);
|
||||
}
|
||||
|
||||
.status-bar {
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
top: 20px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
background: white;
|
||||
padding: 10px 20px;
|
||||
border-radius: 4px;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.2);
|
||||
background: #242424;
|
||||
padding: 14px 24px;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 4px 16px rgba(0,0,0,0.4);
|
||||
z-index: 1000;
|
||||
display: none;
|
||||
border: 1px solid #3a3a3a;
|
||||
}
|
||||
|
||||
.status-bar.visible {
|
||||
@ -245,35 +316,39 @@
|
||||
}
|
||||
|
||||
.status-bar.loading {
|
||||
background: #e3f2fd;
|
||||
color: #1565c0;
|
||||
background: rgba(14, 165, 233, 0.2);
|
||||
color: #0ea5e9;
|
||||
border-color: rgba(14, 165, 233, 0.3);
|
||||
}
|
||||
|
||||
.status-bar.success {
|
||||
background: #e8f5e9;
|
||||
color: #2e7d32;
|
||||
background: rgba(34, 197, 94, 0.2);
|
||||
color: #22c55e;
|
||||
border-color: rgba(34, 197, 94, 0.3);
|
||||
}
|
||||
|
||||
.status-bar.error {
|
||||
background: #ffebee;
|
||||
color: #c62828;
|
||||
background: rgba(239, 68, 68, 0.2);
|
||||
color: #ef4444;
|
||||
border-color: rgba(239, 68, 68, 0.3);
|
||||
}
|
||||
|
||||
.no-results {
|
||||
text-align: center;
|
||||
padding: 40px 20px;
|
||||
padding: 60px 20px;
|
||||
color: #666;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.loading-spinner {
|
||||
display: inline-block;
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
border: 2px solid #1565c0;
|
||||
border: 2px solid #0ea5e9;
|
||||
border-radius: 50%;
|
||||
border-top-color: transparent;
|
||||
animation: spin 0.8s linear infinite;
|
||||
margin-right: 8px;
|
||||
margin-right: 10px;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
@ -282,7 +357,15 @@
|
||||
|
||||
@media (max-width: 1024px) {
|
||||
.container {
|
||||
grid-template-columns: 300px 1fr;
|
||||
grid-template-columns: 320px 1fr;
|
||||
}
|
||||
|
||||
.search-bar {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.price-inputs input {
|
||||
width: 100px;
|
||||
}
|
||||
}
|
||||
|
||||
@ -301,20 +384,40 @@
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="search-bar">
|
||||
<input type="text" id="searchTerm" placeholder="Search term (e.g., Cube Nuroad)" value="Fahrrad">
|
||||
<input type="number" id="minPrice" placeholder="Min €" value="300" min="0">
|
||||
<input type="number" id="maxPrice" placeholder="Max €" value="900" min="0">
|
||||
<input type="number" id="maxPages" placeholder="Pages" value="1" min="1" max="20">
|
||||
<button id="searchBtn">Search</button>
|
||||
<button id="cancelBtn" class="cancel" style="display: none;">Cancel</button>
|
||||
<div class="form-group">
|
||||
<label>Suchbegriff</label>
|
||||
<input type="text" id="searchTerm" placeholder="z.B. Fahrrad" value="Fahrrad">
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label>Preisspanne</label>
|
||||
<div class="price-inputs">
|
||||
<input type="number" id="minPrice" placeholder="0 €" value="0" min="0" max="1000000000">
|
||||
<span class="price-separator">–</span>
|
||||
<input type="number" id="maxPrice" placeholder="∞ €" value="" min="0" max="1000000000">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label>Anzahl Inserate</label>
|
||||
<select id="numListings">
|
||||
<option value="25" selected>25 Inserate</option>
|
||||
<option value="50">50 Inserate</option>
|
||||
<option value="100">100 Inserate</option>
|
||||
<option value="250">250 Inserate</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<button id="searchBtn">Suchen</button>
|
||||
<button id="cancelBtn" class="cancel" style="display: none;">Abbrechen</button>
|
||||
</div>
|
||||
|
||||
<div class="results-panel">
|
||||
<div class="results-header">
|
||||
<div class="results-count">No results</div>
|
||||
<div class="results-count">Keine Ergebnisse</div>
|
||||
|
||||
<div id="progressInfo" class="progress-info">
|
||||
<div class="progress-text">Scraping listings...</div>
|
||||
<div class="progress-text">Inserate werden geladen...</div>
|
||||
<div class="progress-bar">
|
||||
<div class="progress-fill"></div>
|
||||
</div>
|
||||
@ -322,12 +425,12 @@
|
||||
</div>
|
||||
|
||||
<div class="sort-control">
|
||||
<label>Sort:</label>
|
||||
<label>Sortierung:</label>
|
||||
<select id="sortSelect">
|
||||
<option value="date-desc">Date (newest)</option>
|
||||
<option value="date-asc">Date (oldest)</option>
|
||||
<option value="price-asc">Price (low to high)</option>
|
||||
<option value="price-desc">Price (high to low)</option>
|
||||
<option value="date-desc">Datum (neueste)</option>
|
||||
<option value="date-asc">Datum (älteste)</option>
|
||||
<option value="price-asc">Preis (niedrig → hoch)</option>
|
||||
<option value="price-desc">Preis (hoch → niedrig)</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
@ -392,7 +495,7 @@
|
||||
progressInfo.classList.add('active');
|
||||
const percentage = (current / total) * 100;
|
||||
progressFill.style.width = percentage + '%';
|
||||
progressText.textContent = `Scraping listings: ${current}/${total}`;
|
||||
progressText.textContent = `Inserate werden geladen: ${current}/${total}`;
|
||||
|
||||
// Calculate ETA
|
||||
if (scrapeStartTime && current > 0) {
|
||||
@ -405,9 +508,9 @@
|
||||
const seconds = etaSeconds % 60;
|
||||
|
||||
if (minutes > 0) {
|
||||
etaText.textContent = `ETA: ~${minutes}m ${seconds}s`;
|
||||
etaText.textContent = `Noch ca. ${minutes}m ${seconds}s`;
|
||||
} else {
|
||||
etaText.textContent = `ETA: ~${seconds}s`;
|
||||
etaText.textContent = `Noch ca. ${seconds}s`;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -429,12 +532,12 @@
|
||||
: '';
|
||||
|
||||
const popupContent = `
|
||||
<div style="min-width: 200px;">
|
||||
<div style="min-width: 200px; 8px;">
|
||||
<strong style="font-size: 14px;">${listing.title}</strong><br>
|
||||
${imageHtml}
|
||||
<span style="color: #0066cc; font-weight: bold; font-size: 16px;">€${listing.price}</span><br>
|
||||
<span style="color: #0066cc; font-weight: bold; font-size: 16px;">${listing.price} €</span><br>
|
||||
<span style="color: #666; font-size: 12px;">${listing.address}</span><br>
|
||||
<a href="${listing.url}" target="_blank" style="color: #0066cc; text-decoration: none; font-weight: 600;">Open in new tab →</a>
|
||||
<a href="${listing.url}" target="_blank" style="color: #0066cc; text-decoration: none; font-weight: 600;">Link öffnen →</a>
|
||||
</div>
|
||||
`;
|
||||
|
||||
@ -463,7 +566,7 @@
|
||||
|
||||
// Format date
|
||||
function formatDate(dateString) {
|
||||
if (!dateString) return 'Unknown date';
|
||||
if (!dateString) return 'Unbekanntes Datum';
|
||||
const date = new Date(dateString);
|
||||
return date.toLocaleDateString('de-DE');
|
||||
}
|
||||
@ -474,19 +577,19 @@
|
||||
const resultsCount = document.querySelector('.results-count');
|
||||
|
||||
if (listings.length === 0) {
|
||||
resultsList.innerHTML = '<div class="no-results">No listings found</div>';
|
||||
resultsCount.textContent = 'No results';
|
||||
resultsList.innerHTML = '<div class="no-results">Keine Inserate gefunden</div>';
|
||||
resultsCount.textContent = 'Keine Ergebnisse';
|
||||
return;
|
||||
}
|
||||
|
||||
resultsCount.textContent = `${listings.length} result${listings.length !== 1 ? 's' : ''}`;
|
||||
resultsCount.textContent = `${listings.length} Inserat${listings.length !== 1 ? 'e' : ''}`;
|
||||
|
||||
resultsList.innerHTML = listings.map(listing => `
|
||||
<div class="result-item" data-id="${listing.id}">
|
||||
${listing.image ? `<img src="${listing.image}" class="result-image" alt="${listing.title}">` : '<div class="result-image"></div>'}
|
||||
<div class="result-content">
|
||||
<div class="result-title">${listing.title}</div>
|
||||
<div class="result-price">€${listing.price}</div>
|
||||
<div class="result-price">${listing.price} €</div>
|
||||
<div class="result-meta">
|
||||
<div class="result-location">
|
||||
<span>📍</span>
|
||||
@ -518,7 +621,6 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Open listing in new tab
|
||||
window.open(listing.url, '_blank');
|
||||
}
|
||||
});
|
||||
@ -587,10 +689,7 @@
|
||||
document.getElementById('searchBtn').disabled = false;
|
||||
document.getElementById('cancelBtn').style.display = 'none';
|
||||
updateProgress(0, 0);
|
||||
showStatus(`Completed! Scraped ${allListings.length} listings`, 'success');
|
||||
|
||||
// DO NOT clear session ID - keep it for potential future use
|
||||
// DO NOT reset listings or markers
|
||||
showStatus(`Fertig! ${allListings.length} Inserate geladen`, 'success');
|
||||
console.log('Final listings count:', allListings.length);
|
||||
return false;
|
||||
}
|
||||
@ -602,7 +701,7 @@
|
||||
isScrapingActive = false;
|
||||
document.getElementById('searchBtn').disabled = false;
|
||||
document.getElementById('cancelBtn').style.display = 'none';
|
||||
showStatus('Error occurred during scraping', 'error');
|
||||
showStatus('Fehler beim Laden der Inserate', 'error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -622,11 +721,12 @@
|
||||
async function searchListings() {
|
||||
const searchTerm = document.getElementById('searchTerm').value.trim();
|
||||
const minPrice = parseInt(document.getElementById('minPrice').value) || 0;
|
||||
const maxPrice = parseInt(document.getElementById('maxPrice').value) || 10000;
|
||||
const maxPages = parseInt(document.getElementById('maxPages').value) || 5;
|
||||
const maxPriceInput = document.getElementById('maxPrice').value;
|
||||
const maxPrice = maxPriceInput ? parseInt(maxPriceInput) : 1000000000;
|
||||
const numListings = parseInt(document.getElementById('numListings').value) || 25;
|
||||
|
||||
if (!searchTerm) {
|
||||
showStatus('Please enter a search term', 'error');
|
||||
showStatus('Bitte Suchbegriff eingeben', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
@ -636,7 +736,7 @@
|
||||
selectedListingId = null;
|
||||
document.getElementById('resultsList').innerHTML = '';
|
||||
|
||||
showStatus('Searching for listings...', 'loading');
|
||||
showStatus('Suche nach Inseraten...', 'loading');
|
||||
|
||||
try {
|
||||
const response = await fetch(`${API_BASE_URL}/api/search`, {
|
||||
@ -648,7 +748,7 @@
|
||||
search_term: searchTerm,
|
||||
min_price: minPrice,
|
||||
max_price: maxPrice,
|
||||
max_pages: maxPages
|
||||
num_listings: numListings
|
||||
})
|
||||
});
|
||||
|
||||
@ -660,12 +760,12 @@
|
||||
currentSessionId = data.session_id;
|
||||
|
||||
if (data.total === 0) {
|
||||
showStatus('No listings found', 'error');
|
||||
showStatus('Keine Inserate gefunden', 'error');
|
||||
document.getElementById('searchBtn').disabled = false;
|
||||
return;
|
||||
}
|
||||
|
||||
showStatus(`Found ${data.total} listings. Starting scrape...`, 'success');
|
||||
showStatus(`${data.total} Inserate gefunden. Lade Details...`, 'success');
|
||||
|
||||
// Show cancel button
|
||||
document.getElementById('cancelBtn').style.display = 'inline-block';
|
||||
@ -678,7 +778,7 @@
|
||||
|
||||
} catch (error) {
|
||||
console.error('Search error:', error);
|
||||
showStatus('Error: Could not connect to API server', 'error');
|
||||
showStatus('Fehler: Verbindung zum Server fehlgeschlagen', 'error');
|
||||
document.getElementById('searchBtn').disabled = false;
|
||||
}
|
||||
}
|
||||
@ -696,7 +796,7 @@
|
||||
document.getElementById('searchBtn').disabled = false;
|
||||
document.getElementById('cancelBtn').style.display = 'none';
|
||||
updateProgress(0, 0);
|
||||
showStatus(`Cancelled. Showing ${allListings.length} scraped listings`, 'error');
|
||||
showStatus(`Abgebrochen. ${allListings.length} Inserate geladen`, 'error');
|
||||
|
||||
} catch (error) {
|
||||
console.error('Cancel error:', error);
|
||||
|
||||
Reference in New Issue
Block a user