fix parsing
This commit is contained in:
+78
-77
@@ -306,6 +306,33 @@ def flatten_dataset(record: dict) -> dict[str, str]:
|
||||
return fields
|
||||
|
||||
|
||||
# A real dataset's Data array bundles several sub-reports concatenated
|
||||
# together, so these two field names each occur multiple times per dataset
|
||||
# with unrelated values (confirmed via their distinct per-entry "key"
|
||||
# UUIDs) - flatten_dataset()'s last-value-wins is unsafe for them.
|
||||
SIGNAL_TIMESTAMP_FIELDS = ("car_captured_time", "car_captured_utc_timestamp")
|
||||
|
||||
|
||||
def latest_signal_timestamp_unix(record: dict) -> float | None:
|
||||
"""Returns the most recent unix timestamp found under any of
|
||||
SIGNAL_TIMESTAMP_FIELDS in the raw Data array, or ``None`` if none are
|
||||
present/parseable."""
|
||||
best = None
|
||||
for entry in record.get("Data", []):
|
||||
if entry.get("dataFieldName") not in SIGNAL_TIMESTAMP_FIELDS:
|
||||
continue
|
||||
raw = entry.get("value")
|
||||
if not raw:
|
||||
continue
|
||||
try:
|
||||
unix = parse_iso8601_to_unix(raw)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if best is None or unix > best:
|
||||
best = unix
|
||||
return best
|
||||
|
||||
|
||||
def persist_raw_dataset(vin: str, record: dict, *, suffix: str = "") -> None:
|
||||
"""Writes the JSON extracted from a downloaded dataset to disk, so it
|
||||
can be inspected later to refine the field mapping below. Only the
|
||||
@@ -320,11 +347,12 @@ def persist_raw_dataset(vin: str, record: dict, *, suffix: str = "") -> None:
|
||||
log.debug("Persisted raw dataset for VIN %s to %s", vin, dest)
|
||||
|
||||
|
||||
def fetch_latest_dataset_fields(vin: str) -> dict[str, str] | None:
|
||||
def fetch_latest_dataset_fields(vin: str) -> dict[str, str | float | None] | None:
|
||||
"""Logs in, downloads the latest dataset for *vin* and returns the
|
||||
flattened fields. Returns ``None`` if no datasets are (yet) available -
|
||||
this is explicitly NOT a failure. Raises an exception on login/HTTP/parse
|
||||
errors (= failure)."""
|
||||
flattened fields, plus a synthetic "_signal_timestamp_unix" key (see
|
||||
latest_signal_timestamp_unix()). Returns ``None`` if no datasets are
|
||||
(yet) available - this is explicitly NOT a failure. Raises an exception
|
||||
on login/HTTP/parse errors (= failure)."""
|
||||
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": random.choice(config.user_agents)})
|
||||
@@ -385,7 +413,9 @@ def fetch_latest_dataset_fields(vin: str) -> dict[str, str] | None:
|
||||
except OSError as exc:
|
||||
log.warning("Could not persist raw dataset for VIN %s: %s", vin, exc)
|
||||
|
||||
return flatten_dataset(record)
|
||||
fields = flatten_dataset(record)
|
||||
fields["_signal_timestamp_unix"] = latest_signal_timestamp_unix(record)
|
||||
return fields
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -454,7 +484,7 @@ g_position_created_timestamp_seconds = Gauge(
|
||||
)
|
||||
g_locked = Gauge(
|
||||
f"{P}locked",
|
||||
"Combined lock status of all doors, trunk and hood (1=locked)",
|
||||
"Vehicle locked (1=locked), as a single combined flag from the portal",
|
||||
["vin"],
|
||||
)
|
||||
g_is_parked = Gauge(f"{P}is_parked", "Vehicle parked (1=yes)", ["vin"])
|
||||
@@ -466,7 +496,8 @@ g_service_due_in_days = Gauge(
|
||||
)
|
||||
g_last_vehicle_signal_timestamp_seconds = Gauge(
|
||||
f"{P}last_vehicle_signal_timestamp_seconds",
|
||||
"Unix timestamp of the last vehicle signal (carCapturedUTCTimestamp)",
|
||||
"Unix timestamp of the most recent vehicle signal in the latest dataset "
|
||||
"(max of all car_captured_time/car_captured_utc_timestamp entries)",
|
||||
["vin"],
|
||||
)
|
||||
|
||||
@@ -487,18 +518,19 @@ c_http_requests_total = Counter(
|
||||
["vin"],
|
||||
)
|
||||
|
||||
# Enum metrics. The state space is best-effort (compiled from the sample
|
||||
# dataset and comparable projects) plus an "UNKNOWN" fallback for unknown raw
|
||||
# values - these are additionally logged as WARNING so the list can be
|
||||
# extended if needed.
|
||||
# Enum metrics. driving_mode/plug_connection_state/next_service_type are not
|
||||
# delivered by the "continuous" data feed at all (confirmed against 255 real
|
||||
# datasets - see README) and are kept only for API completeness; they will
|
||||
# never actually populate. charging_state's states below are the values
|
||||
# actually observed in those 255 real datasets, plus an "UNKNOWN" fallback
|
||||
# for anything new - unknown raw values are additionally logged as WARNING
|
||||
# so the list can be extended if needed.
|
||||
DRIVING_MODE_STATES = ["standard", "eco", "comfort", "sport", "individual", "UNKNOWN"]
|
||||
CHARGING_STATE_STATES = [
|
||||
"OFF",
|
||||
"READY_FOR_CHARGING",
|
||||
"NOT_READY_FOR_CHARGING",
|
||||
"CHARGING",
|
||||
"CONSERVING",
|
||||
"ERROR",
|
||||
"CHARGE_STATE_NOT_READY_FOR_CHARGING",
|
||||
"CHARGE_STATE_READY_FOR_CHARGING",
|
||||
"CHARGE_STATE_CHARGING_HV_BATTERY",
|
||||
"CHARGE_STATE_CHARGE_PURPOSE_REACHED_AND_NOT_CONSERVATION_CHARGING",
|
||||
"UNKNOWN",
|
||||
]
|
||||
PLUG_CONNECTION_STATES = ["CONNECTED", "DISCONNECTED", "UNKNOWN"]
|
||||
@@ -532,59 +564,38 @@ e_service_type = Enum(
|
||||
states=SERVICE_TYPE_STATES,
|
||||
)
|
||||
|
||||
# Field name (dataFieldName in the dataset) -> (Gauge, conversion function)
|
||||
# Field name (dataFieldName in the dataset) -> (Gauge, conversion function).
|
||||
# Verified against 255 real datasets in raw_json/: each of these fields has
|
||||
# exactly one distinct originating "key" per file (mileage.value has two,
|
||||
# but they agreed on value in 0/242 sampled disagreements) - i.e. they are
|
||||
# genuinely unambiguous single-valued fields, unlike e.g. "timestamp" or
|
||||
# "car_captured_time" which bundle multiple unrelated sub-reports under the
|
||||
# same dataFieldName (see latest_signal_timestamp_unix() above).
|
||||
#
|
||||
# driver_present, cruising_range_km, position_longitude/_latitude/_created,
|
||||
# is_parked, service_due_in_days and the driving_mode/plug_connection_state/
|
||||
# next_service_type enums have no known source field at all in this data
|
||||
# feed (see README) and are intentionally left unmapped below - their Gauge/
|
||||
# Enum objects stay defined but will simply never be set.
|
||||
GAUGE_FIELD_MAP: list[tuple[str, Gauge, callable]] = [
|
||||
("mileage_info.value", g_mileage_km, parse_float),
|
||||
("hvsoc_info.value", g_hvsoc_percent, parse_float),
|
||||
("Driver Presence", g_driver_present, parse_bool_as_float),
|
||||
("batteryStatus.cruisingRange.range", g_cruising_range_km, parse_float),
|
||||
(
|
||||
"hvbatterytemperature_info.max_temperature.value",
|
||||
g_hvbattery_temp_max_celsius,
|
||||
parse_float,
|
||||
),
|
||||
(
|
||||
"hvbatterytemperature_info.min_temperature.value",
|
||||
g_hvbattery_temp_min_celsius,
|
||||
parse_float,
|
||||
),
|
||||
("chargingStatus.chargePower_kW", g_charge_power_kw, parse_float),
|
||||
("targetSoc_pct", g_target_soc_percent, parse_float),
|
||||
("positionCreated", g_position_created_timestamp_seconds, parse_iso8601_to_unix),
|
||||
("isParked", g_is_parked, parse_bool_as_float),
|
||||
("parking_brake_info.value", g_parking_brake_engaged, parse_bool_as_float),
|
||||
("service_maintenance_info.due_in_time.value", g_service_due_in_days, parse_float),
|
||||
(
|
||||
"carCapturedUTCTimestamp",
|
||||
g_last_vehicle_signal_timestamp_seconds,
|
||||
parse_iso8601_to_unix,
|
||||
),
|
||||
("mileage.value", g_mileage_km, parse_float),
|
||||
("battery_state_report.soc", g_hvsoc_percent, parse_float),
|
||||
("max_temperature", g_hvbattery_temp_max_celsius, parse_float),
|
||||
("min_temperature", g_hvbattery_temp_min_celsius, parse_float),
|
||||
("battery_state_report.charge_power", g_charge_power_kw, parse_float),
|
||||
("settings.target_soc", g_target_soc_percent, parse_float),
|
||||
("parking_brake", g_parking_brake_engaged, parse_bool_as_float),
|
||||
("locked", g_locked, parse_bool_as_float),
|
||||
]
|
||||
|
||||
ENUM_FIELD_MAP: list[tuple[str, Enum, list[str]]] = [
|
||||
("drivingMode", e_driving_mode, DRIVING_MODE_STATES),
|
||||
("chargingStatus.currentChargeState", e_charging_state, CHARGING_STATE_STATES),
|
||||
(
|
||||
"plugStatusItem.plugConnectionState",
|
||||
e_plug_connection_state,
|
||||
PLUG_CONNECTION_STATES,
|
||||
"charging_state_report.current_charge_state",
|
||||
e_charging_state,
|
||||
CHARGING_STATE_STATES,
|
||||
),
|
||||
("service_maintenance_info.service_type", e_service_type, SERVICE_TYPE_STATES),
|
||||
]
|
||||
|
||||
# Lock components -> internal name. If a component is missing from a dataset
|
||||
# entirely (even across all scrapes so far), it is treated as "LOCKED" (safe
|
||||
# default), so a single door actively reporting locked does not on its own
|
||||
# report "vehicle locked" if another component is still unknown.
|
||||
LOCK_FIELDS = {
|
||||
"door_info.front_left.door_lock_status.value": "front_left",
|
||||
"door_info.front_right.door_lock_status.value": "front_right",
|
||||
"door_info.rear_left.door_lock_status.value": "rear_left",
|
||||
"door_info.rear_right.door_lock_status.value": "rear_right",
|
||||
"trunk_lid_info.trunk_lid_lock_status.value": "trunk",
|
||||
"hood_info.hood_lock_status.value": "hood",
|
||||
}
|
||||
|
||||
|
||||
class UptimeCollector:
|
||||
"""Computes uptime freshly on every scrape instead of maintaining a
|
||||
@@ -609,10 +620,7 @@ class UptimeCollector:
|
||||
class VehiclePoller:
|
||||
def __init__(self, vin: str) -> None:
|
||||
self.vin = vin
|
||||
self.lock_components: dict[str, str] = {
|
||||
name: "LOCKED" for name in LOCK_FIELDS.values()
|
||||
}
|
||||
self.last_signal_ts_raw: str | None = None
|
||||
self.last_signal_unix: float | None = None
|
||||
|
||||
def apply_fields(self, fields: dict[str, str]) -> None:
|
||||
vin = self.vin
|
||||
@@ -634,14 +642,6 @@ class VehiclePoller:
|
||||
log.warning("Unknown enum value for %s (VIN %s): %r", key, vin, raw)
|
||||
enum_metric.labels(vin=vin).state(state)
|
||||
|
||||
for key, comp_name in LOCK_FIELDS.items():
|
||||
if key in fields:
|
||||
self.lock_components[comp_name] = str(fields[key])
|
||||
locked = (
|
||||
0.0 if any(v == "UNLOCKED" for v in self.lock_components.values()) else 1.0
|
||||
)
|
||||
g_locked.labels(vin=vin).set(locked)
|
||||
|
||||
if "longitude" in fields and "latitude" in fields:
|
||||
try:
|
||||
g_position_longitude.labels(vin=vin).set(
|
||||
@@ -651,9 +651,10 @@ class VehiclePoller:
|
||||
except (TypeError, ValueError) as exc:
|
||||
log.warning("Could not parse position (VIN %s): %s", vin, exc)
|
||||
|
||||
new_ts = fields.get("carCapturedUTCTimestamp")
|
||||
if new_ts is not None and new_ts != self.last_signal_ts_raw:
|
||||
self.last_signal_ts_raw = new_ts
|
||||
new_signal_unix = fields.get("_signal_timestamp_unix")
|
||||
if new_signal_unix is not None and new_signal_unix != self.last_signal_unix:
|
||||
self.last_signal_unix = new_signal_unix
|
||||
g_last_vehicle_signal_timestamp_seconds.labels(vin=vin).set(new_signal_unix)
|
||||
g_last_successful_scrape_timestamp_seconds.labels(vin=vin).set(time.time())
|
||||
g_health.labels(vin=vin).set(1.0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user