From 69239b8cdf40ce4b62bfc613b414ce66711302b615d089f30ec722b6fc2effba Mon Sep 17 00:00:00 2001 From: localhorst Date: Mon, 17 Aug 2026 10:05:39 +0200 Subject: [PATCH] fix parsing --- README.md | 55 +++++++------ vw-eu-data-act-exporter.py | 155 +++++++++++++++++++------------------ 2 files changed, 111 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index b6c5263..af62aef 100644 --- a/README.md +++ b/README.md @@ -35,35 +35,44 @@ All metrics carry the `vin` label; all names carry the configurable prefix |---|---|---| | `mileage_km` | Gauge | Odometer reading | | `hvsoc_percent` | Gauge | HV battery state of charge in % | -| `driver_present` | Gauge (bool) | Driver detected in vehicle | -| `cruising_range_km` | Gauge | Remaining range in km | +| `driver_present` †| Gauge (bool) | Driver detected in vehicle | +| `cruising_range_km` †| Gauge | Remaining range in km | | `hvbattery_temperature_max_celsius` | Gauge | Max. HV battery temperature | | `hvbattery_temperature_min_celsius` | Gauge | Min. HV battery temperature | -| `charging_state` | Enum | `chargingStatus.currentChargeState` | +| `charging_state` | Enum | Current charging state | | `charge_power_kw` | Gauge | Current charging power in kW | -| `plug_connection_state` | Enum | Plug connection status | +| `plug_connection_state` †| Enum | Plug connection status | | `target_soc_percent` | Gauge | Charge target in % | -| `position_longitude` / `position_latitude` | Gauge | Last known position (only set when **both** coordinates were present in the same dataset) | -| `position_created_timestamp_seconds` | Gauge | Unix timestamp of the last known position | -| `locked` | Gauge (bool) | Combined lock status of all doors, trunk and hood (only `unlocked` if at least one component actively reports `UNLOCKED`) | -| `is_parked` | Gauge (bool) | Vehicle parked | +| `position_longitude` / `position_latitude` †| Gauge | Last known position (only set when **both** coordinates were present in the same dataset) | +| `position_created_timestamp_seconds` †| Gauge | Unix timestamp of the last known position | +| `locked` | Gauge (bool) | Vehicle locked, as a single combined flag reported directly by the portal | +| `is_parked` †| Gauge (bool) | Vehicle parked | | `parking_brake_engaged` | Gauge (bool) | Parking brake engaged | -| `driving_mode` | Enum | Active driving mode | -| `next_service_type` | Enum | Next due service type | -| `service_due_in_days` | Gauge | Remaining days until the next service | -| `last_vehicle_signal_timestamp_seconds` | Gauge | Timestamp of the last vehicle signal (`carCapturedUTCTimestamp`) | +| `driving_mode` †| Enum | Active driving mode | +| `next_service_type` †| Enum | Next due service type | +| `service_due_in_days` †| Gauge | Remaining days until the next service | +| `last_vehicle_signal_timestamp_seconds` | Gauge | Timestamp of the most recent vehicle signal in the latest dataset | | `uptime_seconds` | Gauge | Uptime of the exporter process | | `last_successful_scrape_timestamp_seconds` | Gauge | Last scrape with **new** data | | `health` | Gauge (bool) | `0` initially and after a failure, `1` once new data has been received at least once | | `http_requests_total` | Counter | Total number of HTTP requests made to the portal (login + API calls) | -**Note on enum metrics:** The state space (e.g. possible values of -`drivingMode` or `chargingStatus.currentChargeState`) is compiled -best-effort from the sample dataset and comparable projects, and each one -includes an `UNKNOWN` fallback. If an unknown raw value shows up, it is -exported as `UNKNOWN` and additionally logged as `WARNING` - the state list -in `vw-eu-data-act-exporter.py` (the `*_STATES` constants) can then be -extended. +† These metrics are defined for API completeness but have no known source +field in the portal's "continuous" data feed - confirmed absent across 255 +real datasets collected via `persist_raw_json` (only +`REPORT_TYPE_ENERGY_CONTENTS`, `REPORT_TYPE_CONFIGURATIONS`, +`REPORT_TYPE_CONSUMPTION_VALUES` and `REPORT_TYPE_ADDITIONAL_CONSUMPTION_VALUES` +were ever delivered). They will stay permanently absent from `/metrics` +output with this data source; this is expected, not a bug. + +**Note on enum metrics:** `charging_state`'s state list +(`CHARGING_STATE_STATES` in `vw-eu-data-act-exporter.py`) is compiled from +values actually observed in 255 real datasets, plus an `UNKNOWN` fallback. +`driving_mode` and `plug_connection_state` (marked † above) are best-effort +guesses that have never been observed at all. If an unknown raw value shows +up for any enum, it is exported as `UNKNOWN` and additionally logged as +`WARNING` - the state list in `vw-eu-data-act-exporter.py` (the `*_STATES` +constants) can then be extended. ## Installation (systemd) @@ -130,9 +139,11 @@ be independent of, and significantly shorter than, `scrape_interval_minutes` (wrong password) or no datasets available in the portal yet (see prerequisite above). - A single metric is missing entirely: the corresponding field has never - been present in any dataset fetched so far (e.g. - `position_longitude`/`_latitude`, if both coordinates were never - delivered at the same time). + been present in any dataset fetched so far. For the metrics marked † + in the table above (e.g. `position_longitude`/`_latitude`, + `driving_mode`) this is expected - they simply aren't part of this + data source. For others, use `persist_raw_json` (see Configuration) to + inspect real payloads and check the field name is still correct. - `WARNING ... Unknown enum value`: a new, not-yet-listed value for an enum metric - the state is exported as `UNKNOWN`, extend the list in the source code if needed. diff --git a/vw-eu-data-act-exporter.py b/vw-eu-data-act-exporter.py index 545a27f..0c867f2 100644 --- a/vw-eu-data-act-exporter.py +++ b/vw-eu-data-act-exporter.py @@ -306,6 +306,33 @@ def flatten_dataset(record: dict) -> dict[str, str]: return fields +# A real dataset's Data array bundles several sub-reports concatenated +# together, so these two field names each occur multiple times per dataset +# with unrelated values (confirmed via their distinct per-entry "key" +# UUIDs) - flatten_dataset()'s last-value-wins is unsafe for them. +SIGNAL_TIMESTAMP_FIELDS = ("car_captured_time", "car_captured_utc_timestamp") + + +def latest_signal_timestamp_unix(record: dict) -> float | None: + """Returns the most recent unix timestamp found under any of + SIGNAL_TIMESTAMP_FIELDS in the raw Data array, or ``None`` if none are + present/parseable.""" + best = None + for entry in record.get("Data", []): + if entry.get("dataFieldName") not in SIGNAL_TIMESTAMP_FIELDS: + continue + raw = entry.get("value") + if not raw: + continue + try: + unix = parse_iso8601_to_unix(raw) + except (TypeError, ValueError): + continue + if best is None or unix > best: + best = unix + return best + + def persist_raw_dataset(vin: str, record: dict, *, suffix: str = "") -> None: """Writes the JSON extracted from a downloaded dataset to disk, so it can be inspected later to refine the field mapping below. Only the @@ -320,11 +347,12 @@ def persist_raw_dataset(vin: str, record: dict, *, suffix: str = "") -> None: log.debug("Persisted raw dataset for VIN %s to %s", vin, dest) -def fetch_latest_dataset_fields(vin: str) -> dict[str, str] | None: +def fetch_latest_dataset_fields(vin: str) -> dict[str, str | float | None] | None: """Logs in, downloads the latest dataset for *vin* and returns the - flattened fields. Returns ``None`` if no datasets are (yet) available - - this is explicitly NOT a failure. Raises an exception on login/HTTP/parse - errors (= failure).""" + flattened fields, plus a synthetic "_signal_timestamp_unix" key (see + latest_signal_timestamp_unix()). Returns ``None`` if no datasets are + (yet) available - this is explicitly NOT a failure. Raises an exception + on login/HTTP/parse errors (= failure).""" session = requests.Session() session.headers.update({"User-Agent": random.choice(config.user_agents)}) @@ -385,7 +413,9 @@ def fetch_latest_dataset_fields(vin: str) -> dict[str, str] | None: except OSError as exc: log.warning("Could not persist raw dataset for VIN %s: %s", vin, exc) - return flatten_dataset(record) + fields = flatten_dataset(record) + fields["_signal_timestamp_unix"] = latest_signal_timestamp_unix(record) + return fields # --------------------------------------------------------------------------- @@ -454,7 +484,7 @@ g_position_created_timestamp_seconds = Gauge( ) g_locked = Gauge( f"{P}locked", - "Combined lock status of all doors, trunk and hood (1=locked)", + "Vehicle locked (1=locked), as a single combined flag from the portal", ["vin"], ) g_is_parked = Gauge(f"{P}is_parked", "Vehicle parked (1=yes)", ["vin"]) @@ -466,7 +496,8 @@ g_service_due_in_days = Gauge( ) g_last_vehicle_signal_timestamp_seconds = Gauge( f"{P}last_vehicle_signal_timestamp_seconds", - "Unix timestamp of the last vehicle signal (carCapturedUTCTimestamp)", + "Unix timestamp of the most recent vehicle signal in the latest dataset " + "(max of all car_captured_time/car_captured_utc_timestamp entries)", ["vin"], ) @@ -487,18 +518,19 @@ c_http_requests_total = Counter( ["vin"], ) -# Enum metrics. The state space is best-effort (compiled from the sample -# dataset and comparable projects) plus an "UNKNOWN" fallback for unknown raw -# values - these are additionally logged as WARNING so the list can be -# extended if needed. +# Enum metrics. driving_mode/plug_connection_state/next_service_type are not +# delivered by the "continuous" data feed at all (confirmed against 255 real +# datasets - see README) and are kept only for API completeness; they will +# never actually populate. charging_state's states below are the values +# actually observed in those 255 real datasets, plus an "UNKNOWN" fallback +# for anything new - unknown raw values are additionally logged as WARNING +# so the list can be extended if needed. DRIVING_MODE_STATES = ["standard", "eco", "comfort", "sport", "individual", "UNKNOWN"] CHARGING_STATE_STATES = [ - "OFF", - "READY_FOR_CHARGING", - "NOT_READY_FOR_CHARGING", - "CHARGING", - "CONSERVING", - "ERROR", + "CHARGE_STATE_NOT_READY_FOR_CHARGING", + "CHARGE_STATE_READY_FOR_CHARGING", + "CHARGE_STATE_CHARGING_HV_BATTERY", + "CHARGE_STATE_CHARGE_PURPOSE_REACHED_AND_NOT_CONSERVATION_CHARGING", "UNKNOWN", ] PLUG_CONNECTION_STATES = ["CONNECTED", "DISCONNECTED", "UNKNOWN"] @@ -532,59 +564,38 @@ e_service_type = Enum( states=SERVICE_TYPE_STATES, ) -# Field name (dataFieldName in the dataset) -> (Gauge, conversion function) +# Field name (dataFieldName in the dataset) -> (Gauge, conversion function). +# Verified against 255 real datasets in raw_json/: each of these fields has +# exactly one distinct originating "key" per file (mileage.value has two, +# but they agreed on value in 0/242 sampled disagreements) - i.e. they are +# genuinely unambiguous single-valued fields, unlike e.g. "timestamp" or +# "car_captured_time" which bundle multiple unrelated sub-reports under the +# same dataFieldName (see latest_signal_timestamp_unix() above). +# +# driver_present, cruising_range_km, position_longitude/_latitude/_created, +# is_parked, service_due_in_days and the driving_mode/plug_connection_state/ +# next_service_type enums have no known source field at all in this data +# feed (see README) and are intentionally left unmapped below - their Gauge/ +# Enum objects stay defined but will simply never be set. GAUGE_FIELD_MAP: list[tuple[str, Gauge, callable]] = [ - ("mileage_info.value", g_mileage_km, parse_float), - ("hvsoc_info.value", g_hvsoc_percent, parse_float), - ("Driver Presence", g_driver_present, parse_bool_as_float), - ("batteryStatus.cruisingRange.range", g_cruising_range_km, parse_float), - ( - "hvbatterytemperature_info.max_temperature.value", - g_hvbattery_temp_max_celsius, - parse_float, - ), - ( - "hvbatterytemperature_info.min_temperature.value", - g_hvbattery_temp_min_celsius, - parse_float, - ), - ("chargingStatus.chargePower_kW", g_charge_power_kw, parse_float), - ("targetSoc_pct", g_target_soc_percent, parse_float), - ("positionCreated", g_position_created_timestamp_seconds, parse_iso8601_to_unix), - ("isParked", g_is_parked, parse_bool_as_float), - ("parking_brake_info.value", g_parking_brake_engaged, parse_bool_as_float), - ("service_maintenance_info.due_in_time.value", g_service_due_in_days, parse_float), - ( - "carCapturedUTCTimestamp", - g_last_vehicle_signal_timestamp_seconds, - parse_iso8601_to_unix, - ), + ("mileage.value", g_mileage_km, parse_float), + ("battery_state_report.soc", g_hvsoc_percent, parse_float), + ("max_temperature", g_hvbattery_temp_max_celsius, parse_float), + ("min_temperature", g_hvbattery_temp_min_celsius, parse_float), + ("battery_state_report.charge_power", g_charge_power_kw, parse_float), + ("settings.target_soc", g_target_soc_percent, parse_float), + ("parking_brake", g_parking_brake_engaged, parse_bool_as_float), + ("locked", g_locked, parse_bool_as_float), ] ENUM_FIELD_MAP: list[tuple[str, Enum, list[str]]] = [ - ("drivingMode", e_driving_mode, DRIVING_MODE_STATES), - ("chargingStatus.currentChargeState", e_charging_state, CHARGING_STATE_STATES), ( - "plugStatusItem.plugConnectionState", - e_plug_connection_state, - PLUG_CONNECTION_STATES, + "charging_state_report.current_charge_state", + e_charging_state, + CHARGING_STATE_STATES, ), - ("service_maintenance_info.service_type", e_service_type, SERVICE_TYPE_STATES), ] -# Lock components -> internal name. If a component is missing from a dataset -# entirely (even across all scrapes so far), it is treated as "LOCKED" (safe -# default), so a single door actively reporting locked does not on its own -# report "vehicle locked" if another component is still unknown. -LOCK_FIELDS = { - "door_info.front_left.door_lock_status.value": "front_left", - "door_info.front_right.door_lock_status.value": "front_right", - "door_info.rear_left.door_lock_status.value": "rear_left", - "door_info.rear_right.door_lock_status.value": "rear_right", - "trunk_lid_info.trunk_lid_lock_status.value": "trunk", - "hood_info.hood_lock_status.value": "hood", -} - class UptimeCollector: """Computes uptime freshly on every scrape instead of maintaining a @@ -609,10 +620,7 @@ class UptimeCollector: class VehiclePoller: def __init__(self, vin: str) -> None: self.vin = vin - self.lock_components: dict[str, str] = { - name: "LOCKED" for name in LOCK_FIELDS.values() - } - self.last_signal_ts_raw: str | None = None + self.last_signal_unix: float | None = None def apply_fields(self, fields: dict[str, str]) -> None: vin = self.vin @@ -634,14 +642,6 @@ class VehiclePoller: log.warning("Unknown enum value for %s (VIN %s): %r", key, vin, raw) enum_metric.labels(vin=vin).state(state) - for key, comp_name in LOCK_FIELDS.items(): - if key in fields: - self.lock_components[comp_name] = str(fields[key]) - locked = ( - 0.0 if any(v == "UNLOCKED" for v in self.lock_components.values()) else 1.0 - ) - g_locked.labels(vin=vin).set(locked) - if "longitude" in fields and "latitude" in fields: try: g_position_longitude.labels(vin=vin).set( @@ -651,9 +651,10 @@ class VehiclePoller: except (TypeError, ValueError) as exc: log.warning("Could not parse position (VIN %s): %s", vin, exc) - new_ts = fields.get("carCapturedUTCTimestamp") - if new_ts is not None and new_ts != self.last_signal_ts_raw: - self.last_signal_ts_raw = new_ts + new_signal_unix = fields.get("_signal_timestamp_unix") + if new_signal_unix is not None and new_signal_unix != self.last_signal_unix: + self.last_signal_unix = new_signal_unix + g_last_vehicle_signal_timestamp_seconds.labels(vin=vin).set(new_signal_unix) g_last_successful_scrape_timestamp_seconds.labels(vin=vin).set(time.time()) g_health.labels(vin=vin).set(1.0)