From a745aaf9d06e00da42b200a9521a6d623c2c04b37253079642a5b19aaddbc792 Mon Sep 17 00:00:00 2001 From: localhorst Date: Sat, 25 Jan 2025 21:51:42 +0100 Subject: [PATCH 1/8] add timestamps to DB model --- server/sql/tables.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/sql/tables.sql b/server/sql/tables.sql index e416098..1841326 100644 --- a/server/sql/tables.sql +++ b/server/sql/tables.sql @@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS lp_ttn_end_device_uplinks ( dev_eui VARCHAR(255), join_eui VARCHAR(255), dev_addr VARCHAR(255), - received_at_utc DATE, + received_at_utc DATE NOT NULL, battery NUMERIC, created_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP @@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS wifi_scan ( lp_ttn_end_device_uplinks_id UUID, mac VARCHAR(255), rssi NUMERIC, + scanned_at_utc DATE NOT NULL; created_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, FOREIGN KEY (lp_ttn_end_device_uplinks_id) REFERENCES lp_ttn_end_device_uplinks(lp_ttn_end_device_uplinks_id) @@ -63,6 +64,7 @@ CREATE TABLE IF NOT EXISTS location ( gnss_longitude DOUBLE, ttn_gw_latitude DOUBLE, ttn_gw_longitude DOUBLE, + gnss_location_at_utc DATE; created_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, FOREIGN KEY (lp_ttn_end_device_uplinks_id) REFERENCES lp_ttn_end_device_uplinks(lp_ttn_end_device_uplinks_id) From 452589d11d5d0a2ea56f025fa68c1231e2f4ca75bdc4a89e4bd768caed694f29 Mon Sep 17 00:00:00 2001 From: localhorst Date: Tue, 28 Jan 2025 21:55:54 +0100 Subject: [PATCH 2/8] add timestamps in DB model --- server/src/models/location.ts | 3 +++ server/src/models/wifiScan.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/server/src/models/location.ts b/server/src/models/location.ts index 86e8d21..c4b5dff 100644 --- a/server/src/models/location.ts +++ b/server/src/models/location.ts @@ -42,6 +42,9 @@ Location.init( ttn_gw_longitude: { type: DataTypes.NUMBER, }, + gnss_location_at_utc: { + type: DataTypes.DATE, + }, created_at_utc: { type: DataTypes.DATE, defaultValue: DataTypes.NOW, diff --git a/server/src/models/wifiScan.ts b/server/src/models/wifiScan.ts index 1f277b0..54493d5 100644 --- a/server/src/models/wifiScan.ts +++ b/server/src/models/wifiScan.ts @@ -30,6 +30,9 @@ WifiScan.init( type: DataTypes.NUMBER, allowNull: false, }, + scanned_at_utc: { + type: DataTypes.DATE, + }, created_at_utc: { type: DataTypes.DATE, defaultValue: DataTypes.NOW, From 59dc57a618c3de9b68c74dc5e7de13230d602631fb195f736b7d361cadb3dc13 Mon Sep 17 00:00:00 2001 From: localhorst Date: Tue, 28 Jan 2025 22:12:48 +0100 Subject: [PATCH 3/8] set gnss timestamp in location --- server/src/controller/ttnController.ts | 11 +++++++++++ server/src/models/location.ts | 1 + server/src/services/locationService.ts | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/server/src/controller/ttnController.ts b/server/src/controller/ttnController.ts index 28c88b1..dc8dc35 100644 --- a/server/src/controller/ttnController.ts +++ b/server/src/controller/ttnController.ts @@ -51,6 +51,14 @@ router.post( )?.measurementValue, }; + const gnssTimestamp = { + timestamp: message.uplink_message.decoded_payload?.messages[0].find( + (e) => e.type === "Latitude" + )?.timestamp + }; + + + const wifiScans = message.uplink_message.decoded_payload?.messages[0] .find((e) => e.type === "Wi-Fi Scan") @@ -96,8 +104,11 @@ router.post( ? { latitude: gnnsLocation.latitude, longitude: gnnsLocation.longitude, + } : undefined, + + gnss_timestamp: new Date(gnssTimestamp.timestamp), }); }; createDatabaseEntries().then(); diff --git a/server/src/models/location.ts b/server/src/models/location.ts index c4b5dff..728645c 100644 --- a/server/src/models/location.ts +++ b/server/src/models/location.ts @@ -10,6 +10,7 @@ export class Location extends Model { public gnss_longitude!: number; public ttn_gw_latitude!: number; public ttn_gw_longitude!: number; + public gnss_timestamp!: Date; public created_at_utc!: Date; public updated_at_utc!: Date; } diff --git a/server/src/services/locationService.ts b/server/src/services/locationService.ts index cb8f10b..ea96d2c 100644 --- a/server/src/services/locationService.ts +++ b/server/src/services/locationService.ts @@ -8,6 +8,7 @@ interface CreateLocationParams { wifi?: Coordinates; gnss?: Coordinates; ttn_gw?: Coordinates; + gnss_timestamp?: Date; } interface CreateLocationTriangulationParams { @@ -18,6 +19,7 @@ interface CreateLocationTriangulationParams { }[]; ttn_gw: LocationSignal[]; gnss?: Coordinates; + gnss_timestamp: Date; } interface LocationSignal extends Coordinates { @@ -65,6 +67,7 @@ export class LocationService { ttn_gw_longitude: data.ttn_gw?.longitude, gnss_latitude: data.gnss?.latitude, gnss_longitude: data.gnss?.longitude, + gnss_timestamp: data.gnss_timestamp, }); } @@ -81,6 +84,7 @@ export class LocationService { wifi: wifi_location, ttn_gw: gateway_location, gnss: data.gnss, + }); } From 62a2dc2c4af3e8b0c619cb5b8df8b6bdd201d67cc3fa6373abc666e709aca7bd Mon Sep 17 00:00:00 2001 From: localhorst Date: Wed, 29 Jan 2025 23:03:48 +0100 Subject: [PATCH 4/8] fix gnss timestamp --- server/sql/tables.sql | 4 ++-- server/src/controller/ttnController.ts | 19 +++++++++---------- server/src/models/location.ts | 2 +- server/src/models/wifiScan.ts | 3 +++ server/src/services/locationService.ts | 6 +++--- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/server/sql/tables.sql b/server/sql/tables.sql index 1841326..fa6b006 100644 --- a/server/sql/tables.sql +++ b/server/sql/tables.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS wifi_scan ( lp_ttn_end_device_uplinks_id UUID, mac VARCHAR(255), rssi NUMERIC, - scanned_at_utc DATE NOT NULL; + scanned_at_utc TIMESTAMP NOT NULL, created_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, FOREIGN KEY (lp_ttn_end_device_uplinks_id) REFERENCES lp_ttn_end_device_uplinks(lp_ttn_end_device_uplinks_id) @@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS location ( gnss_longitude DOUBLE, ttn_gw_latitude DOUBLE, ttn_gw_longitude DOUBLE, - gnss_location_at_utc DATE; + gnss_location_at_utc TIMESTAMP, created_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, FOREIGN KEY (lp_ttn_end_device_uplinks_id) REFERENCES lp_ttn_end_device_uplinks(lp_ttn_end_device_uplinks_id) diff --git a/server/src/controller/ttnController.ts b/server/src/controller/ttnController.ts index dc8dc35..e517d74 100644 --- a/server/src/controller/ttnController.ts +++ b/server/src/controller/ttnController.ts @@ -51,14 +51,14 @@ router.post( )?.measurementValue, }; + const gnssMetadata = message.uplink_message.decoded_payload?.messages[0].find( + (e) => e.type === "Latitude" + ); + const gnssTimestamp = { - timestamp: message.uplink_message.decoded_payload?.messages[0].find( - (e) => e.type === "Latitude" - )?.timestamp + timestamp: gnssMetadata?.timestamp ? new Date(gnssMetadata.timestamp) : undefined }; - - const wifiScans = message.uplink_message.decoded_payload?.messages[0] .find((e) => e.type === "Wi-Fi Scan") @@ -102,13 +102,12 @@ router.post( gnss: gnnsLocation.latitude && gnnsLocation.longitude ? { - latitude: gnnsLocation.latitude, - longitude: gnnsLocation.longitude, + latitude: gnnsLocation.latitude, + longitude: gnnsLocation.longitude, - } + } : undefined, - - gnss_timestamp: new Date(gnssTimestamp.timestamp), + gnss_timestamp: gnssTimestamp.timestamp, }); }; createDatabaseEntries().then(); diff --git a/server/src/models/location.ts b/server/src/models/location.ts index 728645c..9307797 100644 --- a/server/src/models/location.ts +++ b/server/src/models/location.ts @@ -10,7 +10,7 @@ export class Location extends Model { public gnss_longitude!: number; public ttn_gw_latitude!: number; public ttn_gw_longitude!: number; - public gnss_timestamp!: Date; + public gnss_location_at_utc!: Date; public created_at_utc!: Date; public updated_at_utc!: Date; } diff --git a/server/src/models/wifiScan.ts b/server/src/models/wifiScan.ts index 54493d5..9254c08 100644 --- a/server/src/models/wifiScan.ts +++ b/server/src/models/wifiScan.ts @@ -6,6 +6,7 @@ export class WifiScan extends Model { public wifi_scan_id!: string; public mac!: string; public rssi!: number; + public scanned_at_utc!: Date; public created_at_utc!: Date; public updated_at_utc!: Date; } @@ -32,6 +33,8 @@ WifiScan.init( }, scanned_at_utc: { type: DataTypes.DATE, + defaultValue: DataTypes.NOW, + allowNull: false, }, created_at_utc: { type: DataTypes.DATE, diff --git a/server/src/services/locationService.ts b/server/src/services/locationService.ts index ea96d2c..9cc90ab 100644 --- a/server/src/services/locationService.ts +++ b/server/src/services/locationService.ts @@ -19,7 +19,7 @@ interface CreateLocationTriangulationParams { }[]; ttn_gw: LocationSignal[]; gnss?: Coordinates; - gnss_timestamp: Date; + gnss_timestamp?: Date; } interface LocationSignal extends Coordinates { @@ -67,7 +67,7 @@ export class LocationService { ttn_gw_longitude: data.ttn_gw?.longitude, gnss_latitude: data.gnss?.latitude, gnss_longitude: data.gnss?.longitude, - gnss_timestamp: data.gnss_timestamp, + gnss_location_at_utc: data.gnss_timestamp, }); } @@ -84,7 +84,7 @@ export class LocationService { wifi: wifi_location, ttn_gw: gateway_location, gnss: data.gnss, - + gnss_timestamp: data.gnss_timestamp, }); } From 5e4fd59148721b42eb3c2cd2a97e36276dc1f0a882dc7164b031f86c6430349a Mon Sep 17 00:00:00 2001 From: Philipp Schweizer Date: Sun, 2 Feb 2025 20:10:29 +0100 Subject: [PATCH 5/8] refactor to use either timestamp when one is undefined --- server/src/controller/ttnController.ts | 30 +++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/server/src/controller/ttnController.ts b/server/src/controller/ttnController.ts index e517d74..b0570d4 100644 --- a/server/src/controller/ttnController.ts +++ b/server/src/controller/ttnController.ts @@ -42,21 +42,22 @@ router.post( )?.measurementValue, }); + const messageData = message.uplink_message.decoded_payload?.messages[0]; + + const latitudeData = messageData?.find((e) => e.type === "Latitude"); + const longitudeData = messageData?.find((e) => e.type === "Longitude"); + const gnnsLocation = { - latitude: message.uplink_message.decoded_payload?.messages[0].find( - (e) => e.type === "Latitude" - )?.measurementValue, - longitude: message.uplink_message.decoded_payload?.messages[0].find( - (e) => e.type === "Longitude" - )?.measurementValue, + latitude: latitudeData?.measurementValue, + longitude: longitudeData?.measurementValue, }; - const gnssMetadata = message.uplink_message.decoded_payload?.messages[0].find( - (e) => e.type === "Latitude" - ); - const gnssTimestamp = { - timestamp: gnssMetadata?.timestamp ? new Date(gnssMetadata.timestamp) : undefined + timestamp: latitudeData?.timestamp + ? new Date(latitudeData.timestamp) + : longitudeData?.timestamp + ? new Date(longitudeData.timestamp) + : undefined, }; const wifiScans = @@ -102,10 +103,9 @@ router.post( gnss: gnnsLocation.latitude && gnnsLocation.longitude ? { - latitude: gnnsLocation.latitude, - longitude: gnnsLocation.longitude, - - } + latitude: gnnsLocation.latitude, + longitude: gnnsLocation.longitude, + } : undefined, gnss_timestamp: gnssTimestamp.timestamp, }); From 93f0c71a6c58513a1374c0be1df547f17ab554ce1df2d055539d8f0c861cab8a Mon Sep 17 00:00:00 2001 From: localhorst Date: Sat, 8 Feb 2025 20:54:06 +0100 Subject: [PATCH 6/8] set wifi timestamp --- server/src/controller/ttnController.ts | 16 +++++++++++----- server/src/services/wifiScanService.ts | 2 ++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/server/src/controller/ttnController.ts b/server/src/controller/ttnController.ts index b0570d4..df9e01e 100644 --- a/server/src/controller/ttnController.ts +++ b/server/src/controller/ttnController.ts @@ -56,9 +56,14 @@ router.post( timestamp: latitudeData?.timestamp ? new Date(latitudeData.timestamp) : longitudeData?.timestamp - ? new Date(longitudeData.timestamp) - : undefined, + ? new Date(longitudeData.timestamp) + : undefined, }; + const wifiTimestamp = (() => { + const messages = message.uplink_message.decoded_payload?.messages?.[0]; + const wifiScan = messages?.find((e: { type: string }) => e.type === "Wi-Fi Scan"); + return wifiScan?.timestamp ? new Date(wifiScan.timestamp) : undefined; + })(); const wifiScans = message.uplink_message.decoded_payload?.messages[0] @@ -67,6 +72,7 @@ router.post( lp_ttn_end_device_uplinks_id, mac: w.mac, rssi: w.rssi, + scanned_at_timestamp: wifiTimestamp, })) ?? []; const ttnGatewayReceptions = message.uplink_message.rx_metadata.map( @@ -103,9 +109,9 @@ router.post( gnss: gnnsLocation.latitude && gnnsLocation.longitude ? { - latitude: gnnsLocation.latitude, - longitude: gnnsLocation.longitude, - } + latitude: gnnsLocation.latitude, + longitude: gnnsLocation.longitude, + } : undefined, gnss_timestamp: gnssTimestamp.timestamp, }); diff --git a/server/src/services/wifiScanService.ts b/server/src/services/wifiScanService.ts index d707606..ede6cb1 100644 --- a/server/src/services/wifiScanService.ts +++ b/server/src/services/wifiScanService.ts @@ -5,12 +5,14 @@ interface CreateWifiScanParams { lp_ttn_end_device_uplinks_id: string; mac: string; rssi: number; + scanned_at_timestamp?: Date; } interface UpdateWifiScanParams { wifi_scan_id: string; mac?: string; rssi?: number; + scanned_at_timestamp?: Date; } @injectable() From 52d521a6ad001ae2f4d7ebb3cf532f3a178c9f6a89c31012002447ab63f21147 Mon Sep 17 00:00:00 2001 From: localhorst Date: Sat, 8 Feb 2025 21:24:40 +0100 Subject: [PATCH 7/8] rename var --- server/src/controller/ttnController.ts | 2 +- server/src/services/wifiScanService.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/controller/ttnController.ts b/server/src/controller/ttnController.ts index df9e01e..936710d 100644 --- a/server/src/controller/ttnController.ts +++ b/server/src/controller/ttnController.ts @@ -72,7 +72,7 @@ router.post( lp_ttn_end_device_uplinks_id, mac: w.mac, rssi: w.rssi, - scanned_at_timestamp: wifiTimestamp, + scanned_at_utc: wifiTimestamp, })) ?? []; const ttnGatewayReceptions = message.uplink_message.rx_metadata.map( diff --git a/server/src/services/wifiScanService.ts b/server/src/services/wifiScanService.ts index ede6cb1..54c9273 100644 --- a/server/src/services/wifiScanService.ts +++ b/server/src/services/wifiScanService.ts @@ -5,14 +5,14 @@ interface CreateWifiScanParams { lp_ttn_end_device_uplinks_id: string; mac: string; rssi: number; - scanned_at_timestamp?: Date; + scanned_at_utc?: Date; } interface UpdateWifiScanParams { wifi_scan_id: string; mac?: string; rssi?: number; - scanned_at_timestamp?: Date; + scanned_at_utc?: Date; } @injectable() From 39c07fcef00d1fc69b284da688e68b027297e765e4c41a51e7774505730a4c8d Mon Sep 17 00:00:00 2001 From: Philipp Schweizer Date: Fri, 14 Feb 2025 21:06:47 +0100 Subject: [PATCH 8/8] refactor: only search once for wifiMessage --- server/src/controller/ttnController.ts | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/server/src/controller/ttnController.ts b/server/src/controller/ttnController.ts index 936710d..c303d56 100644 --- a/server/src/controller/ttnController.ts +++ b/server/src/controller/ttnController.ts @@ -56,24 +56,24 @@ router.post( timestamp: latitudeData?.timestamp ? new Date(latitudeData.timestamp) : longitudeData?.timestamp - ? new Date(longitudeData.timestamp) - : undefined, + ? new Date(longitudeData.timestamp) + : undefined, }; - const wifiTimestamp = (() => { - const messages = message.uplink_message.decoded_payload?.messages?.[0]; - const wifiScan = messages?.find((e: { type: string }) => e.type === "Wi-Fi Scan"); - return wifiScan?.timestamp ? new Date(wifiScan.timestamp) : undefined; - })(); + + const wifiMessage = + message.uplink_message.decoded_payload?.messages[0].find( + (e) => e.type === "Wi-Fi Scan" + ); const wifiScans = - message.uplink_message.decoded_payload?.messages[0] - .find((e) => e.type === "Wi-Fi Scan") - ?.measurementValue?.map((w) => ({ - lp_ttn_end_device_uplinks_id, - mac: w.mac, - rssi: w.rssi, - scanned_at_utc: wifiTimestamp, - })) ?? []; + wifiMessage?.measurementValue?.map((w) => ({ + lp_ttn_end_device_uplinks_id, + mac: w.mac, + rssi: w.rssi, + scanned_at_utc: wifiMessage?.timestamp + ? new Date(wifiMessage.timestamp) + : undefined, + })) ?? []; const ttnGatewayReceptions = message.uplink_message.rx_metadata.map( (g) => ({ @@ -109,9 +109,9 @@ router.post( gnss: gnnsLocation.latitude && gnnsLocation.longitude ? { - latitude: gnnsLocation.latitude, - longitude: gnnsLocation.longitude, - } + latitude: gnnsLocation.latitude, + longitude: gnnsLocation.longitude, + } : undefined, gnss_timestamp: gnssTimestamp.timestamp, });