fix: edit wrong .gitignore :D

Reviewed-on: #18
Reviewed-by: Philipp Schweizer <Philipp.schw@directbox.com>

.gitignore

@@ -0,0 +1 @@
+docker-compose.yml

README.md

@@ -11,10 +11,11 @@ We recommend using [The Things Network](https://www.thethingsnetwork.org/) (TTN)
     - [Database](#database)
     - [Configuration](#configuration)
 2. [TTN Integration](#ttn-integration)
-3. [Add a Location Tracker](#add-a-location-tracker)
+3. [Prometheus Metrics](#prometheus-metrics)
+4. [Add a Location Tracker](#add-a-location-tracker)
     - [Onboard SenseCAP T1000-B](#onboard-sensecap-t1000-b)
     - [Register SenseCAP T1000-B](#register-sensecap-t1000-b)
-4. [Testing](#testing)
+5. [Testing](#testing)
     - [Testing Webhook](#testing-webhook)
     - [Emulating Wigle API](#emulating-wigle-api)
 
@@ -49,6 +50,9 @@ Add a addidtional header:
 - Type: `authorization` 
 - Value: `Bearer your-very-secure-token-from-the-env-file`
 
+### Prometheus Metrics
+Use `https://your.domain.tld/api/metrics` to retrieve useful insides for monitoring.
+
 ## Add a Location Tracker
 We use the [SenseCAP T1000-B](https://www.seeedstudio.com/SenseCAP-Card-Tracker-T1000-B-p-5698.html) from seeedstudio because of the fair price and multiple location providers. However, you can use any LoRaWAN-enabled tracker that is compatible with TTN and supports the required payload fields.
 
@@ -60,7 +64,7 @@ We use the [SenseCAP T1000-B](https://www.seeedstudio.com/SenseCAP-Card-Tracker-
 5. In the `Settings` tab select `The Things Network` as Platform under `LoRa`
 6. Copy `Device EUI`, `AppEUI` and `AppKey`
 7. Save LoRa settings
-8. Under `Geolocation` select Geolocation Strategy as `Bluetooth+Wi-Fi+GNSS`
+8. Under `Geolocation` select Geolocation Strategy as `GNSS+Wi-Fi`
 9. Set `GNSS Max Scan Time (s)` to 120
 10. Save Geolocation settings
 11. Exit App with return arrow to trigger re-init of SenseCAP T1000-B

server/.gitignore

@@ -308,7 +308,4 @@ cython_debug/
 # Built Visual Studio Code Extensions
 *.vsix
 
-config.py
-
-#docker
-docker-compose.yml
+config.py

server/sql/tables.sql

@@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS lp_ttn_end_device_uplinks (
     dev_eui VARCHAR(255),
     join_eui VARCHAR(255),
     dev_addr VARCHAR(255),
-    received_at_utc DATE,
+    received_at_utc DATE NOT NULL,
     battery NUMERIC,
     created_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
@@ -16,6 +16,7 @@ CREATE TABLE IF NOT EXISTS wifi_scan (
     lp_ttn_end_device_uplinks_id UUID,
     mac VARCHAR(255),
     rssi NUMERIC,
+    scanned_at_utc TIMESTAMP NOT NULL,
     created_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     FOREIGN KEY (lp_ttn_end_device_uplinks_id) REFERENCES lp_ttn_end_device_uplinks(lp_ttn_end_device_uplinks_id)
@@ -63,6 +64,7 @@ CREATE TABLE IF NOT EXISTS location (
     gnss_longitude DOUBLE,
     ttn_gw_latitude DOUBLE,
     ttn_gw_longitude DOUBLE,
+    gnss_location_at_utc TIMESTAMP,
     created_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at_utc TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
     FOREIGN KEY (lp_ttn_end_device_uplinks_id) REFERENCES lp_ttn_end_device_uplinks(lp_ttn_end_device_uplinks_id)

server/src/controller/metricsController.ts

@@ -4,64 +4,47 @@ import { Counter, Gauge, collectDefaultMetrics, register } from "prom-client";
 import { LocationService } from "../services/locationService";
 import { WifiLocationService } from "../services/wifiLocationService";
 import { TtnGatewayReceptionService } from "../services/ttnGatewayReceptionService";
+import { MetricsService } from "../services/metricsService";
 
 const router = express.Router();
 const locationService = container.resolve(LocationService);
 const wifiLocationService = container.resolve(WifiLocationService);
 const ttnGatewayReceptionService = container.resolve(TtnGatewayReceptionService);
+const metricsService = container.resolve(MetricsService);
 
-// Collect default system metrics (e.g., CPU, memory usage)
-const prefix = 'locationhub_';
-collectDefaultMetrics({ prefix });
-
-// Define a custom Counter metric
-const requestCounter = new Counter({
-  name: `${prefix}http_requests_total`,
-  help: "Total number of HTTP requests",
-  labelNames: ["method", "route", "status"],
-});
-
-const locationsTotal = new Gauge({
-  name: `${prefix}locations_total`,
-  help: "Total number of location entries in database",
-  labelNames: ["database"],
-});
-
-const wifiLocationTotal = new Gauge({
-  name: `${prefix}wifi_locations_total`,
-  help: "Total number of wifi location entries in database",
-  labelNames: ["database"],
-});
-
-const wifiLocationNotResolvable = new Gauge({
-  name: `${prefix}wifi_locations_not_resolvable`,
-  help: "Unresolved number of wifi location entries in database",
-  labelNames: ["database"],
-});
-
-const wifiLocationRequestLimitExceeded = new Gauge({
-  name: `${prefix}wifi_locations_request_limit_exceeded`,
-  help: "Unresolved number of wifi location because request limit exceeded entries in database",
-  labelNames: ["database"],
-});
-
-const ttnGatewayReceptions = new Gauge({
-  name: `${prefix}ttn_gateway_receptions`,
-  help: "Total number of TTN Gateway receptions entries in database",
-  labelNames: ["database"],
-});
-
+const requestCounter = metricsService.getRequestCounter();
+const locationsTotal = metricsService.getLocationsTotal();
+const gnssLocationsTotal = metricsService.getGnssLocationsTotal()
+const wifiLocationTotal = metricsService.getWifiLocationTotal();
+const wifiLocationNotResolvable = metricsService.getWifiLocationResolvable();
+const wifiLocationRequestLimitExceeded = metricsService.getLWifiLocationRequestLimitExceeded();
+const ttnGatewayReceptions = metricsService.getTnnGatewayReceptions();
 
 // Define the metrics endpoint
 router.get("/", async (req: Request, res: Response) => {
   try {
-    console.log("Metric Endpoint triggered");
+    const [
+      allLocations,
+      gnssLocations,
+      allWifiLocations,
+      wifiLocationsNotResolvable,
+      wifiLocationsRequestLimitExceeded,
+      allTtnGatewayReceptions
+    ] = await Promise.all([
+      locationService.getAllLocations(),
+      locationService.getAllGnssLocations(),
+      wifiLocationService.getAllWifiLocations(),
+      wifiLocationService.getAllWifiLocationsByNotResolvable(),
+      wifiLocationService.getAllWifiLocationsByRequestLimitExceeded(),
+      ttnGatewayReceptionService.getAllGatewayReceptions()
+    ]);
 
-    locationsTotal.set((await locationService.getAllLocations()).length);
-    wifiLocationTotal.set((await wifiLocationService.getAllWifiLocations()).length);
-    wifiLocationNotResolvable.set((await wifiLocationService.getAllWifiLocationsByNotResolvable()).length);
-    wifiLocationRequestLimitExceeded.set((await wifiLocationService.getAllWifiLocationsByRequestLimitExceeded()).length);
-    ttnGatewayReceptions.set((await ttnGatewayReceptionService.getAllGatewayReceptions()).length);
+    locationsTotal.set(allLocations.length);
+    gnssLocationsTotal.set(gnssLocations.length);
+    wifiLocationTotal.set(allWifiLocations.length);
+    wifiLocationNotResolvable.set(wifiLocationsNotResolvable.length);
+    wifiLocationRequestLimitExceeded.set(wifiLocationsRequestLimitExceeded.length);
+    ttnGatewayReceptions.set(allTtnGatewayReceptions.length);
 
     // Increment the counter with labels
     requestCounter.inc({ method: req.method, route: req.route.path, status: 200 });
@@ -72,7 +55,6 @@ router.get("/", async (req: Request, res: Response) => {
   } catch (error) {
     // Increment the counter for errors
     requestCounter.inc({ method: req.method, route: req.route.path, status: 500 });
-
     console.error("Error running metrics endpoint:", error);
     res.status(500).json({ error: "Error running metrics endpoint" });
   }

server/src/controller/ttnController.ts

@@ -42,23 +42,38 @@ router.post(
           )?.measurementValue,
         });
 
+      const messageData = message.uplink_message.decoded_payload?.messages[0];
+
+      const latitudeData = messageData?.find((e) => e.type === "Latitude");
+      const longitudeData = messageData?.find((e) => e.type === "Longitude");
+
       const gnnsLocation = {
-        latitude: message.uplink_message.decoded_payload?.messages[0].find(
-          (e) => e.type === "Latitude"
-        )?.measurementValue,
-        longitude: message.uplink_message.decoded_payload?.messages[0].find(
-          (e) => e.type === "Longitude"
-        )?.measurementValue,
+        latitude: latitudeData?.measurementValue,
+        longitude: longitudeData?.measurementValue,
       };
 
+      const gnssTimestamp = {
+        timestamp: latitudeData?.timestamp
+          ? new Date(latitudeData.timestamp)
+          : longitudeData?.timestamp
+          ? new Date(longitudeData.timestamp)
+          : undefined,
+      };
+
+      const wifiMessage =
+        message.uplink_message.decoded_payload?.messages[0].find(
+          (e) => e.type === "Wi-Fi Scan"
+        );
+
       const wifiScans =
-        message.uplink_message.decoded_payload?.messages[0]
-          .find((e) => e.type === "Wi-Fi Scan")
-          ?.measurementValue?.map((w) => ({
-            lp_ttn_end_device_uplinks_id,
-            mac: w.mac,
-            rssi: w.rssi,
-          })) ?? [];
+        wifiMessage?.measurementValue?.map((w) => ({
+          lp_ttn_end_device_uplinks_id,
+          mac: w.mac,
+          rssi: w.rssi,
+          scanned_at_utc: wifiMessage?.timestamp
+            ? new Date(wifiMessage.timestamp)
+            : undefined,
+        })) ?? [];
 
       const ttnGatewayReceptions = message.uplink_message.rx_metadata.map(
         (g) => ({
@@ -98,6 +113,7 @@ router.post(
                   longitude: gnnsLocation.longitude,
                 }
               : undefined,
+          gnss_timestamp: gnssTimestamp.timestamp,
         });
       };
       createDatabaseEntries().then();

server/src/models/location.ts

@@ -10,6 +10,7 @@ export class Location extends Model {
   public gnss_longitude!: number;
   public ttn_gw_latitude!: number;
   public ttn_gw_longitude!: number;
+  public gnss_location_at_utc!: Date;
   public created_at_utc!: Date;
   public updated_at_utc!: Date;
 }
@@ -42,6 +43,9 @@ Location.init(
     ttn_gw_longitude: {
       type: DataTypes.NUMBER,
     },
+    gnss_location_at_utc: {
+      type: DataTypes.DATE,
+    },
     created_at_utc: {
       type: DataTypes.DATE,
       defaultValue: DataTypes.NOW,

server/src/models/wifiScan.ts

@@ -6,6 +6,7 @@ export class WifiScan extends Model {
   public wifi_scan_id!: string;
   public mac!: string;
   public rssi!: number;
+  public scanned_at_utc!: Date;
   public created_at_utc!: Date;
   public updated_at_utc!: Date;
 }
@@ -30,6 +31,11 @@ WifiScan.init(
       type: DataTypes.NUMBER,
       allowNull: false,
     },
+    scanned_at_utc: {
+      type: DataTypes.DATE,
+      defaultValue: DataTypes.NOW,
+      allowNull: false,
+    },
     created_at_utc: {
       type: DataTypes.DATE,
       defaultValue: DataTypes.NOW,

server/src/repositories/locationRepository.ts

@@ -1,10 +1,11 @@
+import { Attributes, FindOptions } from "sequelize";
 import { injectable } from "tsyringe";
 import { Location } from "../models/location";
 
 @injectable()
 export class LocationRepository {
-  public async findAll() {
-    return await Location.findAll();
+  public async findAll(options?: FindOptions<Attributes<Location>>) {
+    return await Location.findAll(options);
   }
 
   public async findById(id: string) {

server/src/services/locationService.ts

@@ -1,4 +1,5 @@
 import { inject, injectable } from "tsyringe";
+import { Op } from 'sequelize';
 import { Location } from "../models/location";
 import { LocationRepository } from "../repositories/locationRepository";
 import { WifiLocationService } from "./wifiLocationService";
@@ -8,6 +9,7 @@ interface CreateLocationParams {
   wifi?: Coordinates;
   gnss?: Coordinates;
   ttn_gw?: Coordinates;
+  gnss_timestamp?: Date;
 }
 
 interface CreateLocationTriangulationParams {
@@ -18,6 +20,7 @@ interface CreateLocationTriangulationParams {
   }[];
   ttn_gw: LocationSignal[];
   gnss?: Coordinates;
+  gnss_timestamp?: Date;
 }
 
 interface LocationSignal extends Coordinates {
@@ -52,6 +55,15 @@ export class LocationService {
     return this.repository.findAll();
   }
 
+  public async getAllGnssLocations() {
+    return this.repository.findAll({
+      where: {
+        gnss_latitude: { [Op.ne]: null },
+        gnss_longitude: { [Op.ne]: null }
+      }
+    });
+  }
+
   public async getLocationById(id: string) {
     return this.repository.findById(id);
   }
@@ -65,6 +77,7 @@ export class LocationService {
       ttn_gw_longitude: data.ttn_gw?.longitude,
       gnss_latitude: data.gnss?.latitude,
       gnss_longitude: data.gnss?.longitude,
+      gnss_location_at_utc: data.gnss_timestamp,
     });
   }
 
@@ -81,6 +94,7 @@ export class LocationService {
       wifi: wifi_location,
       ttn_gw: gateway_location,
       gnss: data.gnss,
+      gnss_timestamp: data.gnss_timestamp,
     });
   }
 

server/src/services/metricsService.ts

@@ -0,0 +1,70 @@
+import {injectable } from "tsyringe";
+import { Counter, Gauge, collectDefaultMetrics} from "prom-client";
+
+// Collect default system metrics (e.g., CPU, memory usage)
+const prefix = 'locationhub_';
+collectDefaultMetrics({ prefix });
+
+@injectable()
+export class MetricsService {
+  constructor(
+  ) { }
+
+  public getRequestCounter() {
+    const requestCounter = new Counter({
+      name: `${prefix}http_requests_total`,
+      help: "Total number of HTTP requests",
+      labelNames: ["method", "route", "status"],
+    });
+    return requestCounter;
+  }
+
+  public getLocationsTotal() {
+    const locationsTotal = new Gauge({
+      name: `${prefix}locations_total`,
+      help: "Total number of location entries in database",
+      labelNames: ["database"],
+    });
+    return locationsTotal;
+  }
+  public getGnssLocationsTotal() {
+    const gnssLocationsTotal = new Gauge({
+      name: `${prefix}gnss_locations_total`,
+      help: "Total number of location entries with GNSS in database",
+      labelNames: ["database"],
+    });
+    return gnssLocationsTotal;
+  }
+  public getWifiLocationTotal() {
+    const wifiLocationTotal = new Gauge({
+      name: `${prefix}wifi_locations_total`,
+      help: "Total number of wifi location entries in database",
+      labelNames: ["database"],
+    });
+    return wifiLocationTotal;
+  }
+  public getWifiLocationResolvable() {
+    const wifiLocationNotResolvable = new Gauge({
+      name: `${prefix}wifi_locations_not_resolvable`,
+      help: "Unresolved number of wifi location entries in database",
+      labelNames: ["database"],
+    });
+    return wifiLocationNotResolvable;
+  }
+  public getLWifiLocationRequestLimitExceeded() {
+    const wifiLocationRequestLimitExceeded = new Gauge({
+      name: `${prefix}wifi_locations_request_limit_exceeded`,
+      help: "Unresolved number of wifi location because request limit exceeded entries in database",
+      labelNames: ["database"],
+    });
+    return wifiLocationRequestLimitExceeded;
+  }
+  public getTnnGatewayReceptions() {
+    const ttnGatewayReceptions = new Gauge({
+      name: `${prefix}ttn_gateway_receptions`,
+      help: "Total number of TTN Gateway receptions entries in database",
+      labelNames: ["database"],
+    });
+    return ttnGatewayReceptions;
+  }
+}

server/src/services/wifiScanService.ts

@@ -5,12 +5,14 @@ interface CreateWifiScanParams {
   lp_ttn_end_device_uplinks_id: string;
   mac: string;
   rssi: number;
+  scanned_at_utc?: Date;
 }
 
 interface UpdateWifiScanParams {
   wifi_scan_id: string;
   mac?: string;
   rssi?: number;
+  scanned_at_utc?: Date;
 }
 
 @injectable()