improve systemd unit file

Reviewed-on: #18
Reviewed-by: Philipp Schweizer <Philipp.schw@directbox.com>

.gitignore

@@ -0,0 +1 @@
+docker-compose.yml

README.md

@@ -11,10 +11,11 @@ We recommend using [The Things Network](https://www.thethingsnetwork.org/) (TTN)
     - [Database](#database)
     - [Configuration](#configuration)
 2. [TTN Integration](#ttn-integration)
-3. [Add a Location Tracker](#add-a-location-tracker)
+3. [Prometheus Metrics](#prometheus-metrics)
+4. [Add a Location Tracker](#add-a-location-tracker)
     - [Onboard SenseCAP T1000-B](#onboard-sensecap-t1000-b)
     - [Register SenseCAP T1000-B](#register-sensecap-t1000-b)
-4. [Testing](#testing)
+5. [Testing](#testing)
     - [Testing Webhook](#testing-webhook)
     - [Emulating Wigle API](#emulating-wigle-api)
 
@@ -49,9 +50,15 @@ Add a addidtional header:
 - Type: `authorization` 
 - Value: `Bearer your-very-secure-token-from-the-env-file`
 
+### Prometheus Metrics
+Use `https://your.domain.tld/api/metrics` to retrieve useful insides for monitoring.
+
 ## Add a Location Tracker
 We use the [SenseCAP T1000-B](https://www.seeedstudio.com/SenseCAP-Card-Tracker-T1000-B-p-5698.html) from seeedstudio because of the fair price and multiple location providers. However, you can use any LoRaWAN-enabled tracker that is compatible with TTN and supports the required payload fields.
 
+## Troubleshooting
+Run `journalctl -u locationhub.service -f` to see log output.
+
 ### Onboard SenseCAP T1000-B
 1. Download and install the App [SenseCraft](https://play.google.com/store/apps/details?id=cc.seeed.sensecapmate) 
 2. Skip the user account at startup with `Skip` in the upper right corner

server/.gitignore

@@ -309,6 +309,3 @@ cython_debug/
 *.vsix
 
 config.py
-
-#docker
-docker-compose.yml

server/package-lock.json

@@ -14,6 +14,7 @@
         "express": "^4.21.2",
         "http-status-codes": "^2.3.0",
         "mariadb": "^3.4.0",
+        "prom-client": "^15.1.3",
         "reflect-metadata": "^0.2.2",
         "sequelize": "^6.37.5",
         "swagger-jsdoc": "^6.2.8",
@@ -120,6 +121,15 @@
       "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==",
       "license": "MIT"
     },
+    "node_modules/@opentelemetry/api": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz",
+      "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
     "node_modules/@scarf/scarf": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/@scarf/scarf/-/scarf-1.4.0.tgz",
@@ -386,6 +396,12 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/bintrees": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz",
+      "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==",
+      "license": "MIT"
+    },
     "node_modules/body-parser": {
       "version": "1.20.3",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
@@ -1434,6 +1450,19 @@
         "url": "https://github.com/sponsors/jonschlinkert"
       }
     },
+    "node_modules/prom-client": {
+      "version": "15.1.3",
+      "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz",
+      "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/api": "^1.4.0",
+        "tdigest": "^0.1.1"
+      },
+      "engines": {
+        "node": "^16 || ^18 || >=20"
+      }
+    },
     "node_modules/proxy-addr": {
       "version": "2.0.7",
       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -1873,6 +1902,15 @@
         "express": ">=4.0.0 || >=5.0.0-beta"
       }
     },
+    "node_modules/tdigest": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz",
+      "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==",
+      "license": "MIT",
+      "dependencies": {
+        "bintrees": "1.0.2"
+      }
+    },
     "node_modules/to-regex-range": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",

server/package.json

@@ -24,6 +24,7 @@
     "express": "^4.21.2",
     "http-status-codes": "^2.3.0",
     "mariadb": "^3.4.0",
+    "prom-client": "^15.1.3",
     "reflect-metadata": "^0.2.2",
     "sequelize": "^6.37.5",
     "swagger-jsdoc": "^6.2.8",

server/scripts/locationhub.service

@@ -1,22 +1,29 @@
 [Unit]
-Description=LocationHub
+Description=LocationHub Service
+Documentation=https://git.mosad.xyz/localhorst/LocationHub
 After=network.target systemd-networkd-wait-online.service mysqld.service
 
 [Service]
 Type=simple
 User=locationhub
+Group=locationhub
 WorkingDirectory=/home/locationhub/git/LocationHub/server/
-ExecStart=/usr/bin/npm run dev
+
+# Combine commands for build and start
+ExecStart=/bin/bash -c "/usr/bin/npm run build && /usr/bin/npm run start"
+
+# Restart policies
 Restart=on-failure
-StandardOutput=append:/var/log/LocationHub.log
-StandardError=append:/var/log/LocationHub.log
+RestartSec=5s
+
+# Logging configuration
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=locationhub
+
+# Resource control (optional but helps stability)
+MemoryLimit=512M
+CPUQuota=50%
 
 [Install]
 WantedBy=multi-user.target
-```
-Activate Systemd Job
-```
-systemctl daemon-reload
-systemctl enable locationhub.service
-systemctl start locationhub.service
-systemctl status locationhub.service

server/src/controller/metricsController.ts

@@ -0,0 +1,63 @@
+import express, { Request, Response } from "express";
+import { container } from "tsyringe";
+import { Counter, Gauge, collectDefaultMetrics, register } from "prom-client";
+import { LocationService } from "../services/locationService";
+import { WifiLocationService } from "../services/wifiLocationService";
+import { TtnGatewayReceptionService } from "../services/ttnGatewayReceptionService";
+import { MetricsService } from "../services/metricsService";
+
+const router = express.Router();
+const locationService = container.resolve(LocationService);
+const wifiLocationService = container.resolve(WifiLocationService);
+const ttnGatewayReceptionService = container.resolve(TtnGatewayReceptionService);
+const metricsService = container.resolve(MetricsService);
+
+const requestCounter = metricsService.getRequestCounter();
+const locationsTotal = metricsService.getLocationsTotal();
+const gnssLocationsTotal = metricsService.getGnssLocationsTotal()
+const wifiLocationTotal = metricsService.getWifiLocationTotal();
+const wifiLocationNotResolvable = metricsService.getWifiLocationResolvable();
+const wifiLocationRequestLimitExceeded = metricsService.getLWifiLocationRequestLimitExceeded();
+const ttnGatewayReceptions = metricsService.getTnnGatewayReceptions();
+
+// Define the metrics endpoint
+router.get("/", async (req: Request, res: Response) => {
+  try {
+    const [
+      allLocations,
+      gnssLocations,
+      allWifiLocations,
+      wifiLocationsNotResolvable,
+      wifiLocationsRequestLimitExceeded,
+      allTtnGatewayReceptions
+    ] = await Promise.all([
+      locationService.getAllLocations(),
+      locationService.getAllGnssLocations(),
+      wifiLocationService.getAllWifiLocations(),
+      wifiLocationService.getAllWifiLocationsByNotResolvable(),
+      wifiLocationService.getAllWifiLocationsByRequestLimitExceeded(),
+      ttnGatewayReceptionService.getAllGatewayReceptions()
+    ]);
+
+    locationsTotal.set(allLocations.length);
+    gnssLocationsTotal.set(gnssLocations.length);
+    wifiLocationTotal.set(allWifiLocations.length);
+    wifiLocationNotResolvable.set(wifiLocationsNotResolvable.length);
+    wifiLocationRequestLimitExceeded.set(wifiLocationsRequestLimitExceeded.length);
+    ttnGatewayReceptions.set(allTtnGatewayReceptions.length);
+
+    // Increment the counter with labels
+    requestCounter.inc({ method: req.method, route: req.route.path, status: 200 });
+
+    // Expose metrics in Prometheus format
+    res.set("Content-Type", register.contentType);
+    res.send(await register.metrics());
+  } catch (error) {
+    // Increment the counter for errors
+    requestCounter.inc({ method: req.method, route: req.route.path, status: 500 });
+    console.error("Error running metrics endpoint:", error);
+    res.status(500).json({ error: "Error running metrics endpoint" });
+  }
+});
+
+export default router;

server/src/controller/ttnController.ts

@@ -56,24 +56,24 @@ router.post(
         timestamp: latitudeData?.timestamp
           ? new Date(latitudeData.timestamp)
           : longitudeData?.timestamp
-            ? new Date(longitudeData.timestamp)
-            : undefined,
+          ? new Date(longitudeData.timestamp)
+          : undefined,
       };
-      const wifiTimestamp = (() => {
-        const messages = message.uplink_message.decoded_payload?.messages?.[0];
-        const wifiScan = messages?.find((e: { type: string }) => e.type === "Wi-Fi Scan");
-        return wifiScan?.timestamp ? new Date(wifiScan.timestamp) : undefined;
-      })();
+
+      const wifiMessage =
+        message.uplink_message.decoded_payload?.messages[0].find(
+          (e) => e.type === "Wi-Fi Scan"
+        );
 
       const wifiScans =
-        message.uplink_message.decoded_payload?.messages[0]
-          .find((e) => e.type === "Wi-Fi Scan")
-          ?.measurementValue?.map((w) => ({
-            lp_ttn_end_device_uplinks_id,
-            mac: w.mac,
-            rssi: w.rssi,
-            scanned_at_timestamp: wifiTimestamp,
-          })) ?? [];
+        wifiMessage?.measurementValue?.map((w) => ({
+          lp_ttn_end_device_uplinks_id,
+          mac: w.mac,
+          rssi: w.rssi,
+          scanned_at_utc: wifiMessage?.timestamp
+            ? new Date(wifiMessage.timestamp)
+            : undefined,
+        })) ?? [];
 
       const ttnGatewayReceptions = message.uplink_message.rx_metadata.map(
         (g) => ({
@@ -109,9 +109,9 @@ router.post(
           gnss:
             gnnsLocation.latitude && gnnsLocation.longitude
               ? {
-                latitude: gnnsLocation.latitude,
-                longitude: gnnsLocation.longitude,
-              }
+                  latitude: gnnsLocation.latitude,
+                  longitude: gnnsLocation.longitude,
+                }
               : undefined,
           gnss_timestamp: gnssTimestamp.timestamp,
         });

server/src/index.ts

@@ -10,6 +10,7 @@ import ttnGatewayReceptionRoutes from "./controller/ttnGatewayReceptionControlle
 import wifiLocationRoutes from "./controller/wifiLocationController";
 import wifiLocationHistoryRoutes from "./controller/wifiLocationHistoryController";
 import wifiScanRoutes from "./controller/wifiScanController";
+import metricsRoutes from "./controller/metricsController";
 
 dotenv.config();
 
@@ -26,6 +27,7 @@ app.use("/api/wifi-location", wifiLocationRoutes);
 app.use("/api/wifi-scans", wifiScanRoutes);
 app.use("/api/locations", locationRoutes);
 app.use("/api/ttn", ttnRoutes);
+app.use("/api/metrics", metricsRoutes);
 
 app.listen(PORT, () => {
   console.log(`🚀 Server runs here: http://localhost:${PORT}`);

server/src/repositories/locationRepository.ts

@@ -1,10 +1,11 @@
+import { Attributes, FindOptions } from "sequelize";
 import { injectable } from "tsyringe";
 import { Location } from "../models/location";
 
 @injectable()
 export class LocationRepository {
-  public async findAll() {
-    return await Location.findAll();
+  public async findAll(options?: FindOptions<Attributes<Location>>) {
+    return await Location.findAll(options);
   }
 
   public async findById(id: string) {

server/src/services/locationService.ts

@@ -1,4 +1,5 @@
 import { inject, injectable } from "tsyringe";
+import { Op } from 'sequelize';
 import { Location } from "../models/location";
 import { LocationRepository } from "../repositories/locationRepository";
 import { WifiLocationService } from "./wifiLocationService";
@@ -54,6 +55,15 @@ export class LocationService {
     return this.repository.findAll();
   }
 
+  public async getAllGnssLocations() {
+    return this.repository.findAll({
+      where: {
+        gnss_latitude: { [Op.ne]: null },
+        gnss_longitude: { [Op.ne]: null }
+      }
+    });
+  }
+
   public async getLocationById(id: string) {
     return this.repository.findById(id);
   }

server/src/services/metricsService.ts

@@ -0,0 +1,70 @@
+import {injectable } from "tsyringe";
+import { Counter, Gauge, collectDefaultMetrics} from "prom-client";
+
+// Collect default system metrics (e.g., CPU, memory usage)
+const prefix = 'locationhub_';
+collectDefaultMetrics({ prefix });
+
+@injectable()
+export class MetricsService {
+  constructor(
+  ) { }
+
+  public getRequestCounter() {
+    const requestCounter = new Counter({
+      name: `${prefix}http_requests_total`,
+      help: "Total number of HTTP requests",
+      labelNames: ["method", "route", "status"],
+    });
+    return requestCounter;
+  }
+
+  public getLocationsTotal() {
+    const locationsTotal = new Gauge({
+      name: `${prefix}locations_total`,
+      help: "Total number of location entries in database",
+      labelNames: ["database"],
+    });
+    return locationsTotal;
+  }
+  public getGnssLocationsTotal() {
+    const gnssLocationsTotal = new Gauge({
+      name: `${prefix}gnss_locations_total`,
+      help: "Total number of location entries with GNSS in database",
+      labelNames: ["database"],
+    });
+    return gnssLocationsTotal;
+  }
+  public getWifiLocationTotal() {
+    const wifiLocationTotal = new Gauge({
+      name: `${prefix}wifi_locations_total`,
+      help: "Total number of wifi location entries in database",
+      labelNames: ["database"],
+    });
+    return wifiLocationTotal;
+  }
+  public getWifiLocationResolvable() {
+    const wifiLocationNotResolvable = new Gauge({
+      name: `${prefix}wifi_locations_not_resolvable`,
+      help: "Unresolved number of wifi location entries in database",
+      labelNames: ["database"],
+    });
+    return wifiLocationNotResolvable;
+  }
+  public getLWifiLocationRequestLimitExceeded() {
+    const wifiLocationRequestLimitExceeded = new Gauge({
+      name: `${prefix}wifi_locations_request_limit_exceeded`,
+      help: "Unresolved number of wifi location because request limit exceeded entries in database",
+      labelNames: ["database"],
+    });
+    return wifiLocationRequestLimitExceeded;
+  }
+  public getTnnGatewayReceptions() {
+    const ttnGatewayReceptions = new Gauge({
+      name: `${prefix}ttn_gateway_receptions`,
+      help: "Total number of TTN Gateway receptions entries in database",
+      labelNames: ["database"],
+    });
+    return ttnGatewayReceptions;
+  }
+}

server/src/services/wifiLocationService.ts

@@ -29,6 +29,18 @@ export class WifiLocationService {
     });
   }
 
+  public async getAllWifiLocationsByNotResolvable() {
+    return this.repository.findAll({
+      where: { location_not_resolvable: true },
+    });
+  }
+
+  public async getAllWifiLocationsByRequestLimitExceeded() {
+    return this.repository.findAll({
+      where: { request_limit_exceeded: true },
+    });
+  }
+
   public async getWifiLocationByMac(mac: string) {
     let wifiLocation = await this.repository.findById(mac);
 

server/src/services/wifiScanService.ts

@@ -5,14 +5,14 @@ interface CreateWifiScanParams {
   lp_ttn_end_device_uplinks_id: string;
   mac: string;
   rssi: number;
-  scanned_at_timestamp?: Date;
+  scanned_at_utc?: Date;
 }
 
 interface UpdateWifiScanParams {
   wifi_scan_id: string;
   mac?: string;
   rssi?: number;
-  scanned_at_timestamp?: Date;
+  scanned_at_utc?: Date;
 }
 
 @injectable()