From 574a63b2a3d8aa53eb96a7a414953b659773a78a67a651a43bd79cb38632a8e8 Mon Sep 17 00:00:00 2001 From: localhorst Date: Sat, 11 Jan 2025 21:59:11 +0100 Subject: [PATCH 01/10] add metrics api --- server/src/controller/metricsController.ts | 17 +++++++++++++++++ server/src/index.ts | 2 ++ 2 files changed, 19 insertions(+) create mode 100644 server/src/controller/metricsController.ts diff --git a/server/src/controller/metricsController.ts b/server/src/controller/metricsController.ts new file mode 100644 index 0000000..bd637fe --- /dev/null +++ b/server/src/controller/metricsController.ts @@ -0,0 +1,17 @@ +import express, { Request, Response } from "express"; +import { container } from "tsyringe"; +//import { LocationService } from "../services/locationService"; + +//const locationService = container.resolve(LocationService); +const router = express.Router(); + +router.get("/", async (req: Request, res: Response) => { + try { + console.log("Metric Endpoint triggered") + res.status(200).send(); + } catch (error) { + res.status(500).json({ error: "Error running metrics endpoint" }); + } +}); + +export default router; diff --git a/server/src/index.ts b/server/src/index.ts index a1f4292..708eae7 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -10,6 +10,7 @@ import ttnGatewayReceptionRoutes from "./controller/ttnGatewayReceptionControlle import wifiLocationRoutes from "./controller/wifiLocationController"; import wifiLocationHistoryRoutes from "./controller/wifiLocationHistoryController"; import wifiScanRoutes from "./controller/wifiScanController"; +import metricsRoutes from "./controller/metricsController"; dotenv.config(); @@ -26,6 +27,7 @@ app.use("/api/wifi-location", wifiLocationRoutes); app.use("/api/wifi-scans", wifiScanRoutes); app.use("/api/locations", locationRoutes); app.use("/api/ttn", ttnRoutes); +app.use("/api/metrics", metricsRoutes); app.listen(PORT, () => { console.log(`🚀 Server runs here: http://localhost:${PORT}`); From 9b00853d5a7200727501d9dffa9f05e82c882e6ba638809697471b436f0b1fb7 Mon Sep 17 00:00:00 2001 From: localhorst Date: Sat, 11 Jan 2025 22:59:07 +0100 Subject: [PATCH 02/10] use prom-client --- server/package-lock.json | 38 ++++++++++++++++++++++ server/package.json | 1 + server/src/controller/metricsController.ts | 31 +++++++++++++++--- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/server/package-lock.json b/server/package-lock.json index b8236af..7e49a45 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -14,6 +14,7 @@ "express": "^4.21.2", "http-status-codes": "^2.3.0", "mariadb": "^3.4.0", + "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "sequelize": "^6.37.5", "swagger-jsdoc": "^6.2.8", @@ -120,6 +121,15 @@ "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==", "license": "MIT" }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@scarf/scarf": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@scarf/scarf/-/scarf-1.4.0.tgz", @@ -386,6 +396,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -1434,6 +1450,19 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -1873,6 +1902,15 @@ "express": ">=4.0.0 || >=5.0.0-beta" } }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", diff --git a/server/package.json b/server/package.json index d2fde30..533fd05 100644 --- a/server/package.json +++ b/server/package.json @@ -24,6 +24,7 @@ "express": "^4.21.2", "http-status-codes": "^2.3.0", "mariadb": "^3.4.0", + "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "sequelize": "^6.37.5", "swagger-jsdoc": "^6.2.8", diff --git a/server/src/controller/metricsController.ts b/server/src/controller/metricsController.ts index bd637fe..eb39302 100644 --- a/server/src/controller/metricsController.ts +++ b/server/src/controller/metricsController.ts @@ -1,15 +1,36 @@ import express, { Request, Response } from "express"; -import { container } from "tsyringe"; -//import { LocationService } from "../services/locationService"; +import { Counter, collectDefaultMetrics, register } from "prom-client"; -//const locationService = container.resolve(LocationService); const router = express.Router(); + +// Collect default system metrics (e.g., CPU, memory usage) +const prefix = 'locationhub_'; +collectDefaultMetrics({ prefix }); + +// Define a custom Counter metric +const requestCounter = new Counter({ + name: "http_requests_total", + help: "Total number of HTTP requests", + labelNames: ["method", "route", "status"], // Labels for filtering in Prometheus +}); + +// Define the metrics endpoint router.get("/", async (req: Request, res: Response) => { try { - console.log("Metric Endpoint triggered") - res.status(200).send(); + console.log("Metric Endpoint triggered"); + + // Increment the counter with labels + requestCounter.inc({ method: req.method, route: req.route.path, status: 200 }); + + // Expose metrics in Prometheus format + res.set("Content-Type", register.contentType); + res.send(await register.metrics()); } catch (error) { + // Increment the counter for errors + requestCounter.inc({ method: req.method, route: req.route.path, status: 500 }); + + console.error("Error running metrics endpoint:", error); res.status(500).json({ error: "Error running metrics endpoint" }); } }); From e8154d1e1385c54f6c9525a5d02de54749968c5c0e046245e857218f55f83010 Mon Sep 17 00:00:00 2001 From: localhorst Date: Sat, 11 Jan 2025 23:38:44 +0100 Subject: [PATCH 03/10] wigle api request counter --- server/src/controller/metricsController.ts | 36 +++++++++++++++++++--- server/src/proxy/wigle.ts | 6 ++++ server/src/services/metricsService.ts | 23 ++++++++++++++ 3 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 server/src/services/metricsService.ts diff --git a/server/src/controller/metricsController.ts b/server/src/controller/metricsController.ts index eb39302..71b06a9 100644 --- a/server/src/controller/metricsController.ts +++ b/server/src/controller/metricsController.ts @@ -1,8 +1,14 @@ import express, { Request, Response } from "express"; -import { Counter, collectDefaultMetrics, register } from "prom-client"; +import { container } from "tsyringe"; +import { Counter, Gauge, collectDefaultMetrics, register } from "prom-client"; +import { MetricsService } from "../services/metricsService"; +import { LocationService } from "../services/locationService"; +import { WifiLocationService } from "../services/wifiLocationService"; const router = express.Router(); - +const metricsService = container.resolve(MetricsService); +const locationService = container.resolve(LocationService); +const wifiLocationService = container.resolve(WifiLocationService); // Collect default system metrics (e.g., CPU, memory usage) const prefix = 'locationhub_'; @@ -10,9 +16,27 @@ collectDefaultMetrics({ prefix }); // Define a custom Counter metric const requestCounter = new Counter({ - name: "http_requests_total", + name: `${prefix}http_requests_total`, help: "Total number of HTTP requests", - labelNames: ["method", "route", "status"], // Labels for filtering in Prometheus + labelNames: ["method", "route", "status"], +}); + +const locationsTotal = new Gauge({ + name: `${prefix}locations_total`, + help: "Total number of location entries in database", + labelNames: ["database"], +}); + +const wifiLocationTotal = new Gauge({ + name: `${prefix}wifi_locations_total`, + help: "Total number of wifi location entries in database", + labelNames: ["database"], +}); + +const wigleApiRequest = new Gauge({ + name: `${prefix}wigle_api_requests_total`, + help: "Total number of wifi location entries in database", + labelNames: ["wigle"], }); // Define the metrics endpoint @@ -20,6 +44,10 @@ router.get("/", async (req: Request, res: Response) => { try { console.log("Metric Endpoint triggered"); + locationsTotal.set((await locationService.getAllLocations()).length); + wifiLocationTotal.set((await wifiLocationService.getAllWifiLocations()).length); + wigleApiRequest.set(metricsService.getWigleApiNumberOfRequests()); + // Increment the counter with labels requestCounter.inc({ method: req.method, route: req.route.path, status: 200 }); diff --git a/server/src/proxy/wigle.ts b/server/src/proxy/wigle.ts index 5bc5747..d6903de 100644 --- a/server/src/proxy/wigle.ts +++ b/server/src/proxy/wigle.ts @@ -1,3 +1,8 @@ +import { container } from "tsyringe"; +import { MetricsService } from "../services/metricsService"; + +const metricsService = container.resolve(MetricsService); + interface WigleApiResonse { response?: WifiLocationResponse, status_code: number, @@ -47,6 +52,7 @@ interface Result { export const getLocationForWifi = async ( mac: string ): Promise => { + metricsService.incWigleApiNumberOfRequests(); try { const url = `${process.env.WIGLE_BASE_URL!}${process.env .WIGLE_NETWORK_SEARCH!}?netid=${encodeURIComponent(mac)}`; diff --git a/server/src/services/metricsService.ts b/server/src/services/metricsService.ts new file mode 100644 index 0000000..35e6309 --- /dev/null +++ b/server/src/services/metricsService.ts @@ -0,0 +1,23 @@ +import { inject, injectable } from "tsyringe"; + +@injectable() +export class MetricsService { + private wigleApiNumberOfRequests: number; + + constructor() { + this.wigleApiNumberOfRequests = 0; // Initialize the variable + } + + // Method to increment the request count + public incWigleApiNumberOfRequests() { + console.log("inc") + this.wigleApiNumberOfRequests += 1; // Increment the number of requests + console.log(this.wigleApiNumberOfRequests) + } + + // Method to get the current request count + public getWigleApiNumberOfRequests() { + console.log("get: " + this.wigleApiNumberOfRequests) + return this.wigleApiNumberOfRequests; + } +} \ No newline at end of file From 8a4eadefcb5fe3aa0cebf4d5655262d64893687bfa4936514aaa600f12ac269f Mon Sep 17 00:00:00 2001 From: localhorst Date: Tue, 14 Jan 2025 21:03:44 +0100 Subject: [PATCH 04/10] use db wifi_location as metric source --- server/src/controller/metricsController.ts | 24 ++++++++++++++-------- server/src/proxy/wigle.ts | 6 ------ server/src/services/metricsService.ts | 23 --------------------- server/src/services/wifiLocationService.ts | 12 +++++++++++ 4 files changed, 27 insertions(+), 38 deletions(-) delete mode 100644 server/src/services/metricsService.ts diff --git a/server/src/controller/metricsController.ts b/server/src/controller/metricsController.ts index 71b06a9..65d9438 100644 --- a/server/src/controller/metricsController.ts +++ b/server/src/controller/metricsController.ts @@ -1,12 +1,10 @@ import express, { Request, Response } from "express"; import { container } from "tsyringe"; import { Counter, Gauge, collectDefaultMetrics, register } from "prom-client"; -import { MetricsService } from "../services/metricsService"; import { LocationService } from "../services/locationService"; import { WifiLocationService } from "../services/wifiLocationService"; const router = express.Router(); -const metricsService = container.resolve(MetricsService); const locationService = container.resolve(LocationService); const wifiLocationService = container.resolve(WifiLocationService); @@ -24,21 +22,28 @@ const requestCounter = new Counter({ const locationsTotal = new Gauge({ name: `${prefix}locations_total`, help: "Total number of location entries in database", - labelNames: ["database"], + labelNames: ["database"], }); const wifiLocationTotal = new Gauge({ name: `${prefix}wifi_locations_total`, help: "Total number of wifi location entries in database", - labelNames: ["database"], + labelNames: ["database"], }); -const wigleApiRequest = new Gauge({ - name: `${prefix}wigle_api_requests_total`, - help: "Total number of wifi location entries in database", - labelNames: ["wigle"], +const wifiLocationNotResolvable = new Gauge({ + name: `${prefix}wifi_locations_not_resolvable`, + help: "Unresolved number of wifi location entries in database", + labelNames: ["database"], }); +const wifiLocationRequestLimitExceeded = new Gauge({ + name: `${prefix}wifi_locations_request_limit_exceeded`, + help: "Unresolved number of wifi location because request limit exceeded entries in database", + labelNames: ["database"], +}); + + // Define the metrics endpoint router.get("/", async (req: Request, res: Response) => { try { @@ -46,7 +51,8 @@ router.get("/", async (req: Request, res: Response) => { locationsTotal.set((await locationService.getAllLocations()).length); wifiLocationTotal.set((await wifiLocationService.getAllWifiLocations()).length); - wigleApiRequest.set(metricsService.getWigleApiNumberOfRequests()); + wifiLocationNotResolvable.set((await wifiLocationService.getAllWifiLocationsByNotResolvable).length); + wifiLocationRequestLimitExceeded.set((await wifiLocationService.getAllWifiLocationsByRequestLimitExceeded).length); // Increment the counter with labels requestCounter.inc({ method: req.method, route: req.route.path, status: 200 }); diff --git a/server/src/proxy/wigle.ts b/server/src/proxy/wigle.ts index d6903de..5bc5747 100644 --- a/server/src/proxy/wigle.ts +++ b/server/src/proxy/wigle.ts @@ -1,8 +1,3 @@ -import { container } from "tsyringe"; -import { MetricsService } from "../services/metricsService"; - -const metricsService = container.resolve(MetricsService); - interface WigleApiResonse { response?: WifiLocationResponse, status_code: number, @@ -52,7 +47,6 @@ interface Result { export const getLocationForWifi = async ( mac: string ): Promise => { - metricsService.incWigleApiNumberOfRequests(); try { const url = `${process.env.WIGLE_BASE_URL!}${process.env .WIGLE_NETWORK_SEARCH!}?netid=${encodeURIComponent(mac)}`; diff --git a/server/src/services/metricsService.ts b/server/src/services/metricsService.ts deleted file mode 100644 index 35e6309..0000000 --- a/server/src/services/metricsService.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { inject, injectable } from "tsyringe"; - -@injectable() -export class MetricsService { - private wigleApiNumberOfRequests: number; - - constructor() { - this.wigleApiNumberOfRequests = 0; // Initialize the variable - } - - // Method to increment the request count - public incWigleApiNumberOfRequests() { - console.log("inc") - this.wigleApiNumberOfRequests += 1; // Increment the number of requests - console.log(this.wigleApiNumberOfRequests) - } - - // Method to get the current request count - public getWigleApiNumberOfRequests() { - console.log("get: " + this.wigleApiNumberOfRequests) - return this.wigleApiNumberOfRequests; - } -} \ No newline at end of file diff --git a/server/src/services/wifiLocationService.ts b/server/src/services/wifiLocationService.ts index 2617db8..a635f5f 100644 --- a/server/src/services/wifiLocationService.ts +++ b/server/src/services/wifiLocationService.ts @@ -29,6 +29,18 @@ export class WifiLocationService { }); } + public async getAllWifiLocationsByNotResolvable() { + return this.repository.findAll({ + where: { location_not_resolvable: true }, + }); + } + + public async getAllWifiLocationsByRequestLimitExceeded() { + return this.repository.findAll({ + where: { request_limit_exceeded: true }, + }); + } + public async getWifiLocationByMac(mac: string) { let wifiLocation = await this.repository.findById(mac); From 85e3509731ea92d1277b5edccabc00c9e7f16e3e766f4174109624aaab477b31 Mon Sep 17 00:00:00 2001 From: localhorst Date: Wed, 15 Jan 2025 17:32:31 +0100 Subject: [PATCH 05/10] embarrassing --- server/src/controller/metricsController.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/controller/metricsController.ts b/server/src/controller/metricsController.ts index 65d9438..74edcef 100644 --- a/server/src/controller/metricsController.ts +++ b/server/src/controller/metricsController.ts @@ -51,8 +51,8 @@ router.get("/", async (req: Request, res: Response) => { locationsTotal.set((await locationService.getAllLocations()).length); wifiLocationTotal.set((await wifiLocationService.getAllWifiLocations()).length); - wifiLocationNotResolvable.set((await wifiLocationService.getAllWifiLocationsByNotResolvable).length); - wifiLocationRequestLimitExceeded.set((await wifiLocationService.getAllWifiLocationsByRequestLimitExceeded).length); + wifiLocationNotResolvable.set((await wifiLocationService.getAllWifiLocationsByNotResolvable()).length); + wifiLocationRequestLimitExceeded.set((await wifiLocationService.getAllWifiLocationsByRequestLimitExceeded()).length); // Increment the counter with labels requestCounter.inc({ method: req.method, route: req.route.path, status: 200 }); From abf6b9af82cc00f32251d48b4c8541e9a34cc49036ab7515d1fa39a60f65cefa Mon Sep 17 00:00:00 2001 From: localhorst Date: Thu, 16 Jan 2025 21:50:00 +0100 Subject: [PATCH 06/10] add metric TTN Uplinks --- server/src/controller/metricsController.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/src/controller/metricsController.ts b/server/src/controller/metricsController.ts index 74edcef..69f12bc 100644 --- a/server/src/controller/metricsController.ts +++ b/server/src/controller/metricsController.ts @@ -3,10 +3,12 @@ import { container } from "tsyringe"; import { Counter, Gauge, collectDefaultMetrics, register } from "prom-client"; import { LocationService } from "../services/locationService"; import { WifiLocationService } from "../services/wifiLocationService"; +import { TtnGatewayReceptionService } from "../services/ttnGatewayReceptionService"; const router = express.Router(); const locationService = container.resolve(LocationService); const wifiLocationService = container.resolve(WifiLocationService); +const ttnGatewayReceptionService = container.resolve(TtnGatewayReceptionService); // Collect default system metrics (e.g., CPU, memory usage) const prefix = 'locationhub_'; @@ -43,6 +45,12 @@ const wifiLocationRequestLimitExceeded = new Gauge({ labelNames: ["database"], }); +const ttnGatewayReceptions = new Gauge({ + name: `${prefix}ttn_gateway_receptions`, + help: "Total number of TTN Gateway receptions entries in database", + labelNames: ["database"], +}); + // Define the metrics endpoint router.get("/", async (req: Request, res: Response) => { @@ -53,6 +61,7 @@ router.get("/", async (req: Request, res: Response) => { wifiLocationTotal.set((await wifiLocationService.getAllWifiLocations()).length); wifiLocationNotResolvable.set((await wifiLocationService.getAllWifiLocationsByNotResolvable()).length); wifiLocationRequestLimitExceeded.set((await wifiLocationService.getAllWifiLocationsByRequestLimitExceeded()).length); + ttnGatewayReceptions.set((await ttnGatewayReceptionService.getAllGatewayReceptions()).length); // Increment the counter with labels requestCounter.inc({ method: req.method, route: req.route.path, status: 200 }); From 34167c4d996b68aa3b249e1ccd0cda933d7163dff2894d47f360fc86b57fedc5 Mon Sep 17 00:00:00 2001 From: localhorst Date: Sat, 18 Jan 2025 19:36:44 +0100 Subject: [PATCH 07/10] export metric for gnss locations --- server/src/controller/metricsController.ts | 7 +++++++ server/src/repositories/locationRepository.ts | 5 +++-- server/src/services/locationService.ts | 10 ++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/server/src/controller/metricsController.ts b/server/src/controller/metricsController.ts index 69f12bc..1f62a40 100644 --- a/server/src/controller/metricsController.ts +++ b/server/src/controller/metricsController.ts @@ -27,6 +27,12 @@ const locationsTotal = new Gauge({ labelNames: ["database"], }); +const gnssLocationsTotal = new Gauge({ + name: `${prefix}gnss_locations_total`, + help: "Total number of location entries with GNSS in database", + labelNames: ["database"], +}); + const wifiLocationTotal = new Gauge({ name: `${prefix}wifi_locations_total`, help: "Total number of wifi location entries in database", @@ -58,6 +64,7 @@ router.get("/", async (req: Request, res: Response) => { console.log("Metric Endpoint triggered"); locationsTotal.set((await locationService.getAllLocations()).length); + gnssLocationsTotal.set((await locationService.getAllGnssLocations()).length); wifiLocationTotal.set((await wifiLocationService.getAllWifiLocations()).length); wifiLocationNotResolvable.set((await wifiLocationService.getAllWifiLocationsByNotResolvable()).length); wifiLocationRequestLimitExceeded.set((await wifiLocationService.getAllWifiLocationsByRequestLimitExceeded()).length); diff --git a/server/src/repositories/locationRepository.ts b/server/src/repositories/locationRepository.ts index 5f0a563..e5b4cb4 100644 --- a/server/src/repositories/locationRepository.ts +++ b/server/src/repositories/locationRepository.ts @@ -1,10 +1,11 @@ +import { Attributes, FindOptions } from "sequelize"; import { injectable } from "tsyringe"; import { Location } from "../models/location"; @injectable() export class LocationRepository { - public async findAll() { - return await Location.findAll(); + public async findAll(options?: FindOptions>) { + return await Location.findAll(options); } public async findById(id: string) { diff --git a/server/src/services/locationService.ts b/server/src/services/locationService.ts index cb8f10b..e04c177 100644 --- a/server/src/services/locationService.ts +++ b/server/src/services/locationService.ts @@ -1,4 +1,5 @@ import { inject, injectable } from "tsyringe"; +import { Op } from 'sequelize'; import { Location } from "../models/location"; import { LocationRepository } from "../repositories/locationRepository"; import { WifiLocationService } from "./wifiLocationService"; @@ -52,6 +53,15 @@ export class LocationService { return this.repository.findAll(); } + public async getAllGnssLocations() { + return this.repository.findAll({ + where: { + gnss_latitude: { [Op.ne]: null }, + gnss_longitude: { [Op.ne]: null } + } + }); + } + public async getLocationById(id: string) { return this.repository.findById(id); } From f969b0a4c0e419c72966f33cefd3434ac1cb692f19010a7ca94bbc8f5eb2e928 Mon Sep 17 00:00:00 2001 From: localhorst Date: Sat, 25 Jan 2025 18:50:25 +0100 Subject: [PATCH 08/10] use Promise.all --- server/src/controller/metricsController.ts | 30 +++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/server/src/controller/metricsController.ts b/server/src/controller/metricsController.ts index 1f62a40..03d5caf 100644 --- a/server/src/controller/metricsController.ts +++ b/server/src/controller/metricsController.ts @@ -57,18 +57,31 @@ const ttnGatewayReceptions = new Gauge({ labelNames: ["database"], }); - // Define the metrics endpoint router.get("/", async (req: Request, res: Response) => { try { - console.log("Metric Endpoint triggered"); + const [ + allLocations, + gnssLocations, + allWifiLocations, + wifiLocationsNotResolvable, + wifiLocationsRequestLimitExceeded, + allTtnGatewayReceptions + ] = await Promise.all([ + locationService.getAllLocations(), + locationService.getAllGnssLocations(), + wifiLocationService.getAllWifiLocations(), + wifiLocationService.getAllWifiLocationsByNotResolvable(), + wifiLocationService.getAllWifiLocationsByRequestLimitExceeded(), + ttnGatewayReceptionService.getAllGatewayReceptions() + ]); - locationsTotal.set((await locationService.getAllLocations()).length); - gnssLocationsTotal.set((await locationService.getAllGnssLocations()).length); - wifiLocationTotal.set((await wifiLocationService.getAllWifiLocations()).length); - wifiLocationNotResolvable.set((await wifiLocationService.getAllWifiLocationsByNotResolvable()).length); - wifiLocationRequestLimitExceeded.set((await wifiLocationService.getAllWifiLocationsByRequestLimitExceeded()).length); - ttnGatewayReceptions.set((await ttnGatewayReceptionService.getAllGatewayReceptions()).length); + locationsTotal.set(allLocations.length); + gnssLocationsTotal.set(gnssLocations.length); + wifiLocationTotal.set(allWifiLocations.length); + wifiLocationNotResolvable.set(wifiLocationsNotResolvable.length); + wifiLocationRequestLimitExceeded.set(wifiLocationsRequestLimitExceeded.length); + ttnGatewayReceptions.set(allTtnGatewayReceptions.length); // Increment the counter with labels requestCounter.inc({ method: req.method, route: req.route.path, status: 200 }); @@ -79,7 +92,6 @@ router.get("/", async (req: Request, res: Response) => { } catch (error) { // Increment the counter for errors requestCounter.inc({ method: req.method, route: req.route.path, status: 500 }); - console.error("Error running metrics endpoint:", error); res.status(500).json({ error: "Error running metrics endpoint" }); } From 4d0e84b84a47e1e0a77d60eab89cab041e7a0c4bb5cadc96c530499cf001999e Mon Sep 17 00:00:00 2001 From: localhorst Date: Sat, 25 Jan 2025 19:08:30 +0100 Subject: [PATCH 09/10] move metrics definition to service --- server/src/controller/metricsController.ts | 55 +++-------------- server/src/services/metricsService.ts | 70 ++++++++++++++++++++++ 2 files changed, 79 insertions(+), 46 deletions(-) create mode 100644 server/src/services/metricsService.ts diff --git a/server/src/controller/metricsController.ts b/server/src/controller/metricsController.ts index 03d5caf..4e0b1ff 100644 --- a/server/src/controller/metricsController.ts +++ b/server/src/controller/metricsController.ts @@ -4,58 +4,21 @@ import { Counter, Gauge, collectDefaultMetrics, register } from "prom-client"; import { LocationService } from "../services/locationService"; import { WifiLocationService } from "../services/wifiLocationService"; import { TtnGatewayReceptionService } from "../services/ttnGatewayReceptionService"; +import { MetricsService } from "../services/metricsService"; const router = express.Router(); const locationService = container.resolve(LocationService); const wifiLocationService = container.resolve(WifiLocationService); const ttnGatewayReceptionService = container.resolve(TtnGatewayReceptionService); +const metricsService = container.resolve(MetricsService); -// Collect default system metrics (e.g., CPU, memory usage) -const prefix = 'locationhub_'; -collectDefaultMetrics({ prefix }); - -// Define a custom Counter metric -const requestCounter = new Counter({ - name: `${prefix}http_requests_total`, - help: "Total number of HTTP requests", - labelNames: ["method", "route", "status"], -}); - -const locationsTotal = new Gauge({ - name: `${prefix}locations_total`, - help: "Total number of location entries in database", - labelNames: ["database"], -}); - -const gnssLocationsTotal = new Gauge({ - name: `${prefix}gnss_locations_total`, - help: "Total number of location entries with GNSS in database", - labelNames: ["database"], -}); - -const wifiLocationTotal = new Gauge({ - name: `${prefix}wifi_locations_total`, - help: "Total number of wifi location entries in database", - labelNames: ["database"], -}); - -const wifiLocationNotResolvable = new Gauge({ - name: `${prefix}wifi_locations_not_resolvable`, - help: "Unresolved number of wifi location entries in database", - labelNames: ["database"], -}); - -const wifiLocationRequestLimitExceeded = new Gauge({ - name: `${prefix}wifi_locations_request_limit_exceeded`, - help: "Unresolved number of wifi location because request limit exceeded entries in database", - labelNames: ["database"], -}); - -const ttnGatewayReceptions = new Gauge({ - name: `${prefix}ttn_gateway_receptions`, - help: "Total number of TTN Gateway receptions entries in database", - labelNames: ["database"], -}); +const requestCounter = metricsService.getRequestCounter(); +const locationsTotal = metricsService.getLocationsTotal(); +const gnssLocationsTotal = metricsService.getGnssLocationsTotal() +const wifiLocationTotal = metricsService.getWifiLocationTotal(); +const wifiLocationNotResolvable = metricsService.getWifiLocationResolvable(); +const wifiLocationRequestLimitExceeded = metricsService.getLWifiLocationRequestLimitExceeded(); +const ttnGatewayReceptions = metricsService.getTnnGatewayReceptions(); // Define the metrics endpoint router.get("/", async (req: Request, res: Response) => { diff --git a/server/src/services/metricsService.ts b/server/src/services/metricsService.ts new file mode 100644 index 0000000..3b22898 --- /dev/null +++ b/server/src/services/metricsService.ts @@ -0,0 +1,70 @@ +import {injectable } from "tsyringe"; +import { Counter, Gauge, collectDefaultMetrics} from "prom-client"; + +// Collect default system metrics (e.g., CPU, memory usage) +const prefix = 'locationhub_'; +collectDefaultMetrics({ prefix }); + +@injectable() +export class MetricsService { + constructor( + ) { } + + public getRequestCounter() { + const requestCounter = new Counter({ + name: `${prefix}http_requests_total`, + help: "Total number of HTTP requests", + labelNames: ["method", "route", "status"], + }); + return requestCounter; + } + + public getLocationsTotal() { + const locationsTotal = new Gauge({ + name: `${prefix}locations_total`, + help: "Total number of location entries in database", + labelNames: ["database"], + }); + return locationsTotal; + } + public getGnssLocationsTotal() { + const gnssLocationsTotal = new Gauge({ + name: `${prefix}gnss_locations_total`, + help: "Total number of location entries with GNSS in database", + labelNames: ["database"], + }); + return gnssLocationsTotal; + } + public getWifiLocationTotal() { + const wifiLocationTotal = new Gauge({ + name: `${prefix}wifi_locations_total`, + help: "Total number of wifi location entries in database", + labelNames: ["database"], + }); + return wifiLocationTotal; + } + public getWifiLocationResolvable() { + const wifiLocationNotResolvable = new Gauge({ + name: `${prefix}wifi_locations_not_resolvable`, + help: "Unresolved number of wifi location entries in database", + labelNames: ["database"], + }); + return wifiLocationNotResolvable; + } + public getLWifiLocationRequestLimitExceeded() { + const wifiLocationRequestLimitExceeded = new Gauge({ + name: `${prefix}wifi_locations_request_limit_exceeded`, + help: "Unresolved number of wifi location because request limit exceeded entries in database", + labelNames: ["database"], + }); + return wifiLocationRequestLimitExceeded; + } + public getTnnGatewayReceptions() { + const ttnGatewayReceptions = new Gauge({ + name: `${prefix}ttn_gateway_receptions`, + help: "Total number of TTN Gateway receptions entries in database", + labelNames: ["database"], + }); + return ttnGatewayReceptions; + } +} From 51112b5870ebaac40d054585cd109a08c075f9db6cc9b75f94ec12de9ff46758 Mon Sep 17 00:00:00 2001 From: localhorst Date: Sat, 25 Jan 2025 19:14:18 +0100 Subject: [PATCH 10/10] update readme --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d1e59fb..3aba7f6 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,11 @@ We recommend using [The Things Network](https://www.thethingsnetwork.org/) (TTN) - [Database](#database) - [Configuration](#configuration) 2. [TTN Integration](#ttn-integration) -3. [Add a Location Tracker](#add-a-location-tracker) +3. [Prometheus Metrics](#prometheus-metrics) +4. [Add a Location Tracker](#add-a-location-tracker) - [Onboard SenseCAP T1000-B](#onboard-sensecap-t1000-b) - [Register SenseCAP T1000-B](#register-sensecap-t1000-b) -4. [Testing](#testing) +5. [Testing](#testing) - [Testing Webhook](#testing-webhook) - [Emulating Wigle API](#emulating-wigle-api) @@ -49,6 +50,9 @@ Add a addidtional header: - Type: `authorization` - Value: `Bearer your-very-secure-token-from-the-env-file` +### Prometheus Metrics +Use `https://your.domain.tld/api/metrics` to retrieve useful insides for monitoring. + ## Add a Location Tracker We use the [SenseCAP T1000-B](https://www.seeedstudio.com/SenseCAP-Card-Tracker-T1000-B-p-5698.html) from seeedstudio because of the fair price and multiple location providers. However, you can use any LoRaWAN-enabled tracker that is compatible with TTN and supports the required payload fields.