From 697f5e31675fac7da0b76e725adb9a83282ca1c3 Mon Sep 17 00:00:00 2001 From: Seil0 Date: Thu, 14 Nov 2019 18:43:00 +0100 Subject: [PATCH] added on disk cache for the timetables * update spring-boot 2.1.9 -> 2.1.10 --- build.gradle | 7 +- .../org/mosad/thecitadelofricks/DataTypes.kt | 4 +- .../controller/CacheController.kt | 62 +++++++++------ .../controller/CachetAPIController.kt | 4 +- .../controller/StartupController.kt | 76 +++++++++++-------- 5 files changed, 89 insertions(+), 64 deletions(-) diff --git a/build.gradle b/build.gradle index b329eb4..09a8f46 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,7 @@ buildscript { ext.kotlin_version = '1.3.50' - ext.spring_boot_version = '2.1.9.RELEASE' + ext.spring_boot_version = '2.1.10.RELEASE' repositories { jcenter() @@ -34,8 +34,9 @@ repositories { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.2" - implementation 'org.jsoup:jsoup:1.12.1' - implementation 'org.springframework.boot:spring-boot-starter-web' + implementation "org.jsoup:jsoup:1.12.1" + implementation "org.springframework.boot:spring-boot-starter-web" + implementation "com.google.code.gson:gson:2.8.6" testImplementation("org.junit.jupiter:junit-jupiter:5.5.1") } diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt b/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt index 074576e..684c4ee 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt @@ -56,9 +56,9 @@ data class TimetableDay(val timeslots: Array> = Array(6) { Arr data class TimetableWeek(val days: Array = Array(6) { TimetableDay() }) -data class TimetableCourseMeta(var updateTime: Long, val courseName: String, val weekIndex: Int, val weekNumberYear: Int, val link: String) +data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, val weekNumberYear: Int = 0, val link: String = "") -data class TimetableCourseWeek(val meta: TimetableCourseMeta, var timetable: TimetableWeek) +data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek()) // data classes for the status part diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt b/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt index 0f71c47..e6a2a64 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt @@ -22,6 +22,7 @@ package org.mosad.thecitadelofricks.controller +import com.google.gson.Gson import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.async import kotlinx.coroutines.launch @@ -32,6 +33,9 @@ import org.mosad.thecitadelofricks.hsoparser.MensaParser import org.mosad.thecitadelofricks.hsoparser.TimetableParser import org.slf4j.Logger import org.slf4j.LoggerFactory +import java.io.BufferedWriter +import java.io.File +import java.io.FileWriter import java.util.* import java.util.concurrent.Executors import kotlin.collections.ArrayList @@ -59,7 +63,7 @@ class CacheController { * get a timetable, since they may not cached, we need to make sure it's cached, otherwise download * @param courseName the name of the course to be requested * @param weekIndex request week number (current week = 0) - * @return the timetable of a course (courseName) + * @return timetable of the course (Type: [TimetableCourseWeek]) */ fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking { val currentTime = System.currentTimeMillis() / 1000 @@ -81,17 +85,11 @@ class CacheController { jobTimetable.await() timetableList.add( - TimetableCourseWeek( - TimetableCourseMeta( - currentTime, - courseName, - weekIndex, - weekNumberYear, - timetableLink - ), + TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetable ) ) + logger.info("added new timetable for $courseName, week $weekIndex") } } @@ -122,7 +120,7 @@ class CacheController { * @param courseName the name of the course to be requested * @param lessonSubject the lesson subject to be requested * @param weekIndex request week number (current week = 0) - * @return a ArrayList of every lesson with lessonSubject for one week + * @return a ArrayList<[Lesson]> of every lesson with lessonSubject for one week */ fun getLesson(courseName: String, lessonSubject: String, weekIndex: Int): ArrayList { val lessonList = ArrayList() @@ -130,14 +128,14 @@ class CacheController { // get all lessons from the weeks timetable val flatMap = getTimetable(courseName, weekIndex).timetable.days.flatMap { it.timeslots.asIterable() } flatMap.forEach { + // TODO Java 11 + //it.stream().filter { x -> x.lessonSubject.contains(lessonSubject) }.findAny().ifPresent { x -> println("${x.lessonSubject}, ${x.lessonTeacher}") } + it.forEach { lesson -> if(lesson.lessonSubject.contains(lessonSubject)) { lessonList.add(lesson) } } - - // TODO Java 11 - //it.stream().filter { x -> x.lessonSubject.contains(lessonSubject) }.findAny().ifPresent { x -> println("${x.lessonSubject}, ${x.lessonTeacher}") } } return lessonList @@ -149,10 +147,10 @@ class CacheController { * during the update process the old data will be returned for a API request */ private fun asyncUpdateCourseList() = GlobalScope.launch { - CourseListParser().getCourseLinks(StartupController.getCourseListURL())?.let { + CourseListParser().getCourseLinks(StartupController.courseListURL)?.let { courseList = CourseList( CourseMeta(System.currentTimeMillis() / 1000, it.size), it - ) + ) } logger.info("updated courses successful at ${Date(courseList.meta.updateTime * 1000)}") @@ -163,13 +161,13 @@ class CacheController { * during the update process the old data will be returned for a API request */ private fun asyncUpdateMensa() = GlobalScope.launch { - val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.getMensaMenuURL()) - val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.getMensaMenuURL())) + val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL) + val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.mensaMenuURL)) // only update if we get valid data if (mensaCurrentWeek != null && mensaNextWeek != null) { mensaMenu = MensaMenu( - MensaMeta(System.currentTimeMillis() / 1000, StartupController.getMensaName()), mensaCurrentWeek, mensaNextWeek + MensaMeta(System.currentTimeMillis() / 1000, StartupController.mensaName), mensaCurrentWeek, mensaNextWeek ) } @@ -192,6 +190,8 @@ class CacheController { executor.execute { timetableCourse.timetable = TimetableParser().getTimeTable(timetableCourse.meta.link) timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000 + + saveTimetableToCache(timetableCourse) // save the updated timetable to the cache directory } } @@ -202,6 +202,20 @@ class CacheController { } } + /** + * save a timetable to the cache directory + * this is only call on async updates, it is NOT call when first getting the timetable + * @param timetable a timetable of the type [TimetableCourseWeek] + */ + private fun saveTimetableToCache(timetable: TimetableCourseWeek) { + println(timetable.timetable.toString()) + + val file = File(StartupController.dirTcorCache, "timetable-${timetable.meta.courseName}-${timetable.meta.weekIndex}.json") + val writer = BufferedWriter(FileWriter(file)) + writer.write(Gson().toJson(timetable)) + writer.close() + } + /** * before the APIController is up, get the data fist * runBlocking: otherwise the api would return no data to requests for a few seconds after startup @@ -209,7 +223,7 @@ class CacheController { private fun initUpdates() = runBlocking { // get all courses on startup val jobCourseUpdate = GlobalScope.async { - CourseListParser().getCourseLinks(StartupController.getCourseListURL())?.let { + CourseListParser().getCourseLinks(StartupController.courseListURL)?.let { courseList = CourseList( CourseMeta(System.currentTimeMillis() / 1000, it.size), it ) @@ -218,13 +232,13 @@ class CacheController { // get the current and next weeks mensa menus val jobMensa = GlobalScope.async{ - val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.getMensaMenuURL()) - val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.getMensaMenuURL())) + val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL) + val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.mensaMenuURL)) // only update if we get valid data if (mensaCurrentWeek != null && mensaNextWeek != null) { mensaMenu = MensaMenu( - MensaMeta(System.currentTimeMillis() / 1000, StartupController.getMensaName()), mensaCurrentWeek, mensaNextWeek + MensaMeta(System.currentTimeMillis() / 1000, StartupController.mensaName), mensaCurrentWeek, mensaNextWeek ) } } @@ -236,8 +250,8 @@ class CacheController { } /** + * update the CourseList every 24h, the Timetables every 3h and the Mensa Menu every hour * doesn't account the change between winter and summer time! - * */ private fun scheduledUpdates() { val currentTime = System.currentTimeMillis() @@ -261,7 +275,7 @@ class CacheController { } // post to status.mosad.xyz every hour, if an API key is present - if (StartupController.getCachetAPIKey() != "0") { + if (StartupController.cachetAPIKey != "0") { Timer().scheduleAtFixedRate(initDelay1h, 3600000) { CachetAPIController.postTotalRequests() } diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt b/src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt index 3d6a7fe..0b6b87d 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt @@ -41,7 +41,7 @@ class CachetAPIController { fun postTotalRequests() { try { - val url = URL("${StartupController.getCachetBaseURL()}/api/v1/metrics/1/points") + val url = URL("${StartupController.cachetBaseURL}/api/v1/metrics/1/points") val jsonInputString = "{\"value\": ${getTotalRequests() -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}" oldTotalRequests = getTotalRequests() @@ -49,7 +49,7 @@ class CachetAPIController { con.requestMethod = "POST" con.setRequestProperty("Content-Type", "application/json; utf-8") con.setRequestProperty("Accept", "application/json") - con.setRequestProperty("X-Cachet-Token", StartupController.getCachetAPIKey()) + con.setRequestProperty("X-Cachet-Token", StartupController.cachetAPIKey) con.doOutput = true val os = con.outputStream diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt b/src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt index 9d8b883..96b352e 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt @@ -22,22 +22,31 @@ package org.mosad.thecitadelofricks.controller +import com.google.gson.Gson +import com.google.gson.JsonParser +import org.mosad.thecitadelofricks.TimetableCourseWeek import org.slf4j.Logger import org.slf4j.LoggerFactory -import java.io.File -import java.io.FileInputStream -import java.io.FileOutputStream +import java.io.* import java.util.* class StartupController { private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java) - private val userHome = System.getProperty("user.home") - private val tcorHome = "$userHome/.tcor" - private val dirTcorHome = File(tcorHome) - private val dirTcorCache = File("$tcorHome/cache") - private val fileConfig = File("$tcorHome/config.xml") + companion object { + val userHome: String = System.getProperty("user.home") + val tcorHome = "$userHome/.tcor" + val dirTcorHome = File(tcorHome) + val dirTcorCache = File("$tcorHome/cache") + val fileConfig = File("$tcorHome/config.xml") + + var cachetAPIKey = "0" + var cachetBaseURL = "https://status.mosad.xyz" + var courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/" + var mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/" + var mensaName = "Offenburg" + } init { // if the tcor directory doesn't exist, create it @@ -56,8 +65,14 @@ class StartupController { } else { createConfig() } + + // TODO read cached timetable files, as they are not initially cached + readCachedTimetables() } + /** + * load the config stored in the config.xml file + */ private fun loadConfig() = try { val properties = Properties() properties.loadFromXML(FileInputStream(fileConfig)) @@ -90,6 +105,9 @@ class StartupController { logger.error("error while loading config", ex) } + /** + * create an initial config file + */ private fun createConfig() = try { val properties = Properties() @@ -105,32 +123,24 @@ class StartupController { logger.error("error while creating config", ex) } - companion object { - private var cachetAPIKey = "0" - private var cachetBaseURL = "https://status.mosad.xyz" - private var courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/" - private var mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/" - private var mensaName = "Offenburg" + /** + * read all previously cached timetables + */ + private fun readCachedTimetables() { + dirTcorCache.walkTopDown().forEach { + if (it.isFile && it.name.endsWith(".json")) { + try { + val fileReader = FileReader(it) + val bufferedReader = BufferedReader(fileReader) + val timetableObject = JsonParser.parseString(bufferedReader.readLine()).asJsonObject - fun getCachetAPIKey(): String { - return cachetAPIKey + CacheController.timetableList.add(Gson().fromJson(timetableObject, TimetableCourseWeek().javaClass)) + bufferedReader.close() + fileReader.close() + } catch (ex: Exception) { + logger.error("error while reading cache", ex) + } + } } - - fun getCachetBaseURL(): String { - return cachetBaseURL - } - - fun getCourseListURL(): String { - return courseListURL - } - - fun getMensaMenuURL(): String { - return mensaMenuURL - } - fun getMensaName(): String { - return mensaName - } - } - } \ No newline at end of file