added on disk cache for the timetables

* update spring-boot 2.1.9 -> 2.1.10

build.gradle

@@ -1,7 +1,7 @@
 
 buildscript {
     ext.kotlin_version = '1.3.50'
-    ext.spring_boot_version = '2.1.9.RELEASE'
+    ext.spring_boot_version = '2.1.10.RELEASE'
 
     repositories {
         jcenter()
@@ -34,8 +34,9 @@ repositories {
 dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
     implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.2"
-    implementation 'org.jsoup:jsoup:1.12.1'
-    implementation 'org.springframework.boot:spring-boot-starter-web'
+    implementation "org.jsoup:jsoup:1.12.1"
+    implementation "org.springframework.boot:spring-boot-starter-web"
+    implementation "com.google.code.gson:gson:2.8.6"
 
     testImplementation("org.junit.jupiter:junit-jupiter:5.5.1")
 }

src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt

@@ -56,9 +56,9 @@ data class TimetableDay(val timeslots: Array<ArrayList<Lesson>> = Array(6) { Arr
 
 data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() })
 
-data class TimetableCourseMeta(var updateTime: Long, val courseName: String, val weekIndex: Int, val weekNumberYear: Int, val link: String)
+data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, val weekNumberYear: Int = 0, val link: String = "")
 
-data class TimetableCourseWeek(val meta: TimetableCourseMeta, var timetable: TimetableWeek)
+data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek())
 
 
 // data classes for the status part

src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt

@@ -22,6 +22,7 @@
 
 package org.mosad.thecitadelofricks.controller
 
+import com.google.gson.Gson
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.async
 import kotlinx.coroutines.launch
@@ -32,6 +33,9 @@ import org.mosad.thecitadelofricks.hsoparser.MensaParser
 import org.mosad.thecitadelofricks.hsoparser.TimetableParser
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
+import java.io.BufferedWriter
+import java.io.File
+import java.io.FileWriter
 import java.util.*
 import java.util.concurrent.Executors
 import kotlin.collections.ArrayList
@@ -59,7 +63,7 @@ class CacheController {
          * get a timetable, since they may not cached, we need to make sure it's cached, otherwise download
          * @param courseName the name of the course to be requested
          * @param weekIndex request week number (current week = 0)
-         * @return the timetable of a course (courseName)
+         * @return timetable of the course (Type: [TimetableCourseWeek])
          */
         fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking {
             val currentTime = System.currentTimeMillis() / 1000
@@ -81,17 +85,11 @@ class CacheController {
                     jobTimetable.await()
 
                     timetableList.add(
-                        TimetableCourseWeek(
-                            TimetableCourseMeta(
-                                currentTime,
-                                courseName,
-                                weekIndex,
-                                weekNumberYear,
-                                timetableLink
-                            ),
+                        TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink),
                             timetable
                         )
                     )
+
                     logger.info("added new timetable for $courseName, week $weekIndex")
                 }
             }
@@ -122,7 +120,7 @@ class CacheController {
          * @param courseName the name of the course to be requested
          * @param lessonSubject the lesson subject to be requested
          * @param weekIndex request week number (current week = 0)
-         * @return a ArrayList of every lesson with lessonSubject for one week
+         * @return a ArrayList<[Lesson]> of every lesson with lessonSubject for one week
          */
         fun getLesson(courseName: String, lessonSubject: String, weekIndex: Int): ArrayList<Lesson> {
             val lessonList = ArrayList<Lesson>()
@@ -130,14 +128,14 @@ class CacheController {
             // get all lessons from the weeks timetable
             val flatMap = getTimetable(courseName, weekIndex).timetable.days.flatMap { it.timeslots.asIterable() }
             flatMap.forEach {
+                // TODO Java 11
+                //it.stream().filter { x -> x.lessonSubject.contains(lessonSubject) }.findAny().ifPresent { x -> println("${x.lessonSubject}, ${x.lessonTeacher}") }
+
                 it.forEach { lesson ->
                     if(lesson.lessonSubject.contains(lessonSubject)) {
                         lessonList.add(lesson)
                     }
                 }
-
-                // TODO Java 11
-                //it.stream().filter { x -> x.lessonSubject.contains(lessonSubject) }.findAny().ifPresent { x -> println("${x.lessonSubject}, ${x.lessonTeacher}") }
             }
 
             return lessonList
@@ -149,10 +147,10 @@ class CacheController {
      * during the update process the old data will be returned for a API request
      */
     private fun asyncUpdateCourseList() = GlobalScope.launch {
-        CourseListParser().getCourseLinks(StartupController.getCourseListURL())?.let {
+        CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
             courseList = CourseList(
                 CourseMeta(System.currentTimeMillis() / 1000, it.size), it
-                )
+            )
         }
 
         logger.info("updated courses successful at ${Date(courseList.meta.updateTime * 1000)}")
@@ -163,13 +161,13 @@ class CacheController {
      * during the update process the old data will be returned for a API request
      */
     private fun asyncUpdateMensa() = GlobalScope.launch {
-        val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.getMensaMenuURL())
-        val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.getMensaMenuURL()))
+        val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
+        val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.mensaMenuURL))
 
         // only update if we get valid data
         if (mensaCurrentWeek != null && mensaNextWeek != null) {
             mensaMenu = MensaMenu(
-                MensaMeta(System.currentTimeMillis() / 1000, StartupController.getMensaName()), mensaCurrentWeek, mensaNextWeek
+                MensaMeta(System.currentTimeMillis() / 1000, StartupController.mensaName), mensaCurrentWeek, mensaNextWeek
             )
         }
 
@@ -192,6 +190,8 @@ class CacheController {
                 executor.execute {
                     timetableCourse.timetable = TimetableParser().getTimeTable(timetableCourse.meta.link)
                     timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000
+
+                    saveTimetableToCache(timetableCourse) // save the updated timetable to the cache directory
                 }
 
             }
@@ -202,6 +202,20 @@ class CacheController {
         }
     }
 
+    /**
+     * save a timetable to the cache directory
+     * this is only call on async updates, it is NOT call when first getting the timetable
+     * @param timetable a timetable of the type [TimetableCourseWeek]
+     */
+    private fun saveTimetableToCache(timetable: TimetableCourseWeek) {
+        println(timetable.timetable.toString())
+
+        val file = File(StartupController.dirTcorCache, "timetable-${timetable.meta.courseName}-${timetable.meta.weekIndex}.json")
+        val writer = BufferedWriter(FileWriter(file))
+        writer.write(Gson().toJson(timetable))
+        writer.close()
+    }
+
     /**
      * before the APIController is up, get the data fist
      * runBlocking: otherwise the api would return no data to requests for a few seconds after startup
@@ -209,7 +223,7 @@ class CacheController {
     private fun initUpdates() = runBlocking {
         // get all courses on startup
         val jobCourseUpdate = GlobalScope.async {
-            CourseListParser().getCourseLinks(StartupController.getCourseListURL())?.let {
+            CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
                 courseList = CourseList(
                     CourseMeta(System.currentTimeMillis() / 1000, it.size), it
                 )
@@ -218,13 +232,13 @@ class CacheController {
 
         // get the current and next weeks mensa menus
         val jobMensa = GlobalScope.async{
-            val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.getMensaMenuURL())
-            val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.getMensaMenuURL()))
+            val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
+            val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.mensaMenuURL))
 
             // only update if we get valid data
             if (mensaCurrentWeek != null && mensaNextWeek != null) {
                 mensaMenu = MensaMenu(
-                    MensaMeta(System.currentTimeMillis() / 1000, StartupController.getMensaName()), mensaCurrentWeek, mensaNextWeek
+                    MensaMeta(System.currentTimeMillis() / 1000, StartupController.mensaName), mensaCurrentWeek, mensaNextWeek
                 )
             }
         }
@@ -236,8 +250,8 @@ class CacheController {
     }
 
     /**
+     * update the CourseList every 24h, the Timetables every 3h and the Mensa Menu every hour
      * doesn't account the change between winter and summer time!
-     *
      */
     private fun scheduledUpdates() {
         val currentTime = System.currentTimeMillis()
@@ -261,7 +275,7 @@ class CacheController {
         }
 
         // post to status.mosad.xyz every hour, if an API key is present
-        if (StartupController.getCachetAPIKey() != "0") {
+        if (StartupController.cachetAPIKey != "0") {
             Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
                 CachetAPIController.postTotalRequests()
             }

src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt

@@ -41,7 +41,7 @@ class CachetAPIController {
 
          fun postTotalRequests() {
              try {
-                 val url = URL("${StartupController.getCachetBaseURL()}/api/v1/metrics/1/points")
+                 val url = URL("${StartupController.cachetBaseURL}/api/v1/metrics/1/points")
                  val jsonInputString = "{\"value\": ${getTotalRequests() -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
                  oldTotalRequests = getTotalRequests()
 
@@ -49,7 +49,7 @@ class CachetAPIController {
                  con.requestMethod = "POST"
                  con.setRequestProperty("Content-Type", "application/json; utf-8")
                  con.setRequestProperty("Accept", "application/json")
-                 con.setRequestProperty("X-Cachet-Token", StartupController.getCachetAPIKey())
+                 con.setRequestProperty("X-Cachet-Token", StartupController.cachetAPIKey)
                  con.doOutput = true
 
                  val os = con.outputStream

src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt

@@ -22,22 +22,31 @@
 
 package org.mosad.thecitadelofricks.controller
 
+import com.google.gson.Gson
+import com.google.gson.JsonParser
+import org.mosad.thecitadelofricks.TimetableCourseWeek
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
-import java.io.File
-import java.io.FileInputStream
-import java.io.FileOutputStream
+import java.io.*
 import java.util.*
 
 class StartupController {
 
     private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
 
-    private val userHome = System.getProperty("user.home")
-    private val tcorHome = "$userHome/.tcor"
-    private val dirTcorHome = File(tcorHome)
-    private val dirTcorCache = File("$tcorHome/cache")
-    private val fileConfig = File("$tcorHome/config.xml")
+    companion object {
+        val userHome: String = System.getProperty("user.home")
+        val tcorHome = "$userHome/.tcor"
+        val dirTcorHome = File(tcorHome)
+        val dirTcorCache = File("$tcorHome/cache")
+        val fileConfig = File("$tcorHome/config.xml")
+
+        var cachetAPIKey = "0"
+        var cachetBaseURL = "https://status.mosad.xyz"
+        var courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/"
+        var mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
+        var mensaName = "Offenburg"
+    }
 
     init {
         // if the tcor directory doesn't exist, create it
@@ -56,8 +65,14 @@ class StartupController {
         } else {
             createConfig()
         }
+
+        // TODO read cached timetable files, as they are not initially cached
+        readCachedTimetables()
     }
 
+    /**
+     * load the config stored in the config.xml file
+     */
     private fun loadConfig() = try {
         val properties = Properties()
         properties.loadFromXML(FileInputStream(fileConfig))
@@ -90,6 +105,9 @@ class StartupController {
         logger.error("error while loading config", ex)
     }
 
+    /**
+     * create an initial config file
+     */
     private fun createConfig() = try {
         val properties = Properties()
 
@@ -105,32 +123,24 @@ class StartupController {
         logger.error("error while creating config", ex)
     }
 
-    companion object {
-        private var cachetAPIKey = "0"
-        private var cachetBaseURL = "https://status.mosad.xyz"
-        private var courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/"
-        private var mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
-        private var mensaName = "Offenburg"
-
-        fun getCachetAPIKey(): String {
-            return cachetAPIKey
+    /**
+     * read all previously cached timetables
+     */
+    private fun readCachedTimetables() {
+        dirTcorCache.walkTopDown().forEach {
+            if (it.isFile && it.name.endsWith(".json")) {
+                try {
+                    val fileReader = FileReader(it)
+                    val bufferedReader = BufferedReader(fileReader)
+                    val timetableObject = JsonParser.parseString(bufferedReader.readLine()).asJsonObject
+
+                    CacheController.timetableList.add(Gson().fromJson(timetableObject, TimetableCourseWeek().javaClass))
+                    bufferedReader.close()
+                    fileReader.close()
+                } catch (ex: Exception) {
+                    logger.error("error while reading cache", ex)
+                }
+            }
         }
-
-        fun getCachetBaseURL(): String {
-            return cachetBaseURL
-        }
-
-        fun getCourseListURL(): String {
-            return courseListURL
-        }
-
-        fun getMensaMenuURL(): String {
-            return mensaMenuURL
-        }
-        fun getMensaName(): String {
-            return mensaName
-        }
-
     }
-
 }