clean up APICOntroller & new CacheController

* added a CacheController to hold all cache variables and clean up the APIController
* more consistent naming of variables
* update mensaMenu and courseList only if the request was successful

build.gradle

@@ -38,4 +38,4 @@ compileTestKotlin {
 }
 
 group 'org.mosad'
-version '1.1.1'
+version '1.1.2'

src/main/kotlin/org/mosad/thecitadelofricks/APIController.kt

@@ -22,13 +22,8 @@
 
 package org.mosad.thecitadelofricks
 
-import kotlinx.coroutines.GlobalScope
+import org.mosad.thecitadelofricks.CacheController.Companion.courseList
-import kotlinx.coroutines.async
+import org.mosad.thecitadelofricks.CacheController.Companion.mensaMenu
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.runBlocking
-import org.mosad.thecitadelofricks.hsoparser.CourseListParser
-import org.mosad.thecitadelofricks.hsoparser.MensaParser
-import org.mosad.thecitadelofricks.hsoparser.TimetableParser
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import org.springframework.web.bind.annotation.RequestMapping
@@ -40,74 +35,37 @@ import java.time.LocalDateTime
 import java.util.*
 import kotlin.collections.ArrayList
 import kotlin.collections.HashSet
-import kotlin.concurrent.scheduleAtFixedRate
 
 @RestController
 class APIController {
 
-    // TODO clean up and move stuff to a CacheController
+    private val logger: Logger = LoggerFactory.getLogger(APIController::class.java)
-    // Controller stuff
+    private val cache = CacheController()
-    var logger: Logger = LoggerFactory.getLogger(APIController::class.java)
+
-    private var requestCount = 0
+    private val softwareVersion = "1.1.2"
+    private val apiVersion = "1.1.1"
     private val startTime = System.currentTimeMillis() / 1000
-    private val softwareVersion = "1.1.1"
-    private val apiVersion = "1.1.0"
 
-    // hso parser links (hardcoded)
+    private var requestCount = 0
-    private val mensaLink = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
-    private val mensaName = "Offenburg"
-
-    // cache objects
-    private var coursesLinkList = ArrayList<Course>()
-    private var coursesLastUpdate: Long = 0
-
-    private var timetableList = ArrayList<TimetableCourseWeek>() // this list contains all timetables
-
-    private var mensaCurrentWeek = MensaWeek()
-    private var mensaNextWeek = MensaWeek()
-    private var mensaLastUpdate: Long = 0
-
-    init {
-        initUpdates() // without this 5-10 seconds after startup the response will be empty
-
-        val currentTime = System.currentTimeMillis()
-        val delay24h = (86400000 - ((currentTime + 3600000) % 86400000)) + 60000
-        val delay3h = (10800000 - ((currentTime + 3600000) % 10800000)) + 60000
-
-        // update courses every 24 hours (time in ms)
-        Timer().scheduleAtFixedRate(delay24h, 86400000) {
-            asyncUpdateCourses()
-        }
-
-        // update courses every 3 hours (time in ms)
-        Timer().scheduleAtFixedRate(delay3h, 10800000) {
-            asyncUpdateMensa()
-        }
-
-        // update all already existing timetables every 3 hours (time in ms)
-        Timer().scheduleAtFixedRate(delay3h, 10800000) {
-            asyncUpdateTimetables()
-        }
-    }
 
     @Deprecated("courses is replaced by courseList", replaceWith = ReplaceWith("courseList()"))
     @RequestMapping("/courses")
-    fun courses(): CoursesList {
+    fun courses(): CourseList {
         return courseList()
     }
 
     @RequestMapping("/courseList")
-    fun courseList(): CoursesList {
+    fun courseList(): CourseList {
         logger.info("courseList request at ${LocalDateTime.now()}!")
         requestCount++
-        return  CoursesList(CoursesMeta(coursesLastUpdate, coursesLinkList.size), coursesLinkList)
+        return  courseList
     }
 
     @RequestMapping("/mensamenu")
-    fun mensamenu(): Mensa {
+    fun mensamenu(): MensaMenu {
         logger.info("mensamenu request at ${LocalDateTime.now()}!")
         requestCount++
-        return  Mensa(MensaMeta(mensaLastUpdate, mensaName), mensaCurrentWeek, mensaNextWeek)
+        return  mensaMenu
     }
 
     @RequestMapping("/timetable")
@@ -117,7 +75,7 @@ class APIController {
     ): TimetableCourseWeek {
         logger.info("timetable request at ${LocalDateTime.now()}!")
         requestCount++
-        return getTimetable(courseName, week)
+        return cache.getTimetable(courseName, week)
     }
 
     @RequestMapping("/lessonSubjectList")
@@ -130,7 +88,7 @@ class APIController {
         requestCount++
 
         // get every lesson subject for the given week
-        val flatMap = getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() }
+        val flatMap = cache.getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() }
         flatMap.forEach {
             it.stream().filter { x -> x.lessonSubject.isNotEmpty() }.findAny().ifPresent { x -> lessonSubjectList.add(x.lessonSubject) }
         }
@@ -147,7 +105,7 @@ class APIController {
         val lessonList = ArrayList<Lesson>()
 
         // get all lessons from the weeks timetable
-        val flatMap = getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() }
+        val flatMap = cache.getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() }
         flatMap.forEach {
             it.forEach { lesson ->
                 if(lesson.lessonSubject.contains(lessonSubject)) {
@@ -194,105 +152,11 @@ class APIController {
             apiVersion,
             softwareVersion,
             requestCount,
-            Date(coursesLastUpdate * 1000),
+            Date(courseList.meta.updateTime * 1000),
-            Date(mensaLastUpdate * 1000),
+            Date(mensaMenu.meta.updateTime * 1000),
             hsoCode,
             swfrCode
         )
     }
 
-    /**
-     * this function updates the courses list
-     * during the update process the old data will be returned for a API request
-     */
-    private fun asyncUpdateCourses() = GlobalScope.launch {
-        coursesLinkList = CourseListParser().getCourseLinks()
-        coursesLastUpdate = System.currentTimeMillis() / 1000
-
-        logger.info("updated courses successful at ${Date(coursesLastUpdate * 1000)}")
-    }
-
-    /**
-     * this function updates the mensa menu list
-     * during the update process the old data will be returned for a API request
-     */
-    private fun asyncUpdateMensa() = GlobalScope.launch {
-        mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink)
-        mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaLink))
-        mensaLastUpdate = System.currentTimeMillis() / 1000
-
-        logger.info("updated mensamenu successful at ${Date(mensaLastUpdate * 1000)}")
-    }
-
-    /**
-     * this function updates all existing timetables
-     * during the update process the old data will be returned for a API request
-     */
-    private fun asyncUpdateTimetables() = GlobalScope.launch {
-        timetableList.forEach { timetableCourse ->
-            val updateURL = timetableCourse.meta.link
-            timetableCourse.timetable = TimetableParser().getTimeTable(updateURL)
-            timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000
-        }
-        logger.info("updated ${timetableList.size} timetables successful!")
-    }
-
-    /**
-     * this function checks if we need to update the timetable for a given course and if so does it
-     * during the update process the old data will be returned for a API request
-     * update if the last update was 6 hours ago
-     */
-    private fun checkTimetableCourse(courseName: String, week: Int) = runBlocking {
-        val currentTime = System.currentTimeMillis() / 1000
-        var timetable = TimetableWeek()
-        // check if the timetable already exists and is up to date
-        val result = timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null)
-
-        when (result) {
-            // there is no such course yet, create one
-            null -> {
-                val courseLink = coursesLinkList.stream().filter { x -> x.courseName == courseName }.findFirst().orElse(null).courseLink
-                val timetableMeta = TimetableCourseMeta(currentTime, courseName, week, courseLink.replace("week=0","week=$week"))
-
-                val jobTimetable = GlobalScope.async {
-                    timetable = TimetableParser().getTimeTable(timetableMeta.link)
-                }
-
-                jobTimetable.await()
-
-                timetableList.add(TimetableCourseWeek(timetableMeta, timetable))
-                logger.info("added new timetable for $courseName, week $week")
-            }
-        }
-    }
-
-    private fun getTimetable(courseName: String, week: Int): TimetableCourseWeek {
-        checkTimetableCourse(courseName, week) // check if we need to update and perform the update if so
-        return timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null)
-    }
-
-    private fun initUpdates() = runBlocking {
-        // get all courses on startup
-        val jobCourseUpdate = GlobalScope.async{
-            coursesLinkList = CourseListParser().getCourseLinks()
-            coursesLastUpdate = System.currentTimeMillis() / 1000
-        }
-
-        // get the current and next weeks mensa menus
-        val jobCurrentMensa = GlobalScope.async{
-            mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink)
-        }
-
-        val jobNextMensa = GlobalScope.async{
-            mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink)
-            mensaLastUpdate = System.currentTimeMillis() / 1000
-        }
-
-        jobCourseUpdate.await()
-        jobCurrentMensa.await()
-        jobNextMensa.await()
-
-        logger.info("init updates successful")
-    }
-
 }

src/main/kotlin/org/mosad/thecitadelofricks/CacheController.kt

@@ -0,0 +1,180 @@
+/**
+ * TheCitadelofRicks
+ *
+ * Copyright 2019  <seil0@mosad.xyz>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ *
+ */
+
+package org.mosad.thecitadelofricks
+
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.async
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import org.mosad.thecitadelofricks.hsoparser.CourseListParser
+import org.mosad.thecitadelofricks.hsoparser.MensaParser
+import org.mosad.thecitadelofricks.hsoparser.TimetableParser
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import java.util.*
+import kotlin.collections.ArrayList
+import kotlin.concurrent.scheduleAtFixedRate
+
+class CacheController {
+
+    private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
+
+    // hso parser links (hardcoded)
+    private val courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/"
+    private val mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
+    private val mensaName = "Offenburg"
+
+    // cache objects
+    companion object{
+
+        lateinit var courseList: CourseList
+        lateinit var mensaMenu: MensaMenu
+
+        var timetableList = ArrayList<TimetableCourseWeek>() // this list contains all timetables
+    }
+
+    init {
+        initUpdates()
+        scheduledUpdates()
+    }
+
+    fun getTimetable(courseName: String, week: Int): TimetableCourseWeek = runBlocking {
+        val currentTime = System.currentTimeMillis() / 1000
+        var timetable = TimetableWeek()
+        // check if the timetable already exists and is up to date
+
+        when (timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null)) {
+            // there is no such course yet, create one
+            null -> {
+                val courseLink = courseList.courses.stream().filter { x -> x.courseName == courseName }.findFirst().orElse(null).courseLink
+                val timetableMeta = TimetableCourseMeta(currentTime, courseName, week, courseLink.replace("week=0","week=$week"))
+
+                val jobTimetable = GlobalScope.async {
+                    timetable = TimetableParser().getTimeTable(timetableMeta.link)
+                }
+
+                jobTimetable.await()
+
+                timetableList.add(TimetableCourseWeek(timetableMeta, timetable))
+                logger.info("added new timetable for $courseName, week $week")
+            }
+        }
+
+        return@runBlocking timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null)
+    }
+
+    /**
+     * this function updates the courses list
+     * during the update process the old data will be returned for a API request
+     */
+    private fun asyncUpdateCourses() = GlobalScope.launch {
+        val result = CourseListParser().getCourseLinks(courseListURL)
+        if (result != null) {
+            courseList = CourseList(CourseMeta(System.currentTimeMillis() / 1000, result.size), result)
+        }
+
+        logger.info("updated courses successful at ${Date(courseList.meta.updateTime * 1000)}")
+    }
+
+    /**
+     * this function updates the mensa menu list
+     * during the update process the old data will be returned for a API request
+     */
+    private fun asyncUpdateMensa() = GlobalScope.launch {
+        val mensaCurrentWeek = MensaParser().getMensaMenu(mensaMenuURL)
+        val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaMenuURL))
+
+        // only update if we get valid data
+        if (mensaCurrentWeek != null && mensaNextWeek != null) {
+            mensaMenu = MensaMenu(MensaMeta(System.currentTimeMillis() / 1000, mensaName), mensaCurrentWeek, mensaNextWeek)
+        }
+
+        logger.info("updated mensamenu successful at ${Date(mensaMenu.meta.updateTime * 1000)}")
+    }
+
+    /**
+     * this function updates all existing timetables
+     * during the update process the old data will be returned for a API request
+     */
+    private fun asyncUpdateTimetables() = GlobalScope.launch {
+        timetableList.forEach { timetableCourse ->
+            val updateURL = timetableCourse.meta.link
+            timetableCourse.timetable = TimetableParser().getTimeTable(updateURL)
+            timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000
+        }
+        logger.info("updated ${timetableList.size} timetables successful!")
+    }
+
+    /**
+     * before the APIController is up, get the data fist
+     * runBlocking: otherwise the api would return no data to requests for a few seconds after startup
+     */
+    private fun initUpdates() = runBlocking {
+        // get all courses on startup
+        val jobCourseUpdate = GlobalScope.async {
+            val result = CourseListParser().getCourseLinks(courseListURL)
+            if (result != null) {
+                courseList = CourseList(CourseMeta(System.currentTimeMillis() / 1000, result.size), result)
+            }
+        }
+
+        // get the current and next weeks mensa menus
+        val jobMensa = GlobalScope.async{
+            val mensaCurrentWeek = MensaParser().getMensaMenu(mensaMenuURL)
+            val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaMenuURL))
+
+            // only update if we get valid data
+            if (mensaCurrentWeek != null && mensaNextWeek != null) {
+                mensaMenu = MensaMenu(MensaMeta(System.currentTimeMillis() / 1000, mensaName), mensaCurrentWeek, mensaNextWeek)
+            }
+        }
+
+        jobCourseUpdate.await()
+        jobMensa.await()
+
+        logger.info("init updates successful")
+    }
+
+    // TODO rework!
+    private fun scheduledUpdates() {
+        val currentTime = System.currentTimeMillis()
+        val delay24h = (86400000 - ((currentTime + 3600000) % 86400000)) + 60000
+        val delay3h = (10800000 - ((currentTime + 3600000) % 10800000)) + 60000
+
+        // update courses every 24 hours (time in ms)
+        Timer().scheduleAtFixedRate(delay24h, 86400000) {
+            asyncUpdateCourses()
+        }
+
+        // update courses every 3 hours (time in ms)
+        Timer().scheduleAtFixedRate(delay3h, 10800000) {
+            asyncUpdateMensa()
+        }
+
+        // update all already existing timetables every 3 hours (time in ms)
+        Timer().scheduleAtFixedRate(delay3h, 10800000) {
+            asyncUpdateTimetables()
+        }
+    }
+
+}

src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt

@@ -28,9 +28,9 @@ import java.util.*
 // data classes for the course part
 data class Course(val courseName: String, val courseLink: String)
 
-data class CoursesMeta(val updateTime: Long, val totalCourses: Int)
+data class CourseMeta(val updateTime: Long, val totalCourses: Int)
 
-data class CoursesList(val meta: CoursesMeta, val courses: ArrayList<Course>)
+data class CourseList(val meta: CourseMeta, val courses: ArrayList<Course>)
 
 // data classes for the Mensa part
 data class Meal(val day: String, val heading: String, val parts: ArrayList<String>, val additives: String)
@@ -41,7 +41,8 @@ data class MensaWeek(val days: Array<Meals> = Array(7) { Meals(ArrayList()) })
 
 data class MensaMeta(val updateTime: Long, val mensaName: String)
 
-data class Mensa(val meta: MensaMeta, val currentWeek: MensaWeek, val nextWeek: MensaWeek)
+data class MensaMenu(val meta: MensaMeta, val currentWeek: MensaWeek, val nextWeek: MensaWeek) {
+}
 
 // data classes for the timetable part
 data class Lesson(

src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/CourseListParser.kt

@@ -32,13 +32,14 @@ class CourseListParser {
     var logger: org.slf4j.Logger = LoggerFactory.getLogger(MensaParser::class.java)
 
     /**
-     * return a list of all courses at hs-offenburg.de/studium/vorlesungsplaene/
+     * return a list of all courses at courseListURL
-     * @return a ArrayList<Course> with all courses
+     * @param courseListURL the url to the course list page
+     * @return a ArrayList<Course> with all courses or null if the request was not successful
      */
-    fun getCourseLinks(): ArrayList<Course> {
+    fun getCourseLinks(courseListURL: String): ArrayList<Course>? {
         val courseLinkList = ArrayList<Course>()
         try {
-            val courseHTML = Jsoup.connect("https://www.hs-offenburg.de/studium/vorlesungsplaene/").get()
+            val courseHTML = Jsoup.connect(courseListURL).get()
 
             courseHTML.select("ul.index-group").select("li.Class").select("a[href]").forEachIndexed { _, element ->
                 courseLinkList.add(
@@ -50,8 +51,10 @@ class CourseListParser {
             }
         } catch (ex: SocketTimeoutException) {
             logger.warn("timeout from hs-offenburg.de, updating on next attempt!")
+            return null
         } catch (gex: Exception) {
             logger.error("general CourseListParser error", gex)
+            return null
         }
 
         return courseLinkList

src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/MensaParser.kt

@@ -34,12 +34,13 @@ class MensaParser {
 
     /**
      * returns the mensa menu for a week
-     * @param menuLink the url to a mensa menu (swfr)
+     * @param mensaMenuURL the url to a mensa menu (swfr)
+     * @return the menu plan found at menuURL or null if the request was not successful
      */
-    fun getMensaMenu(menuLink: String): MensaWeek {
+    fun getMensaMenu(mensaMenuURL: String): MensaWeek? {
         val mealWeekList = MensaWeek()
         try {
-            val menuHTML = Jsoup.connect(menuLink).timeout(15000).get()
+            val menuHTML = Jsoup.connect(mensaMenuURL).timeout(15000).get()
 
             menuHTML.select("#speiseplan-tabs").select("div.tab-content").select("div.menu-tagesplan")
                 .forEachIndexed { dayIndex, day ->
@@ -56,9 +57,11 @@ class MensaParser {
 
                 }
         } catch (ex: SocketTimeoutException) {
-            logger.warn("timeout from $menuLink, updating on next attempt!")
+            logger.warn("timeout from $mensaMenuURL, updating on next attempt!")
+            return null
         } catch (gex: Exception) {
             logger.error("general MensaParser error", gex)
+            return null
         }
 
         return mealWeekList
@@ -66,10 +69,10 @@ class MensaParser {
 
     /**
      * return the link of the next weeks menus
-     * @param menuLink the current weeks menus link
+     * @param mensaMenuURL the current weeks menus link
      */
-    fun getMenuLinkNextWeek(menuLink: String): String {
+    fun getMenuLinkNextWeek(mensaMenuURL: String): String {
-        val menuHTML = Jsoup.connect(menuLink).get()
+        val menuHTML = Jsoup.connect(mensaMenuURL).get()
 
         return "https://www.swfr.de" + menuHTML.select("#speiseplan-tabs").select("a.next-week").attr("href")
     }

src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt

@@ -37,7 +37,7 @@ class TimetableParser {
      */
     fun getTimeTable(timetableURL: String): TimetableWeek {
         val timetableWeek = TimetableWeek()
-        val scheduleHTML = Jsoup.connect(timetableURL).get() // TODO add a tyr catch block to cover timeouts
+        val scheduleHTML = Jsoup.connect(timetableURL).get() // TODO add a try catch block to cover timeouts
 
         //val week = scheduleHTML.select("h1.timetable-caption").text()
         //println("$week successful!\n")