Merge pull request 'Various improvements' (#17) from hannesbraun/TheCitadelofRicks:fix/timeout-resilience into master

Reviewed-on: #17

build.gradle

@@ -1,25 +1,25 @@
 plugins {
-    id 'org.jetbrains.kotlin.jvm' version '1.4.10'
+    id 'org.jetbrains.kotlin.jvm' version '1.5.31'
-    id 'org.jetbrains.kotlin.plugin.spring' version '1.4.10'
+    id 'org.jetbrains.kotlin.plugin.spring' version '1.5.31'
-    id 'org.springframework.boot' version '2.3.3.RELEASE'
+    id 'org.springframework.boot' version '2.5.5'
-    id 'io.spring.dependency-management' version '1.0.9.RELEASE'
+    id 'io.spring.dependency-management' version '1.0.11.RELEASE'
 }
 
 group 'org.mosad'
 version '1.2.8'
 
 repositories {
-    jcenter()
+    mavenCentral()
 }
 
 dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-web'
     implementation 'org.jetbrains.kotlin:kotlin-stdlib'
-    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.8'
+    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2'
-    implementation 'org.jsoup:jsoup:1.13.1'
+    implementation 'org.jsoup:jsoup:1.14.3'
-    implementation 'com.google.code.gson:gson:2.8.6'
+    implementation 'com.google.code.gson:gson:2.8.8'
 
-    testImplementation 'org.junit.jupiter:junit-jupiter:5.6.2'
+    testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1'
 }
 
 test {
@@ -30,9 +30,17 @@ test {
     }
 }
 
+def jvmTargetVersion = "11"
 compileKotlin {
-    kotlinOptions.jvmTarget = "11"
+    kotlinOptions.jvmTarget = jvmTargetVersion
+    kotlinOptions.freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn"
+}
+compileJava {
+    targetCompatibility = jvmTargetVersion
 }
 compileTestKotlin {
-    kotlinOptions.jvmTarget = "11"
+    kotlinOptions.jvmTarget = jvmTargetVersion
+}
+compileTestJava {
+    targetCompatibility = jvmTargetVersion
 }

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt

@@ -58,7 +58,7 @@ data class TimetableDay(val timeslots: Array<ArrayList<Lesson>> = Array(6) { Arr
 
 data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() })
 
-data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, val weekNumberYear: Int = 0, val link: String = "")
+data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, var weekNumberYear: Int = 0, val link: String = "")
 
 data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek())
 

src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt

@@ -33,11 +33,13 @@ import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import java.io.*
 import java.util.*
+import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.Executors
 import kotlin.collections.ArrayList
-import kotlin.collections.HashMap
 import kotlin.collections.HashSet
 import kotlin.concurrent.scheduleAtFixedRate
+import kotlin.time.Duration
+import kotlin.time.ExperimentalTime
 
 class CacheController {
 
@@ -46,12 +48,12 @@ class CacheController {
         scheduledUpdates()
     }
 
-    companion object{
+    companion object {
         private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
 
         var courseList = CoursesList(CoursesMeta(), sortedMapOf())
-        var mensaMenu = MensaMenu(MensaMeta(0,""), MensaWeek(), MensaWeek())
+        var mensaMenu = MensaMenu(MensaMeta(0, ""), MensaWeek(), MensaWeek())
-        var timetableList = HashMap<String, TimetableCourseWeek>() // this list contains all timetables
+        var timetableList = ConcurrentHashMap<String, TimetableCourseWeek>() // this list contains all timetables
 
         /**
          * get a timetable, since they may not be cached, we need to make sure it's cached, otherwise download
@@ -59,36 +61,52 @@ class CacheController {
          * @param weekIndex request week number (current week = 0)
          * @return timetable of the course (Type: [TimetableCourseWeek])
          */
-        fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking {
+        fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek {
 
             // TODO just for testing
             if (courseName == "TEST_A" || courseName == "TEST_B") {
                 val currentTime = System.currentTimeMillis() / 1000
                 val timetableLink = "https://mosad.xyz"
                 val weekNumberYear = 0
-                val instr = javaClass.getResourceAsStream("/html/Timetable_normal-week.html")
+                val instr = CacheController::class.java.getResourceAsStream("/html/Timetable_normal-week.html")
 
-                val htmlDoc = Jsoup.parse(instr,"UTF-8", "https://www.hs-offenburg.de/")
+                val timetableParser =
-                val timetableTest = TimetableParser().parseTimeTable(htmlDoc)
+                    TimetableParser(htmlDoc = Jsoup.parse(instr, "UTF-8", "https://www.hs-offenburg.de/"))
+                val timetableTest = timetableParser.parseTimeTable()
 
-                return@runBlocking TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetableTest)
+                return TimetableCourseWeek(
+                    TimetableCourseMeta(
+                        currentTime,
+                        courseName,
+                        weekIndex,
+                        weekNumberYear,
+                        timetableLink
+                    ), timetableTest ?: TimetableWeek()
+                )
             }
 
-            return@runBlocking timetableList.getOrPut("$courseName-$weekIndex") {
+            val key = "$courseName-$weekIndex"
+            return if (timetableList.contains(key)) {
+                timetableList[key]!!
+            } else {
                 val timetableLink = courseList.courses[courseName]
                     ?.courseLink
                     ?.replace("week=0", "week=$weekIndex") ?: ""
                 val currentTime = System.currentTimeMillis() / 1000
-                var timetable = TimetableWeek()
-                var weekNumberYear = 0
 
-                val timetableJobs = listOf(
+                val timetableParser = TimetableParser(timetableLink)
-                    async { timetable = TimetableParser().getTimeTable(timetableLink) },
+                val weekNumberYear = timetableParser.parseWeekNumberYear()
-                    async { weekNumberYear = TimetableParser().getWeekNumberYear(timetableLink) }
+                val timetable = timetableParser.parseTimeTable()
-                )
 
-                timetableJobs.awaitAll()
+                TimetableCourseWeek(
-                TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetable)
+                    TimetableCourseMeta(
+                        currentTime,
+                        courseName,
+                        weekIndex,
+                        weekNumberYear ?: 0,
+                        timetableLink
+                    ), timetable ?: TimetableWeek()
+                ).also { if (timetable != null) timetableList[key] = it }
             }
         }
 
@@ -133,7 +151,7 @@ class CacheController {
 
         /**
          * this function updates the courseList
-         * during the update process the old data will be returned for a API request
+         * during the update process the old data will be returned for an API request
          */
         private fun asyncUpdateCourseList() = GlobalScope.launch {
             CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
@@ -149,7 +167,7 @@ class CacheController {
 
         /**
          * this function updates the mensa menu list
-         * during the update process the old data will be returned for a API request
+         * during the update process the old data will be returned for an API request
          */
         private fun asyncUpdateMensa() = GlobalScope.launch {
             val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
@@ -165,7 +183,7 @@ class CacheController {
 
         /**
          * this function updates all existing timetables
-         * during the update process the old data will be returned for a API request
+         * during the update process the old data will be returned for an API request
          * a FixedThreadPool is used to make parallel requests for faster updates
          */
         private fun asyncUpdateTimetables() = GlobalScope.launch {
@@ -177,7 +195,10 @@ class CacheController {
             try {
                 timetableList.forEach { timetableCourse ->
                     executor.execute {
-                        timetableCourse.value.timetable = TimetableParser().getTimeTable(timetableCourse.value.meta.link)
+                        val timetableParser = TimetableParser(timetableCourse.value.meta.link)
+                        timetableCourse.value.timetable = timetableParser.parseTimeTable() ?: return@execute
+                        timetableCourse.value.meta.weekNumberYear =
+                            timetableParser.parseWeekNumberYear() ?: return@execute
                         timetableCourse.value.meta.updateTime = System.currentTimeMillis() / 1000
 
                         saveTimetableToCache(timetableCourse.value) // save the updated timetable to the cache directory
@@ -228,30 +249,39 @@ class CacheController {
          * update the CourseList every 24h, the Timetables every 3h and the Mensa Menu every hour
          * doesn't account the change between winter and summer time!
          */
+        @OptIn(ExperimentalTime::class)
         private fun scheduledUpdates() {
             val currentTime = System.currentTimeMillis()
-            val initDelay24h = (86400000 - ((currentTime + 3600000) % 86400000)) + 60000
+
-            val initDelay3h = (10800000 - ((currentTime + 3600000) % 10800000)) + 60000
+            val duration24h = Duration.hours(24).inWholeMilliseconds
-            val initDelay1h = (3600000 - ((currentTime + 3600000) % 3600000)) + 60000
+            val duration3h = Duration.hours(3).inWholeMilliseconds
+            val duration1h = Duration.hours(1).inWholeMilliseconds
+            val duration1m = Duration.minutes(1).inWholeMilliseconds
+
+            // Calculate the initial delay to make the update time independent of the start time
+            fun calcInitDelay(period: Long) = (period - ((currentTime + duration1h) % period)) + duration1m
+            val initDelay24h = calcInitDelay(duration24h)
+            val initDelay3h = calcInitDelay(duration3h)
+            val initDelay1h = calcInitDelay(duration1h)
 
             // update courseList every 24 hours (time in ms)
-            Timer().scheduleAtFixedRate(initDelay24h, 86400000) {
+            Timer().scheduleAtFixedRate(initDelay24h, duration24h) {
                 asyncUpdateCourseList()
             }
 
             // update all already existing timetables every 3 hours (time in ms)
-            Timer().scheduleAtFixedRate(initDelay3h, 10800000) {
+            Timer().scheduleAtFixedRate(initDelay3h, duration3h) {
                 asyncUpdateTimetables()
             }
 
             // update courses every hour (time in ms)
-            Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
+            Timer().scheduleAtFixedRate(initDelay1h, duration1h) {
                 asyncUpdateMensa()
             }
 
             // post to status.mosad.xyz every hour, if an API key is present
             if (StartupController.cachetAPIKey != "0") {
-                Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
+                Timer().scheduleAtFixedRate(initDelay1h, duration1h) {
                     CachetAPIController.postTotalRequests()
                 }
             }

src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt

@@ -22,7 +22,7 @@
 
 package org.mosad.thecitadelofricks.controller
 
-import org.mosad.thecitadelofricks.controller.StatusController.Companion.getTotalRequests
+import org.mosad.thecitadelofricks.controller.StatusController.Companion.totalRequests
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import java.io.BufferedReader
@@ -42,8 +42,8 @@ class CachetAPIController {
          fun postTotalRequests() {
              try {
                  val url = URL("${StartupController.cachetBaseURL}/api/v1/metrics/1/points")
-                 val jsonInputString = "{\"value\": ${getTotalRequests() -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
+                 val jsonInputString = "{\"value\": ${totalRequests -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
-                 oldTotalRequests = getTotalRequests()
+                 oldTotalRequests = totalRequests
 
                  val con = url.openConnection() as HttpURLConnection
                  con.requestMethod = "POST"

src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt

@@ -135,7 +135,8 @@ class StartupController {
                 try {
                     val timetableObject = JsonParser.parseString(bufferedReader.readLine()).asJsonObject
                     val timetable = Gson().fromJson(timetableObject, TimetableCourseWeek().javaClass)
-                    CacheController.timetableList.put("${timetable.meta.courseName}-${timetable.meta.weekIndex}", timetable)
+                    CacheController.timetableList["${timetable.meta.courseName}-${timetable.meta.weekIndex}"] =
+                        timetable
                 } catch (ex: Exception) {
                     logger.error("error while reading cache", ex)
                 } finally {

src/main/kotlin/org/mosad/thecitadelofricks/controller/StatusController.kt

@@ -39,10 +39,14 @@ class StatusController {
     companion object {
         private val logger: Logger = LoggerFactory.getLogger(StatusController::class.java)
 
-        private var totalRequests = 0
+        var totalRequests = 0
-        private var mensaMenuRequests = 0
+            private set
-        private var courseListRequests = 0
+        var mensaMenuRequests = 0
-        private var timetableRequests = HashMap<String, Int>()
+            private set
+        var courseListRequests = 0
+            private set
+        var timetableRequests = HashMap<String, Int>()
+            private set
 
         /**
          * if a mensamenu/courseList/timetable is requested update the specific and total request count
@@ -58,29 +62,10 @@ class StatusController {
         }
 
         fun updateTimetableRequests(courseName: String) {
-            timetableRequests[courseName] = timetableRequests.getOrPut(courseName) {0} + 1
+            timetableRequests[courseName] = (timetableRequests[courseName] ?: 0) + 1
             totalRequests++
         }
 
-        /**
-         * getters and setters
-         */
-        fun getTotalRequests(): Int {
-            return totalRequests
-        }
-
-        fun getMensaMenuRequests(): Int {
-            return mensaMenuRequests
-        }
-
-        fun getCourseListRequests(): Int {
-            return courseListRequests
-        }
-
-        fun getTimetableRequests(): HashMap<String, Int> {
-            return timetableRequests
-        }
-
         fun getStatus(): Status {
             val currentTime = System.currentTimeMillis() / 1000
             val minutes = (currentTime - startTime) % 3600 / 60
@@ -112,10 +97,10 @@ class StatusController {
                 "$days days, $hours:$minutes",
                 apiVersion,
                 softwareVersion,
-                getTotalRequests(),
+                totalRequests,
-                getMensaMenuRequests(),
+                mensaMenuRequests,
-                getCourseListRequests(),
+                courseListRequests,
-                getTimetableRequests(),
+                timetableRequests,
                 CacheController.timetableList.size,
                 Date(CacheController.courseList.meta.updateTime * 1000),
                 Date(CacheController.mensaMenu.meta.updateTime * 1000),

src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt

@@ -22,34 +22,49 @@
 
 package org.mosad.thecitadelofricks.hsoparser
 
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.sync.Semaphore
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
 import org.mosad.thecitadelofricks.Lesson
 import org.mosad.thecitadelofricks.TimetableWeek
 import org.slf4j.LoggerFactory
 
-class TimetableParser {
+/**
+ * @param timetableURL the URL of the timetable you want to get
+ * @param htmlDoc the html document to use (the timetableURL will be ignored if this value is present)
+ */
+class TimetableParser(timetableURL: String? = null, htmlDoc: Document? = null) {
     private var logger: org.slf4j.Logger = LoggerFactory.getLogger(TimetableParser::class.java)
     private val days = arrayOf("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday")
 
-    /**
+    companion object {
-     * get the timetable from the given url
+        val semaphore = Semaphore(3, 0)
-     * the timetable is organised per row not per column;
+    }
-     * Mon 1, Tue 1, Wed 1, Thur 1, Fri 1, Sat 1, Mon 2 and so on
+
-     * @param timetableURL the URL of the timetable you want to get
+    private val htmlDoc: Document? = htmlDoc ?: timetableURL?.let {
-     */
+        runBlocking {
-    fun getTimeTable(timetableURL: String): TimetableWeek {
+            try {
-        return try {
+                // Only allow sending a limited amount of requests at the same time
-            parseTimeTable(Jsoup.connect(timetableURL).get())
+                semaphore.acquire()
-        } catch (gex: Exception) {
+                Jsoup.connect(timetableURL).get()
-            logger.error("general TimetableParser error", gex)
+            } catch (gex: Exception) {
-            TimetableWeek()
+                logger.error("general TimetableParser error", gex)
+                null
+            } finally {
+                semaphore.release()
+            }
         }
     }
 
-    fun parseTimeTable(htmlDoc: Document): TimetableWeek {
+    /**
+     * parse the timetable from the previously given url
+     * the timetable is organised per row not per column;
+     * Mon 1, Tue 1, Wed 1, Thur 1, Fri 1, Sat 1, Mon 2 and so on
+     */
+    fun parseTimeTable(): TimetableWeek? = htmlDoc?.let {
         val timetableWeek = TimetableWeek()
-        val rows = htmlDoc.select("table.timetable").select("tr[scope=\"row\"]")
+        val rows = it.select("table.timetable").select("tr[scope=\"row\"]")
 
         var sDay = -1
         var sRow = -1
@@ -61,7 +76,7 @@ class TimetableParser {
             var lessonIndexDay = 0 // the index of the lesson per timeslot
 
             // elements are now all lessons, including empty ones
-            row.select("td.lastcol, td[style]").forEach {element ->
+            row.select("td.lastcol, td[style]").forEach { element ->
 
                 // if there is a lecture with rowspan="2", we need to shift everything by one to the left. This is stupid and ugly there needs to bee an API
                 if ((sDay > -1 && sRow > -1) && (sDay == day && ((sRow + 1) == rowIndex))) {
@@ -104,8 +119,7 @@ class TimetableParser {
 
                 lessonIndexDay++
 
-                if (element.hasClass("lastcol"))
+                if (element.hasClass("lastcol")) {
-                {
                     day++
                     lessonIndexDay = 0
                 }
@@ -117,20 +131,10 @@ class TimetableParser {
     }
 
     /**
-     * get the week number of the year for the timetable
+     * parse the week number of the year for the timetable
-     * @param timetableURL the URL of the timetable you want to get
      */
-    fun getWeekNumberYear(timetableURL: String): Int {
+    fun parseWeekNumberYear(): Int? = htmlDoc?.let {
-        return try {
+        it.select("h1.timetable-caption").text().substringAfter("- ")
-            parseWeekNumberYear(Jsoup.connect(timetableURL).get())
-        } catch (gex: Exception) {
-            logger.error("general TimetableParser error", gex)
-            0
-        }
-    }
-
-    fun parseWeekNumberYear(htmlDoc: Document): Int {
-        return htmlDoc.select("h1.timetable-caption").text().substringAfter("- ")
             .substringBefore(".").replace(" ", "").toInt()
     }
 

src/test/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParserTest.kt

@@ -32,8 +32,8 @@ class TimetableParserTest {
     @Test
     fun parseTimetableNormalWeek() {
         val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path)
-        val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
+        val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
-        val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim()
+        val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim()
         val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_normal-week.txt").readText().trim()
 
         Assertions.assertEquals(expectedTimetable, actualTimetable)
@@ -42,8 +42,8 @@ class TimetableParserTest {
     @Test
     fun parseTimetableEmptyWeek() {
         val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_empty-week.html").path)
-        val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
+        val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
-        val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim()
+        val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim()
         val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_empty-week.txt").readText().trim()
 
         Assertions.assertEquals(expectedTimetable, actualTimetable)
@@ -52,8 +52,8 @@ class TimetableParserTest {
     @Test
     fun parseWeekNumberYear() {
         val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path)
-        val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
+        val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
-        val actualWeekNumberYear = TimetableParser().parseWeekNumberYear(htmlDoc)
+        val actualWeekNumberYear = TimetableParser(htmlDoc = htmlDoc).parseWeekNumberYear()
 
         Assertions.assertEquals(42, actualWeekNumberYear)
     }