From 22f17d10e03cb348d5e491b337019acec29291dc Mon Sep 17 00:00:00 2001 From: Hannes Braun Date: Wed, 13 Oct 2021 15:57:26 +0200 Subject: [PATCH] Timetable fixes - Only one request is made to get the timetable HTML document for parsing the timetable and the weekNumberYear - On timeouts or other errors, the cached data won't be overwritten with emptiness anymore - The scheduled updates will now also update the weekNumberYear --- .../org/mosad/thecitadelofricks/DataTypes.kt | 2 +- .../controller/CacheController.kt | 47 +++++++++++++------ .../hsoparser/TimetableParser.kt | 47 ++++++++++--------- .../hsoparser/TimetableParserTest.kt | 12 ++--- 4 files changed, 66 insertions(+), 42 deletions(-) diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt b/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt index 69e72b5..86160fe 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt @@ -58,7 +58,7 @@ data class TimetableDay(val timeslots: Array> = Array(6) { Arr data class TimetableWeek(val days: Array = Array(6) { TimetableDay() }) -data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, val weekNumberYear: Int = 0, val link: String = "") +data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, var weekNumberYear: Int = 0, val link: String = "") data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek()) diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt b/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt index 9c2ff53..76788ba 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt @@ -59,7 +59,7 @@ class CacheController { * @param weekIndex request week number (current week = 0) * @return timetable of the course (Type: [TimetableCourseWeek]) */ - fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking { + fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek { // TODO just for testing if (courseName == "TEST_A" || courseName == "TEST_B") { @@ -68,27 +68,43 @@ class CacheController { val weekNumberYear = 0 val instr = javaClass.getResourceAsStream("/html/Timetable_normal-week.html") - val htmlDoc = Jsoup.parse(instr,"UTF-8", "https://www.hs-offenburg.de/") - val timetableTest = TimetableParser().parseTimeTable(htmlDoc) + val timetableParser = + TimetableParser(htmlDoc = Jsoup.parse(instr, "UTF-8", "https://www.hs-offenburg.de/")) + val timetableTest = timetableParser.parseTimeTable() - return@runBlocking TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetableTest) + return TimetableCourseWeek( + TimetableCourseMeta( + currentTime, + courseName, + weekIndex, + weekNumberYear, + timetableLink + ), timetableTest ?: TimetableWeek() + ) } - return@runBlocking timetableList.getOrPut("$courseName-$weekIndex") { + val key = "$courseName-$weekIndex" + return if (timetableList.contains(key)) { + timetableList[key]!! + } else { val timetableLink = courseList.courses[courseName] ?.courseLink ?.replace("week=0", "week=$weekIndex") ?: "" val currentTime = System.currentTimeMillis() / 1000 - var timetable = TimetableWeek() - var weekNumberYear = 0 - val timetableJobs = listOf( - async { timetable = TimetableParser().getTimeTable(timetableLink) }, - async { weekNumberYear = TimetableParser().getWeekNumberYear(timetableLink) } - ) + val timetableParser = TimetableParser(timetableLink) + val weekNumberYear = timetableParser.parseWeekNumberYear() + val timetable = timetableParser.parseTimeTable() - timetableJobs.awaitAll() - TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetable) + TimetableCourseWeek( + TimetableCourseMeta( + currentTime, + courseName, + weekIndex, + weekNumberYear ?: 0, + timetableLink + ), timetable ?: TimetableWeek() + ).also { timetableList[key] = it } } } @@ -177,7 +193,10 @@ class CacheController { try { timetableList.forEach { timetableCourse -> executor.execute { - timetableCourse.value.timetable = TimetableParser().getTimeTable(timetableCourse.value.meta.link) + val timetableParser = TimetableParser(timetableCourse.value.meta.link) + timetableCourse.value.timetable = timetableParser.parseTimeTable() ?: return@execute + timetableCourse.value.meta.weekNumberYear = + timetableParser.parseWeekNumberYear() ?: return@execute timetableCourse.value.meta.updateTime = System.currentTimeMillis() / 1000 saveTimetableToCache(timetableCourse.value) // save the updated timetable to the cache directory diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt index fd91e1f..c544530 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt @@ -28,26 +28,37 @@ import org.mosad.thecitadelofricks.Lesson import org.mosad.thecitadelofricks.TimetableWeek import org.slf4j.LoggerFactory -class TimetableParser { +/** + * @param timetableURL the URL of the timetable you want to get + * @param htmlDoc the html document to use (the timetableURL will be ignored if this value is present) + */ +class TimetableParser(timetableURL: String? = null, htmlDoc: Document? = null) { private var logger: org.slf4j.Logger = LoggerFactory.getLogger(TimetableParser::class.java) private val days = arrayOf("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday") + private val htmlDoc: Document? = + htmlDoc + ?: if (timetableURL == null) { + null + } else { + try { + Jsoup.connect(timetableURL).get() + } catch (gex: Exception) { + logger.error("general TimetableParser error", gex) + null + } + } + /** - * get the timetable from the given url + * parse the timetable from the previously given url * the timetable is organised per row not per column; * Mon 1, Tue 1, Wed 1, Thur 1, Fri 1, Sat 1, Mon 2 and so on - * @param timetableURL the URL of the timetable you want to get */ - fun getTimeTable(timetableURL: String): TimetableWeek { - return try { - parseTimeTable(Jsoup.connect(timetableURL).get()) - } catch (gex: Exception) { - logger.error("general TimetableParser error", gex) - TimetableWeek() + fun parseTimeTable(): TimetableWeek? { + if (htmlDoc == null) { + return null } - } - fun parseTimeTable(htmlDoc: Document): TimetableWeek { val timetableWeek = TimetableWeek() val rows = htmlDoc.select("table.timetable").select("tr[scope=\"row\"]") @@ -117,19 +128,13 @@ class TimetableParser { } /** - * get the week number of the year for the timetable - * @param timetableURL the URL of the timetable you want to get + * parse the week number of the year for the timetable */ - fun getWeekNumberYear(timetableURL: String): Int { - return try { - parseWeekNumberYear(Jsoup.connect(timetableURL).get()) - } catch (gex: Exception) { - logger.error("general TimetableParser error", gex) - 0 + fun parseWeekNumberYear(): Int? { + if (htmlDoc == null) { + return null } - } - fun parseWeekNumberYear(htmlDoc: Document): Int { return htmlDoc.select("h1.timetable-caption").text().substringAfter("- ") .substringBefore(".").replace(" ", "").toInt() } diff --git a/src/test/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParserTest.kt b/src/test/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParserTest.kt index 9bc7bde..8fb1ead 100644 --- a/src/test/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParserTest.kt +++ b/src/test/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParserTest.kt @@ -32,8 +32,8 @@ class TimetableParserTest { @Test fun parseTimetableNormalWeek() { val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path) - val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/") - val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim() + val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/") + val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim() val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_normal-week.txt").readText().trim() Assertions.assertEquals(expectedTimetable, actualTimetable) @@ -42,8 +42,8 @@ class TimetableParserTest { @Test fun parseTimetableEmptyWeek() { val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_empty-week.html").path) - val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/") - val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim() + val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/") + val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim() val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_empty-week.txt").readText().trim() Assertions.assertEquals(expectedTimetable, actualTimetable) @@ -52,8 +52,8 @@ class TimetableParserTest { @Test fun parseWeekNumberYear() { val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path) - val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/") - val actualWeekNumberYear = TimetableParser().parseWeekNumberYear(htmlDoc) + val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/") + val actualWeekNumberYear = TimetableParser(htmlDoc = htmlDoc).parseWeekNumberYear() Assertions.assertEquals(42, actualWeekNumberYear) }