diff --git a/build.gradle b/build.gradle index d5feda5..3cf7e2a 100644 --- a/build.gradle +++ b/build.gradle @@ -1,25 +1,25 @@ plugins { - id 'org.jetbrains.kotlin.jvm' version '1.4.10' - id 'org.jetbrains.kotlin.plugin.spring' version '1.4.10' - id 'org.springframework.boot' version '2.3.3.RELEASE' - id 'io.spring.dependency-management' version '1.0.9.RELEASE' + id 'org.jetbrains.kotlin.jvm' version '1.5.31' + id 'org.jetbrains.kotlin.plugin.spring' version '1.5.31' + id 'org.springframework.boot' version '2.5.5' + id 'io.spring.dependency-management' version '1.0.11.RELEASE' } group 'org.mosad' version '1.2.8' repositories { - jcenter() + mavenCentral() } dependencies { implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.jetbrains.kotlin:kotlin-stdlib' - implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.8' - implementation 'org.jsoup:jsoup:1.13.1' - implementation 'com.google.code.gson:gson:2.8.6' + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2' + implementation 'org.jsoup:jsoup:1.14.3' + implementation 'com.google.code.gson:gson:2.8.8' - testImplementation 'org.junit.jupiter:junit-jupiter:5.6.2' + testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1' } test { @@ -30,9 +30,17 @@ test { } } +def jvmTargetVersion = "11" compileKotlin { - kotlinOptions.jvmTarget = "11" + kotlinOptions.jvmTarget = jvmTargetVersion + kotlinOptions.freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn" +} +compileJava { + targetCompatibility = jvmTargetVersion } compileTestKotlin { - kotlinOptions.jvmTarget = "11" + kotlinOptions.jvmTarget = jvmTargetVersion +} +compileTestJava { + targetCompatibility = jvmTargetVersion } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 12d38de..ffed3a2 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt b/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt index 69e72b5..86160fe 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt @@ -58,7 +58,7 @@ data class TimetableDay(val timeslots: Array> = Array(6) { Arr data class TimetableWeek(val days: Array = Array(6) { TimetableDay() }) -data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, val weekNumberYear: Int = 0, val link: String = "") +data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, var weekNumberYear: Int = 0, val link: String = "") data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek()) diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt b/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt index 9c2ff53..f438870 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt @@ -33,11 +33,13 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import java.io.* import java.util.* +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.Executors import kotlin.collections.ArrayList -import kotlin.collections.HashMap import kotlin.collections.HashSet import kotlin.concurrent.scheduleAtFixedRate +import kotlin.time.Duration +import kotlin.time.ExperimentalTime class CacheController { @@ -46,12 +48,12 @@ class CacheController { scheduledUpdates() } - companion object{ + companion object { private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java) var courseList = CoursesList(CoursesMeta(), sortedMapOf()) - var mensaMenu = MensaMenu(MensaMeta(0,""), MensaWeek(), MensaWeek()) - var timetableList = HashMap() // this list contains all timetables + var mensaMenu = MensaMenu(MensaMeta(0, ""), MensaWeek(), MensaWeek()) + var timetableList = ConcurrentHashMap() // this list contains all timetables /** * get a timetable, since they may not be cached, we need to make sure it's cached, otherwise download @@ -59,36 +61,52 @@ class CacheController { * @param weekIndex request week number (current week = 0) * @return timetable of the course (Type: [TimetableCourseWeek]) */ - fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking { + fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek { // TODO just for testing if (courseName == "TEST_A" || courseName == "TEST_B") { val currentTime = System.currentTimeMillis() / 1000 val timetableLink = "https://mosad.xyz" val weekNumberYear = 0 - val instr = javaClass.getResourceAsStream("/html/Timetable_normal-week.html") + val instr = CacheController::class.java.getResourceAsStream("/html/Timetable_normal-week.html") - val htmlDoc = Jsoup.parse(instr,"UTF-8", "https://www.hs-offenburg.de/") - val timetableTest = TimetableParser().parseTimeTable(htmlDoc) + val timetableParser = + TimetableParser(htmlDoc = Jsoup.parse(instr, "UTF-8", "https://www.hs-offenburg.de/")) + val timetableTest = timetableParser.parseTimeTable() - return@runBlocking TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetableTest) + return TimetableCourseWeek( + TimetableCourseMeta( + currentTime, + courseName, + weekIndex, + weekNumberYear, + timetableLink + ), timetableTest ?: TimetableWeek() + ) } - return@runBlocking timetableList.getOrPut("$courseName-$weekIndex") { + val key = "$courseName-$weekIndex" + return if (timetableList.contains(key)) { + timetableList[key]!! + } else { val timetableLink = courseList.courses[courseName] ?.courseLink ?.replace("week=0", "week=$weekIndex") ?: "" val currentTime = System.currentTimeMillis() / 1000 - var timetable = TimetableWeek() - var weekNumberYear = 0 - val timetableJobs = listOf( - async { timetable = TimetableParser().getTimeTable(timetableLink) }, - async { weekNumberYear = TimetableParser().getWeekNumberYear(timetableLink) } - ) + val timetableParser = TimetableParser(timetableLink) + val weekNumberYear = timetableParser.parseWeekNumberYear() + val timetable = timetableParser.parseTimeTable() - timetableJobs.awaitAll() - TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetable) + TimetableCourseWeek( + TimetableCourseMeta( + currentTime, + courseName, + weekIndex, + weekNumberYear ?: 0, + timetableLink + ), timetable ?: TimetableWeek() + ).also { if (timetable != null) timetableList[key] = it } } } @@ -133,7 +151,7 @@ class CacheController { /** * this function updates the courseList - * during the update process the old data will be returned for a API request + * during the update process the old data will be returned for an API request */ private fun asyncUpdateCourseList() = GlobalScope.launch { CourseListParser().getCourseLinks(StartupController.courseListURL)?.let { @@ -149,7 +167,7 @@ class CacheController { /** * this function updates the mensa menu list - * during the update process the old data will be returned for a API request + * during the update process the old data will be returned for an API request */ private fun asyncUpdateMensa() = GlobalScope.launch { val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL) @@ -165,7 +183,7 @@ class CacheController { /** * this function updates all existing timetables - * during the update process the old data will be returned for a API request + * during the update process the old data will be returned for an API request * a FixedThreadPool is used to make parallel requests for faster updates */ private fun asyncUpdateTimetables() = GlobalScope.launch { @@ -177,7 +195,10 @@ class CacheController { try { timetableList.forEach { timetableCourse -> executor.execute { - timetableCourse.value.timetable = TimetableParser().getTimeTable(timetableCourse.value.meta.link) + val timetableParser = TimetableParser(timetableCourse.value.meta.link) + timetableCourse.value.timetable = timetableParser.parseTimeTable() ?: return@execute + timetableCourse.value.meta.weekNumberYear = + timetableParser.parseWeekNumberYear() ?: return@execute timetableCourse.value.meta.updateTime = System.currentTimeMillis() / 1000 saveTimetableToCache(timetableCourse.value) // save the updated timetable to the cache directory @@ -228,30 +249,39 @@ class CacheController { * update the CourseList every 24h, the Timetables every 3h and the Mensa Menu every hour * doesn't account the change between winter and summer time! */ + @OptIn(ExperimentalTime::class) private fun scheduledUpdates() { val currentTime = System.currentTimeMillis() - val initDelay24h = (86400000 - ((currentTime + 3600000) % 86400000)) + 60000 - val initDelay3h = (10800000 - ((currentTime + 3600000) % 10800000)) + 60000 - val initDelay1h = (3600000 - ((currentTime + 3600000) % 3600000)) + 60000 + + val duration24h = Duration.hours(24).inWholeMilliseconds + val duration3h = Duration.hours(3).inWholeMilliseconds + val duration1h = Duration.hours(1).inWholeMilliseconds + val duration1m = Duration.minutes(1).inWholeMilliseconds + + // Calculate the initial delay to make the update time independent of the start time + fun calcInitDelay(period: Long) = (period - ((currentTime + duration1h) % period)) + duration1m + val initDelay24h = calcInitDelay(duration24h) + val initDelay3h = calcInitDelay(duration3h) + val initDelay1h = calcInitDelay(duration1h) // update courseList every 24 hours (time in ms) - Timer().scheduleAtFixedRate(initDelay24h, 86400000) { + Timer().scheduleAtFixedRate(initDelay24h, duration24h) { asyncUpdateCourseList() } // update all already existing timetables every 3 hours (time in ms) - Timer().scheduleAtFixedRate(initDelay3h, 10800000) { + Timer().scheduleAtFixedRate(initDelay3h, duration3h) { asyncUpdateTimetables() } // update courses every hour (time in ms) - Timer().scheduleAtFixedRate(initDelay1h, 3600000) { + Timer().scheduleAtFixedRate(initDelay1h, duration1h) { asyncUpdateMensa() } // post to status.mosad.xyz every hour, if an API key is present if (StartupController.cachetAPIKey != "0") { - Timer().scheduleAtFixedRate(initDelay1h, 3600000) { + Timer().scheduleAtFixedRate(initDelay1h, duration1h) { CachetAPIController.postTotalRequests() } } diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt b/src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt index a424572..534e3eb 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt @@ -22,7 +22,7 @@ package org.mosad.thecitadelofricks.controller -import org.mosad.thecitadelofricks.controller.StatusController.Companion.getTotalRequests +import org.mosad.thecitadelofricks.controller.StatusController.Companion.totalRequests import org.slf4j.Logger import org.slf4j.LoggerFactory import java.io.BufferedReader @@ -42,8 +42,8 @@ class CachetAPIController { fun postTotalRequests() { try { val url = URL("${StartupController.cachetBaseURL}/api/v1/metrics/1/points") - val jsonInputString = "{\"value\": ${getTotalRequests() -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}" - oldTotalRequests = getTotalRequests() + val jsonInputString = "{\"value\": ${totalRequests -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}" + oldTotalRequests = totalRequests val con = url.openConnection() as HttpURLConnection con.requestMethod = "POST" diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt b/src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt index d26160f..5ec69fc 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt @@ -135,7 +135,8 @@ class StartupController { try { val timetableObject = JsonParser.parseString(bufferedReader.readLine()).asJsonObject val timetable = Gson().fromJson(timetableObject, TimetableCourseWeek().javaClass) - CacheController.timetableList.put("${timetable.meta.courseName}-${timetable.meta.weekIndex}", timetable) + CacheController.timetableList["${timetable.meta.courseName}-${timetable.meta.weekIndex}"] = + timetable } catch (ex: Exception) { logger.error("error while reading cache", ex) } finally { diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/controller/StatusController.kt b/src/main/kotlin/org/mosad/thecitadelofricks/controller/StatusController.kt index b12420a..e32e547 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/controller/StatusController.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/controller/StatusController.kt @@ -39,10 +39,14 @@ class StatusController { companion object { private val logger: Logger = LoggerFactory.getLogger(StatusController::class.java) - private var totalRequests = 0 - private var mensaMenuRequests = 0 - private var courseListRequests = 0 - private var timetableRequests = HashMap() + var totalRequests = 0 + private set + var mensaMenuRequests = 0 + private set + var courseListRequests = 0 + private set + var timetableRequests = HashMap() + private set /** * if a mensamenu/courseList/timetable is requested update the specific and total request count @@ -58,29 +62,10 @@ class StatusController { } fun updateTimetableRequests(courseName: String) { - timetableRequests[courseName] = timetableRequests.getOrPut(courseName) {0} + 1 + timetableRequests[courseName] = (timetableRequests[courseName] ?: 0) + 1 totalRequests++ } - /** - * getters and setters - */ - fun getTotalRequests(): Int { - return totalRequests - } - - fun getMensaMenuRequests(): Int { - return mensaMenuRequests - } - - fun getCourseListRequests(): Int { - return courseListRequests - } - - fun getTimetableRequests(): HashMap { - return timetableRequests - } - fun getStatus(): Status { val currentTime = System.currentTimeMillis() / 1000 val minutes = (currentTime - startTime) % 3600 / 60 @@ -112,10 +97,10 @@ class StatusController { "$days days, $hours:$minutes", apiVersion, softwareVersion, - getTotalRequests(), - getMensaMenuRequests(), - getCourseListRequests(), - getTimetableRequests(), + totalRequests, + mensaMenuRequests, + courseListRequests, + timetableRequests, CacheController.timetableList.size, Date(CacheController.courseList.meta.updateTime * 1000), Date(CacheController.mensaMenu.meta.updateTime * 1000), diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt index fd91e1f..3fc6c4d 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt @@ -22,34 +22,49 @@ package org.mosad.thecitadelofricks.hsoparser +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Semaphore import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.mosad.thecitadelofricks.Lesson import org.mosad.thecitadelofricks.TimetableWeek import org.slf4j.LoggerFactory -class TimetableParser { +/** + * @param timetableURL the URL of the timetable you want to get + * @param htmlDoc the html document to use (the timetableURL will be ignored if this value is present) + */ +class TimetableParser(timetableURL: String? = null, htmlDoc: Document? = null) { private var logger: org.slf4j.Logger = LoggerFactory.getLogger(TimetableParser::class.java) private val days = arrayOf("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday") - /** - * get the timetable from the given url - * the timetable is organised per row not per column; - * Mon 1, Tue 1, Wed 1, Thur 1, Fri 1, Sat 1, Mon 2 and so on - * @param timetableURL the URL of the timetable you want to get - */ - fun getTimeTable(timetableURL: String): TimetableWeek { - return try { - parseTimeTable(Jsoup.connect(timetableURL).get()) - } catch (gex: Exception) { - logger.error("general TimetableParser error", gex) - TimetableWeek() + companion object { + val semaphore = Semaphore(3, 0) + } + + private val htmlDoc: Document? = htmlDoc ?: timetableURL?.let { + runBlocking { + try { + // Only allow sending a limited amount of requests at the same time + semaphore.acquire() + Jsoup.connect(timetableURL).get() + } catch (gex: Exception) { + logger.error("general TimetableParser error", gex) + null + } finally { + semaphore.release() + } } } - fun parseTimeTable(htmlDoc: Document): TimetableWeek { + /** + * parse the timetable from the previously given url + * the timetable is organised per row not per column; + * Mon 1, Tue 1, Wed 1, Thur 1, Fri 1, Sat 1, Mon 2 and so on + */ + fun parseTimeTable(): TimetableWeek? = htmlDoc?.let { val timetableWeek = TimetableWeek() - val rows = htmlDoc.select("table.timetable").select("tr[scope=\"row\"]") + val rows = it.select("table.timetable").select("tr[scope=\"row\"]") var sDay = -1 var sRow = -1 @@ -61,7 +76,7 @@ class TimetableParser { var lessonIndexDay = 0 // the index of the lesson per timeslot // elements are now all lessons, including empty ones - row.select("td.lastcol, td[style]").forEach {element -> + row.select("td.lastcol, td[style]").forEach { element -> // if there is a lecture with rowspan="2", we need to shift everything by one to the left. This is stupid and ugly there needs to bee an API if ((sDay > -1 && sRow > -1) && (sDay == day && ((sRow + 1) == rowIndex))) { @@ -104,8 +119,7 @@ class TimetableParser { lessonIndexDay++ - if (element.hasClass("lastcol")) - { + if (element.hasClass("lastcol")) { day++ lessonIndexDay = 0 } @@ -117,20 +131,10 @@ class TimetableParser { } /** - * get the week number of the year for the timetable - * @param timetableURL the URL of the timetable you want to get + * parse the week number of the year for the timetable */ - fun getWeekNumberYear(timetableURL: String): Int { - return try { - parseWeekNumberYear(Jsoup.connect(timetableURL).get()) - } catch (gex: Exception) { - logger.error("general TimetableParser error", gex) - 0 - } - } - - fun parseWeekNumberYear(htmlDoc: Document): Int { - return htmlDoc.select("h1.timetable-caption").text().substringAfter("- ") + fun parseWeekNumberYear(): Int? = htmlDoc?.let { + it.select("h1.timetable-caption").text().substringAfter("- ") .substringBefore(".").replace(" ", "").toInt() } diff --git a/src/test/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParserTest.kt b/src/test/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParserTest.kt index 9bc7bde..8fb1ead 100644 --- a/src/test/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParserTest.kt +++ b/src/test/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParserTest.kt @@ -32,8 +32,8 @@ class TimetableParserTest { @Test fun parseTimetableNormalWeek() { val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path) - val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/") - val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim() + val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/") + val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim() val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_normal-week.txt").readText().trim() Assertions.assertEquals(expectedTimetable, actualTimetable) @@ -42,8 +42,8 @@ class TimetableParserTest { @Test fun parseTimetableEmptyWeek() { val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_empty-week.html").path) - val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/") - val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim() + val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/") + val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim() val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_empty-week.txt").readText().trim() Assertions.assertEquals(expectedTimetable, actualTimetable) @@ -52,8 +52,8 @@ class TimetableParserTest { @Test fun parseWeekNumberYear() { val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path) - val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/") - val actualWeekNumberYear = TimetableParser().parseWeekNumberYear(htmlDoc) + val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/") + val actualWeekNumberYear = TimetableParser(htmlDoc = htmlDoc).parseWeekNumberYear() Assertions.assertEquals(42, actualWeekNumberYear) }