Browse Source

Merge pull request 'Various improvements' (#17) from hannesbraun/TheCitadelofRicks:fix/timeout-resilience into master

Reviewed-on: #17
pull/21/head
Jannik 8 months ago
parent
commit
dc57a0d0c1
  1. 30
      build.gradle
  2. 2
      gradle/wrapper/gradle-wrapper.properties
  3. 2
      src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt
  4. 94
      src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt
  5. 6
      src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt
  6. 3
      src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt
  7. 41
      src/main/kotlin/org/mosad/thecitadelofricks/controller/StatusController.kt
  8. 64
      src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt
  9. 12
      src/test/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParserTest.kt

30
build.gradle

@ -1,25 +1,25 @@
plugins {
id 'org.jetbrains.kotlin.jvm' version '1.4.10'
id 'org.jetbrains.kotlin.plugin.spring' version '1.4.10'
id 'org.springframework.boot' version '2.3.3.RELEASE'
id 'io.spring.dependency-management' version '1.0.9.RELEASE'
id 'org.jetbrains.kotlin.jvm' version '1.5.31'
id 'org.jetbrains.kotlin.plugin.spring' version '1.5.31'
id 'org.springframework.boot' version '2.5.5'
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
}
group 'org.mosad'
version '1.2.8'
repositories {
jcenter()
mavenCentral()
}
dependencies {
implementation 'org.springframework.boot:spring-boot-starter-web'
implementation 'org.jetbrains.kotlin:kotlin-stdlib'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.8'
implementation 'org.jsoup:jsoup:1.13.1'
implementation 'com.google.code.gson:gson:2.8.6'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2'
implementation 'org.jsoup:jsoup:1.14.3'
implementation 'com.google.code.gson:gson:2.8.8'
testImplementation 'org.junit.jupiter:junit-jupiter:5.6.2'
testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1'
}
test {
@ -30,9 +30,17 @@ test {
}
}
def jvmTargetVersion = "11"
compileKotlin {
kotlinOptions.jvmTarget = "11"
kotlinOptions.jvmTarget = jvmTargetVersion
kotlinOptions.freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn"
}
compileJava {
targetCompatibility = jvmTargetVersion
}
compileTestKotlin {
kotlinOptions.jvmTarget = "11"
kotlinOptions.jvmTarget = jvmTargetVersion
}
compileTestJava {
targetCompatibility = jvmTargetVersion
}

2
gradle/wrapper/gradle-wrapper.properties vendored

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

2
src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt

@ -58,7 +58,7 @@ data class TimetableDay(val timeslots: Array<ArrayList<Lesson>> = Array(6) { Arr
data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() })
data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, val weekNumberYear: Int = 0, val link: String = "")
data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, var weekNumberYear: Int = 0, val link: String = "")
data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek())

94
src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt

@ -33,11 +33,13 @@ import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.io.*
import java.util.*
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.Executors
import kotlin.collections.ArrayList
import kotlin.collections.HashMap
import kotlin.collections.HashSet
import kotlin.concurrent.scheduleAtFixedRate
import kotlin.time.Duration
import kotlin.time.ExperimentalTime
class CacheController {
@ -46,12 +48,12 @@ class CacheController {
scheduledUpdates()
}
companion object{
companion object {
private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
var courseList = CoursesList(CoursesMeta(), sortedMapOf())
var mensaMenu = MensaMenu(MensaMeta(0,""), MensaWeek(), MensaWeek())
var timetableList = HashMap<String, TimetableCourseWeek>() // this list contains all timetables
var mensaMenu = MensaMenu(MensaMeta(0, ""), MensaWeek(), MensaWeek())
var timetableList = ConcurrentHashMap<String, TimetableCourseWeek>() // this list contains all timetables
/**
* get a timetable, since they may not be cached, we need to make sure it's cached, otherwise download
@ -59,36 +61,52 @@ class CacheController {
* @param weekIndex request week number (current week = 0)
* @return timetable of the course (Type: [TimetableCourseWeek])
*/
fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking {
fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek {
// TODO just for testing
if (courseName == "TEST_A" || courseName == "TEST_B") {
val currentTime = System.currentTimeMillis() / 1000
val timetableLink = "https://mosad.xyz"
val weekNumberYear = 0
val instr = javaClass.getResourceAsStream("/html/Timetable_normal-week.html")
val htmlDoc = Jsoup.parse(instr,"UTF-8", "https://www.hs-offenburg.de/")
val timetableTest = TimetableParser().parseTimeTable(htmlDoc)
return@runBlocking TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetableTest)
val instr = CacheController::class.java.getResourceAsStream("/html/Timetable_normal-week.html")
val timetableParser =
TimetableParser(htmlDoc = Jsoup.parse(instr, "UTF-8", "https://www.hs-offenburg.de/"))
val timetableTest = timetableParser.parseTimeTable()
return TimetableCourseWeek(
TimetableCourseMeta(
currentTime,
courseName,
weekIndex,
weekNumberYear,
timetableLink
), timetableTest ?: TimetableWeek()
)
}
return@runBlocking timetableList.getOrPut("$courseName-$weekIndex") {
val key = "$courseName-$weekIndex"
return if (timetableList.contains(key)) {
timetableList[key]!!
} else {
val timetableLink = courseList.courses[courseName]
?.courseLink
?.replace("week=0", "week=$weekIndex") ?: ""
val currentTime = System.currentTimeMillis() / 1000
var timetable = TimetableWeek()
var weekNumberYear = 0
val timetableJobs = listOf(
async { timetable = TimetableParser().getTimeTable(timetableLink) },
async { weekNumberYear = TimetableParser().getWeekNumberYear(timetableLink) }
)
timetableJobs.awaitAll()
TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetable)
val timetableParser = TimetableParser(timetableLink)
val weekNumberYear = timetableParser.parseWeekNumberYear()
val timetable = timetableParser.parseTimeTable()
TimetableCourseWeek(
TimetableCourseMeta(
currentTime,
courseName,
weekIndex,
weekNumberYear ?: 0,
timetableLink
), timetable ?: TimetableWeek()
).also { if (timetable != null) timetableList[key] = it }
}
}
@ -133,7 +151,7 @@ class CacheController {
/**
* this function updates the courseList
* during the update process the old data will be returned for a API request
* during the update process the old data will be returned for an API request
*/
private fun asyncUpdateCourseList() = GlobalScope.launch {
CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
@ -149,7 +167,7 @@ class CacheController {
/**
* this function updates the mensa menu list
* during the update process the old data will be returned for a API request
* during the update process the old data will be returned for an API request
*/
private fun asyncUpdateMensa() = GlobalScope.launch {
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
@ -165,7 +183,7 @@ class CacheController {
/**
* this function updates all existing timetables
* during the update process the old data will be returned for a API request
* during the update process the old data will be returned for an API request
* a FixedThreadPool is used to make parallel requests for faster updates
*/
private fun asyncUpdateTimetables() = GlobalScope.launch {
@ -177,7 +195,10 @@ class CacheController {
try {
timetableList.forEach { timetableCourse ->
executor.execute {
timetableCourse.value.timetable = TimetableParser().getTimeTable(timetableCourse.value.meta.link)
val timetableParser = TimetableParser(timetableCourse.value.meta.link)
timetableCourse.value.timetable = timetableParser.parseTimeTable() ?: return@execute
timetableCourse.value.meta.weekNumberYear =
timetableParser.parseWeekNumberYear() ?: return@execute
timetableCourse.value.meta.updateTime = System.currentTimeMillis() / 1000
saveTimetableToCache(timetableCourse.value) // save the updated timetable to the cache directory
@ -228,30 +249,39 @@ class CacheController {
* update the CourseList every 24h, the Timetables every 3h and the Mensa Menu every hour
* doesn't account the change between winter and summer time!
*/
@OptIn(ExperimentalTime::class)
private fun scheduledUpdates() {
val currentTime = System.currentTimeMillis()
val initDelay24h = (86400000 - ((currentTime + 3600000) % 86400000)) + 60000
val initDelay3h = (10800000 - ((currentTime + 3600000) % 10800000)) + 60000
val initDelay1h = (3600000 - ((currentTime + 3600000) % 3600000)) + 60000
val duration24h = Duration.hours(24).inWholeMilliseconds
val duration3h = Duration.hours(3).inWholeMilliseconds
val duration1h = Duration.hours(1).inWholeMilliseconds
val duration1m = Duration.minutes(1).inWholeMilliseconds
// Calculate the initial delay to make the update time independent of the start time
fun calcInitDelay(period: Long) = (period - ((currentTime + duration1h) % period)) + duration1m
val initDelay24h = calcInitDelay(duration24h)
val initDelay3h = calcInitDelay(duration3h)
val initDelay1h = calcInitDelay(duration1h)
// update courseList every 24 hours (time in ms)
Timer().scheduleAtFixedRate(initDelay24h, 86400000) {
Timer().scheduleAtFixedRate(initDelay24h, duration24h) {
asyncUpdateCourseList()
}
// update all already existing timetables every 3 hours (time in ms)
Timer().scheduleAtFixedRate(initDelay3h, 10800000) {
Timer().scheduleAtFixedRate(initDelay3h, duration3h) {
asyncUpdateTimetables()
}
// update courses every hour (time in ms)
Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
Timer().scheduleAtFixedRate(initDelay1h, duration1h) {
asyncUpdateMensa()
}
// post to status.mosad.xyz every hour, if an API key is present
if (StartupController.cachetAPIKey != "0") {
Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
Timer().scheduleAtFixedRate(initDelay1h, duration1h) {
CachetAPIController.postTotalRequests()
}
}

6
src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt

@ -22,7 +22,7 @@
package org.mosad.thecitadelofricks.controller
import org.mosad.thecitadelofricks.controller.StatusController.Companion.getTotalRequests
import org.mosad.thecitadelofricks.controller.StatusController.Companion.totalRequests
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.io.BufferedReader
@ -42,8 +42,8 @@ class CachetAPIController {
fun postTotalRequests() {
try {
val url = URL("${StartupController.cachetBaseURL}/api/v1/metrics/1/points")
val jsonInputString = "{\"value\": ${getTotalRequests() -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
oldTotalRequests = getTotalRequests()
val jsonInputString = "{\"value\": ${totalRequests -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
oldTotalRequests = totalRequests
val con = url.openConnection() as HttpURLConnection
con.requestMethod = "POST"

3
src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt

@ -135,7 +135,8 @@ class StartupController {
try {
val timetableObject = JsonParser.parseString(bufferedReader.readLine()).asJsonObject
val timetable = Gson().fromJson(timetableObject, TimetableCourseWeek().javaClass)
CacheController.timetableList.put("${timetable.meta.courseName}-${timetable.meta.weekIndex}", timetable)
CacheController.timetableList["${timetable.meta.courseName}-${timetable.meta.weekIndex}"] =
timetable
} catch (ex: Exception) {
logger.error("error while reading cache", ex)
} finally {

41
src/main/kotlin/org/mosad/thecitadelofricks/controller/StatusController.kt

@ -39,10 +39,14 @@ class StatusController {
companion object {
private val logger: Logger = LoggerFactory.getLogger(StatusController::class.java)
private var totalRequests = 0
private var mensaMenuRequests = 0
private var courseListRequests = 0
private var timetableRequests = HashMap<String, Int>()
var totalRequests = 0
private set
var mensaMenuRequests = 0
private set
var courseListRequests = 0
private set
var timetableRequests = HashMap<String, Int>()
private set
/**
* if a mensamenu/courseList/timetable is requested update the specific and total request count
@ -58,29 +62,10 @@ class StatusController {
}
fun updateTimetableRequests(courseName: String) {
timetableRequests[courseName] = timetableRequests.getOrPut(courseName) {0} + 1
timetableRequests[courseName] = (timetableRequests[courseName] ?: 0) + 1
totalRequests++
}
/**
* getters and setters
*/
fun getTotalRequests(): Int {
return totalRequests
}
fun getMensaMenuRequests(): Int {
return mensaMenuRequests
}
fun getCourseListRequests(): Int {
return courseListRequests
}
fun getTimetableRequests(): HashMap<String, Int> {
return timetableRequests
}
fun getStatus(): Status {
val currentTime = System.currentTimeMillis() / 1000
val minutes = (currentTime - startTime) % 3600 / 60
@ -112,10 +97,10 @@ class StatusController {
"$days days, $hours:$minutes",
apiVersion,
softwareVersion,
getTotalRequests(),
getMensaMenuRequests(),
getCourseListRequests(),
getTimetableRequests(),
totalRequests,
mensaMenuRequests,
courseListRequests,
timetableRequests,
CacheController.timetableList.size,
Date(CacheController.courseList.meta.updateTime * 1000),
Date(CacheController.mensaMenu.meta.updateTime * 1000),

64
src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt

@ -22,34 +22,49 @@
package org.mosad.thecitadelofricks.hsoparser
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.Semaphore
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.mosad.thecitadelofricks.Lesson
import org.mosad.thecitadelofricks.TimetableWeek
import org.slf4j.LoggerFactory
class TimetableParser {
/**
* @param timetableURL the URL of the timetable you want to get
* @param htmlDoc the html document to use (the timetableURL will be ignored if this value is present)
*/
class TimetableParser(timetableURL: String? = null, htmlDoc: Document? = null) {
private var logger: org.slf4j.Logger = LoggerFactory.getLogger(TimetableParser::class.java)
private val days = arrayOf("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday")
companion object {
val semaphore = Semaphore(3, 0)
}
private val htmlDoc: Document? = htmlDoc ?: timetableURL?.let {
runBlocking {
try {
// Only allow sending a limited amount of requests at the same time
semaphore.acquire()
Jsoup.connect(timetableURL).get()
} catch (gex: Exception) {
logger.error("general TimetableParser error", gex)
null
} finally {
semaphore.release()
}
}
}
/**
* get the timetable from the given url
* parse the timetable from the previously given url
* the timetable is organised per row not per column;
* Mon 1, Tue 1, Wed 1, Thur 1, Fri 1, Sat 1, Mon 2 and so on
* @param timetableURL the URL of the timetable you want to get
*/
fun getTimeTable(timetableURL: String): TimetableWeek {
return try {
parseTimeTable(Jsoup.connect(timetableURL).get())
} catch (gex: Exception) {
logger.error("general TimetableParser error", gex)
TimetableWeek()
}
}
fun parseTimeTable(htmlDoc: Document): TimetableWeek {
fun parseTimeTable(): TimetableWeek? = htmlDoc?.let {
val timetableWeek = TimetableWeek()
val rows = htmlDoc.select("table.timetable").select("tr[scope=\"row\"]")
val rows = it.select("table.timetable").select("tr[scope=\"row\"]")
var sDay = -1
var sRow = -1
@ -61,7 +76,7 @@ class TimetableParser {
var lessonIndexDay = 0 // the index of the lesson per timeslot
// elements are now all lessons, including empty ones
row.select("td.lastcol, td[style]").forEach {element ->
row.select("td.lastcol, td[style]").forEach { element ->
// if there is a lecture with rowspan="2", we need to shift everything by one to the left. This is stupid and ugly there needs to bee an API
if ((sDay > -1 && sRow > -1) && (sDay == day && ((sRow + 1) == rowIndex))) {
@ -104,8 +119,7 @@ class TimetableParser {
lessonIndexDay++
if (element.hasClass("lastcol"))
{
if (element.hasClass("lastcol")) {
day++
lessonIndexDay = 0
}
@ -117,20 +131,10 @@ class TimetableParser {
}
/**
* get the week number of the year for the timetable
* @param timetableURL the URL of the timetable you want to get
* parse the week number of the year for the timetable
*/
fun getWeekNumberYear(timetableURL: String): Int {
return try {
parseWeekNumberYear(Jsoup.connect(timetableURL).get())
} catch (gex: Exception) {
logger.error("general TimetableParser error", gex)
0
}
}
fun parseWeekNumberYear(htmlDoc: Document): Int {
return htmlDoc.select("h1.timetable-caption").text().substringAfter("- ")
fun parseWeekNumberYear(): Int? = htmlDoc?.let {
it.select("h1.timetable-caption").text().substringAfter("- ")
.substringBefore(".").replace(" ", "").toInt()
}

12
src/test/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParserTest.kt

@ -32,8 +32,8 @@ class TimetableParserTest {
@Test
fun parseTimetableNormalWeek() {
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path)
val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim()
val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim()
val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_normal-week.txt").readText().trim()
Assertions.assertEquals(expectedTimetable, actualTimetable)
@ -42,8 +42,8 @@ class TimetableParserTest {
@Test
fun parseTimetableEmptyWeek() {
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_empty-week.html").path)
val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim()
val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim()
val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_empty-week.txt").readText().trim()
Assertions.assertEquals(expectedTimetable, actualTimetable)
@ -52,8 +52,8 @@ class TimetableParserTest {
@Test
fun parseWeekNumberYear() {
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path)
val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
val actualWeekNumberYear = TimetableParser().parseWeekNumberYear(htmlDoc)
val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
val actualWeekNumberYear = TimetableParser(htmlDoc = htmlDoc).parseWeekNumberYear()
Assertions.assertEquals(42, actualWeekNumberYear)
}

Loading…
Cancel
Save

Du besuchst diese Seite mit einem veralteten IPv4-Internetzugang. Möglicherweise treten in Zukunft Probleme mit der Erreichbarkeit und Performance auf. Bitte frage deinen Internetanbieter oder Netzwerkadministrator nach IPv6-Unterstützung.
You are visiting this site with an outdated IPv4 internet access. You may experience problems with accessibility and performance in the future. Please ask your ISP or network administrator for IPv6 support.
Weitere Infos | More Information
Klicke zum schließen | Click to close