Various improvements #17

Merged
Seil0 merged 11 commits from :fix/timeout-resilience into master 2021-10-24 14:38:13 +02:00
9 changed files with 139 additions and 111 deletions

View File

@ -1,25 +1,25 @@
plugins {
id 'org.jetbrains.kotlin.jvm' version '1.4.10'
id 'org.jetbrains.kotlin.plugin.spring' version '1.4.10'
id 'org.springframework.boot' version '2.3.3.RELEASE'
id 'io.spring.dependency-management' version '1.0.9.RELEASE'
id 'org.jetbrains.kotlin.jvm' version '1.5.31'
id 'org.jetbrains.kotlin.plugin.spring' version '1.5.31'
id 'org.springframework.boot' version '2.5.5'
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
}
group 'org.mosad'
version '1.2.8'
repositories {
jcenter()
mavenCentral()
}
dependencies {
implementation 'org.springframework.boot:spring-boot-starter-web'
implementation 'org.jetbrains.kotlin:kotlin-stdlib'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.8'
implementation 'org.jsoup:jsoup:1.13.1'
implementation 'com.google.code.gson:gson:2.8.6'
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2'
implementation 'org.jsoup:jsoup:1.14.3'
implementation 'com.google.code.gson:gson:2.8.8'
testImplementation 'org.junit.jupiter:junit-jupiter:5.6.2'
testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1'
}
test {
@ -30,9 +30,17 @@ test {
}
}
def jvmTargetVersion = "11"
compileKotlin {
kotlinOptions.jvmTarget = "11"
kotlinOptions.jvmTarget = jvmTargetVersion
kotlinOptions.freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn"
}
compileJava {
targetCompatibility = jvmTargetVersion
}
compileTestKotlin {
kotlinOptions.jvmTarget = "11"
kotlinOptions.jvmTarget = jvmTargetVersion
}
compileTestJava {
targetCompatibility = jvmTargetVersion
}

View File

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

View File

@ -58,7 +58,7 @@ data class TimetableDay(val timeslots: Array<ArrayList<Lesson>> = Array(6) { Arr
data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() })
data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, val weekNumberYear: Int = 0, val link: String = "")
data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, var weekNumberYear: Int = 0, val link: String = "")
data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek())

View File

@ -33,11 +33,13 @@ import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.io.*
import java.util.*
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.Executors
import kotlin.collections.ArrayList
import kotlin.collections.HashMap
import kotlin.collections.HashSet
import kotlin.concurrent.scheduleAtFixedRate
import kotlin.time.Duration
import kotlin.time.ExperimentalTime
class CacheController {
@ -46,12 +48,12 @@ class CacheController {
scheduledUpdates()
}
companion object{
companion object {
private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
var courseList = CoursesList(CoursesMeta(), sortedMapOf())
var mensaMenu = MensaMenu(MensaMeta(0,""), MensaWeek(), MensaWeek())
var timetableList = HashMap<String, TimetableCourseWeek>() // this list contains all timetables
var mensaMenu = MensaMenu(MensaMeta(0, ""), MensaWeek(), MensaWeek())
var timetableList = ConcurrentHashMap<String, TimetableCourseWeek>() // this list contains all timetables
/**
* get a timetable, since they may not be cached, we need to make sure it's cached, otherwise download
@ -59,36 +61,52 @@ class CacheController {
* @param weekIndex request week number (current week = 0)
* @return timetable of the course (Type: [TimetableCourseWeek])
*/
fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking {
fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek {
// TODO just for testing
if (courseName == "TEST_A" || courseName == "TEST_B") {
val currentTime = System.currentTimeMillis() / 1000
val timetableLink = "https://mosad.xyz"
val weekNumberYear = 0
val instr = javaClass.getResourceAsStream("/html/Timetable_normal-week.html")
val instr = CacheController::class.java.getResourceAsStream("/html/Timetable_normal-week.html")
val htmlDoc = Jsoup.parse(instr,"UTF-8", "https://www.hs-offenburg.de/")
val timetableTest = TimetableParser().parseTimeTable(htmlDoc)
val timetableParser =
TimetableParser(htmlDoc = Jsoup.parse(instr, "UTF-8", "https://www.hs-offenburg.de/"))
val timetableTest = timetableParser.parseTimeTable()
return@runBlocking TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetableTest)
return TimetableCourseWeek(
TimetableCourseMeta(
currentTime,
courseName,
weekIndex,
weekNumberYear,
timetableLink
), timetableTest ?: TimetableWeek()
)
}
return@runBlocking timetableList.getOrPut("$courseName-$weekIndex") {
val key = "$courseName-$weekIndex"
return if (timetableList.contains(key)) {
timetableList[key]!!
} else {
val timetableLink = courseList.courses[courseName]
?.courseLink
?.replace("week=0", "week=$weekIndex") ?: ""
val currentTime = System.currentTimeMillis() / 1000
var timetable = TimetableWeek()
var weekNumberYear = 0
val timetableJobs = listOf(
async { timetable = TimetableParser().getTimeTable(timetableLink) },
async { weekNumberYear = TimetableParser().getWeekNumberYear(timetableLink) }
)
val timetableParser = TimetableParser(timetableLink)
val weekNumberYear = timetableParser.parseWeekNumberYear()
val timetable = timetableParser.parseTimeTable()
timetableJobs.awaitAll()
TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetable)
TimetableCourseWeek(
TimetableCourseMeta(
currentTime,
courseName,
weekIndex,
weekNumberYear ?: 0,
timetableLink
), timetable ?: TimetableWeek()
).also { if (timetable != null) timetableList[key] = it }
}
}
@ -133,7 +151,7 @@ class CacheController {
/**
* this function updates the courseList
* during the update process the old data will be returned for a API request
* during the update process the old data will be returned for an API request
*/
private fun asyncUpdateCourseList() = GlobalScope.launch {
CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
@ -149,7 +167,7 @@ class CacheController {
/**
* this function updates the mensa menu list
* during the update process the old data will be returned for a API request
* during the update process the old data will be returned for an API request
*/
private fun asyncUpdateMensa() = GlobalScope.launch {
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
@ -165,7 +183,7 @@ class CacheController {
/**
* this function updates all existing timetables
* during the update process the old data will be returned for a API request
* during the update process the old data will be returned for an API request
* a FixedThreadPool is used to make parallel requests for faster updates
*/
private fun asyncUpdateTimetables() = GlobalScope.launch {
@ -177,7 +195,10 @@ class CacheController {
try {
timetableList.forEach { timetableCourse ->
executor.execute {
timetableCourse.value.timetable = TimetableParser().getTimeTable(timetableCourse.value.meta.link)
val timetableParser = TimetableParser(timetableCourse.value.meta.link)
timetableCourse.value.timetable = timetableParser.parseTimeTable() ?: return@execute
timetableCourse.value.meta.weekNumberYear =
timetableParser.parseWeekNumberYear() ?: return@execute
timetableCourse.value.meta.updateTime = System.currentTimeMillis() / 1000
saveTimetableToCache(timetableCourse.value) // save the updated timetable to the cache directory
@ -228,30 +249,39 @@ class CacheController {
* update the CourseList every 24h, the Timetables every 3h and the Mensa Menu every hour
* doesn't account the change between winter and summer time!
*/
@OptIn(ExperimentalTime::class)
private fun scheduledUpdates() {
val currentTime = System.currentTimeMillis()
val initDelay24h = (86400000 - ((currentTime + 3600000) % 86400000)) + 60000
val initDelay3h = (10800000 - ((currentTime + 3600000) % 10800000)) + 60000
val initDelay1h = (3600000 - ((currentTime + 3600000) % 3600000)) + 60000
val duration24h = Duration.hours(24).inWholeMilliseconds
val duration3h = Duration.hours(3).inWholeMilliseconds
val duration1h = Duration.hours(1).inWholeMilliseconds
val duration1m = Duration.minutes(1).inWholeMilliseconds
// Calculate the initial delay to make the update time independent of the start time
fun calcInitDelay(period: Long) = (period - ((currentTime + duration1h) % period)) + duration1m
val initDelay24h = calcInitDelay(duration24h)
val initDelay3h = calcInitDelay(duration3h)
val initDelay1h = calcInitDelay(duration1h)
// update courseList every 24 hours (time in ms)
Timer().scheduleAtFixedRate(initDelay24h, 86400000) {
Timer().scheduleAtFixedRate(initDelay24h, duration24h) {
asyncUpdateCourseList()
}
// update all already existing timetables every 3 hours (time in ms)
Timer().scheduleAtFixedRate(initDelay3h, 10800000) {
Timer().scheduleAtFixedRate(initDelay3h, duration3h) {
asyncUpdateTimetables()
}
// update courses every hour (time in ms)
Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
Timer().scheduleAtFixedRate(initDelay1h, duration1h) {
asyncUpdateMensa()
}
// post to status.mosad.xyz every hour, if an API key is present
if (StartupController.cachetAPIKey != "0") {
Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
Timer().scheduleAtFixedRate(initDelay1h, duration1h) {
CachetAPIController.postTotalRequests()
}
}

View File

@ -22,7 +22,7 @@
package org.mosad.thecitadelofricks.controller
import org.mosad.thecitadelofricks.controller.StatusController.Companion.getTotalRequests
import org.mosad.thecitadelofricks.controller.StatusController.Companion.totalRequests
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.io.BufferedReader
@ -42,8 +42,8 @@ class CachetAPIController {
fun postTotalRequests() {
try {
val url = URL("${StartupController.cachetBaseURL}/api/v1/metrics/1/points")
val jsonInputString = "{\"value\": ${getTotalRequests() -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
oldTotalRequests = getTotalRequests()
val jsonInputString = "{\"value\": ${totalRequests -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
oldTotalRequests = totalRequests
val con = url.openConnection() as HttpURLConnection
con.requestMethod = "POST"

View File

@ -135,7 +135,8 @@ class StartupController {
try {
val timetableObject = JsonParser.parseString(bufferedReader.readLine()).asJsonObject
val timetable = Gson().fromJson(timetableObject, TimetableCourseWeek().javaClass)
CacheController.timetableList.put("${timetable.meta.courseName}-${timetable.meta.weekIndex}", timetable)
CacheController.timetableList["${timetable.meta.courseName}-${timetable.meta.weekIndex}"] =
timetable
} catch (ex: Exception) {
logger.error("error while reading cache", ex)
} finally {

View File

@ -39,10 +39,14 @@ class StatusController {
companion object {
private val logger: Logger = LoggerFactory.getLogger(StatusController::class.java)
private var totalRequests = 0
private var mensaMenuRequests = 0
private var courseListRequests = 0
private var timetableRequests = HashMap<String, Int>()
var totalRequests = 0
private set
var mensaMenuRequests = 0
private set
var courseListRequests = 0
private set
var timetableRequests = HashMap<String, Int>()
private set
/**
* if a mensamenu/courseList/timetable is requested update the specific and total request count
@ -58,29 +62,10 @@ class StatusController {
}
fun updateTimetableRequests(courseName: String) {
timetableRequests[courseName] = timetableRequests.getOrPut(courseName) {0} + 1
timetableRequests[courseName] = (timetableRequests[courseName] ?: 0) + 1
totalRequests++
}
/**
* getters and setters
*/
fun getTotalRequests(): Int {
return totalRequests
}
fun getMensaMenuRequests(): Int {
return mensaMenuRequests
}
fun getCourseListRequests(): Int {
return courseListRequests
}
fun getTimetableRequests(): HashMap<String, Int> {
return timetableRequests
}
fun getStatus(): Status {
val currentTime = System.currentTimeMillis() / 1000
val minutes = (currentTime - startTime) % 3600 / 60
@ -112,10 +97,10 @@ class StatusController {
"$days days, $hours:$minutes",
apiVersion,
softwareVersion,
getTotalRequests(),
getMensaMenuRequests(),
getCourseListRequests(),
getTimetableRequests(),
totalRequests,
mensaMenuRequests,
courseListRequests,
timetableRequests,
CacheController.timetableList.size,
Date(CacheController.courseList.meta.updateTime * 1000),
Date(CacheController.mensaMenu.meta.updateTime * 1000),

View File

@ -22,34 +22,49 @@
package org.mosad.thecitadelofricks.hsoparser
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.Semaphore
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.mosad.thecitadelofricks.Lesson
import org.mosad.thecitadelofricks.TimetableWeek
import org.slf4j.LoggerFactory
class TimetableParser {
/**
* @param timetableURL the URL of the timetable you want to get
* @param htmlDoc the html document to use (the timetableURL will be ignored if this value is present)
*/
class TimetableParser(timetableURL: String? = null, htmlDoc: Document? = null) {
private var logger: org.slf4j.Logger = LoggerFactory.getLogger(TimetableParser::class.java)
private val days = arrayOf("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday")
/**
* get the timetable from the given url
* the timetable is organised per row not per column;
* Mon 1, Tue 1, Wed 1, Thur 1, Fri 1, Sat 1, Mon 2 and so on
* @param timetableURL the URL of the timetable you want to get
*/
fun getTimeTable(timetableURL: String): TimetableWeek {
return try {
parseTimeTable(Jsoup.connect(timetableURL).get())
companion object {
val semaphore = Semaphore(3, 0)
}
private val htmlDoc: Document? = htmlDoc ?: timetableURL?.let {
runBlocking {
try {
// Only allow sending a limited amount of requests at the same time
semaphore.acquire()
Jsoup.connect(timetableURL).get()
} catch (gex: Exception) {
logger.error("general TimetableParser error", gex)
TimetableWeek()
null
} finally {
semaphore.release()
}
}
}
fun parseTimeTable(htmlDoc: Document): TimetableWeek {
/**
* parse the timetable from the previously given url
* the timetable is organised per row not per column;
* Mon 1, Tue 1, Wed 1, Thur 1, Fri 1, Sat 1, Mon 2 and so on
*/
fun parseTimeTable(): TimetableWeek? = htmlDoc?.let {
val timetableWeek = TimetableWeek()
val rows = htmlDoc.select("table.timetable").select("tr[scope=\"row\"]")
val rows = it.select("table.timetable").select("tr[scope=\"row\"]")
var sDay = -1
var sRow = -1
@ -61,7 +76,7 @@ class TimetableParser {
var lessonIndexDay = 0 // the index of the lesson per timeslot
// elements are now all lessons, including empty ones
row.select("td.lastcol, td[style]").forEach {element ->
row.select("td.lastcol, td[style]").forEach { element ->
// if there is a lecture with rowspan="2", we need to shift everything by one to the left. This is stupid and ugly there needs to bee an API
if ((sDay > -1 && sRow > -1) && (sDay == day && ((sRow + 1) == rowIndex))) {
@ -104,8 +119,7 @@ class TimetableParser {
lessonIndexDay++
if (element.hasClass("lastcol"))
{
if (element.hasClass("lastcol")) {
day++
lessonIndexDay = 0
}
@ -117,20 +131,10 @@ class TimetableParser {
}
/**
* get the week number of the year for the timetable
* @param timetableURL the URL of the timetable you want to get
* parse the week number of the year for the timetable
*/
fun getWeekNumberYear(timetableURL: String): Int {
return try {
parseWeekNumberYear(Jsoup.connect(timetableURL).get())
} catch (gex: Exception) {
logger.error("general TimetableParser error", gex)
0
}
}
fun parseWeekNumberYear(htmlDoc: Document): Int {
return htmlDoc.select("h1.timetable-caption").text().substringAfter("- ")
fun parseWeekNumberYear(): Int? = htmlDoc?.let {
it.select("h1.timetable-caption").text().substringAfter("- ")
.substringBefore(".").replace(" ", "").toInt()
}

View File

@ -32,8 +32,8 @@ class TimetableParserTest {
@Test
fun parseTimetableNormalWeek() {
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path)
val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim()
val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim()
val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_normal-week.txt").readText().trim()
Assertions.assertEquals(expectedTimetable, actualTimetable)
@ -42,8 +42,8 @@ class TimetableParserTest {
@Test
fun parseTimetableEmptyWeek() {
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_empty-week.html").path)
val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim()
val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim()
val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_empty-week.txt").readText().trim()
Assertions.assertEquals(expectedTimetable, actualTimetable)
@ -52,8 +52,8 @@ class TimetableParserTest {
@Test
fun parseWeekNumberYear() {
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path)
val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
val actualWeekNumberYear = TimetableParser().parseWeekNumberYear(htmlDoc)
val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
val actualWeekNumberYear = TimetableParser(htmlDoc = htmlDoc).parseWeekNumberYear()
Assertions.assertEquals(42, actualWeekNumberYear)
}