update all existing courses every 3 hours & some clean up

This commit is contained in:
Jannik 2019-03-13 19:12:05 +01:00
parent 6de94be329
commit f2c1f2bc58
6 changed files with 89 additions and 47 deletions

View File

@ -39,4 +39,4 @@ compileTestKotlin {
} }
group 'org.mosad' group 'org.mosad'
version '1.0-SNAPSHOT' version '1.0.0'

View File

@ -22,28 +22,37 @@
package org.mosad.thecitadelofricks package org.mosad.thecitadelofricks
import kotlinx.coroutines.* import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.async
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import org.mosad.thecitadelofricks.hsoparser.CourseListParser import org.mosad.thecitadelofricks.hsoparser.CourseListParser
import org.mosad.thecitadelofricks.hsoparser.MensaParser import org.mosad.thecitadelofricks.hsoparser.MensaParser
import org.mosad.thecitadelofricks.hsoparser.TimetableParser import org.mosad.thecitadelofricks.hsoparser.TimetableParser
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RequestParam
import org.springframework.web.bind.annotation.RestController import org.springframework.web.bind.annotation.RestController
import java.lang.Exception import java.net.HttpURLConnection
import java.net.* import java.net.URL
import java.time.LocalDateTime
import java.util.* import java.util.*
import kotlin.concurrent.scheduleAtFixedRate import kotlin.concurrent.scheduleAtFixedRate
import java.time.LocalDateTime
@RestController @RestController
class APIController { class APIController {
//private val logger = LoggerFactory.getLogger(DemoApplication::class.java) // Controller stuff
var logger: Logger = LoggerFactory.getLogger(APIController::class.java)
private val startTime = System.currentTimeMillis() / 1000
private val apiVersion = "1.0.0"
// hso parser links (hardcoded)
private val mensaLink = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/" private val mensaLink = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
private val mensaName = "Offenburg" private val mensaName = "Offenburg"
// cache objects
private var coursesLinkList = ArrayList<Course>() private var coursesLinkList = ArrayList<Course>()
private var coursesLastUpdate: Long = 0 private var coursesLastUpdate: Long = 0
@ -62,36 +71,45 @@ class APIController {
asyncUpdateCourses() asyncUpdateCourses()
} }
// update courses every 6 hours (time in ms) // update courses every 3 hours (time in ms)
Timer().scheduleAtFixedRate(0, 21600000) { Timer().scheduleAtFixedRate(0, 10800000) {
asyncUpdateMensa() asyncUpdateMensa()
} }
// update all already existing timetables every 3 hours (time in ms)
Timer().scheduleAtFixedRate(0, 10800000) {
asyncUpdateTimetables()
}
} }
@RequestMapping("/courses") @RequestMapping("/courses")
fun courses(): CoursesList { fun courses(): CoursesList {
println("courses request at " + System.currentTimeMillis() / 1000 + "!") logger.info("courses request at ${LocalDateTime.now()}!")
return CoursesList(CoursesMeta(coursesLinkList.size, coursesLastUpdate), coursesLinkList) return CoursesList(CoursesMeta(coursesLinkList.size, coursesLastUpdate), coursesLinkList)
} }
@RequestMapping("/mensamenu") @RequestMapping("/mensamenu")
fun mensamenu(): Mensa { fun mensamenu(): Mensa {
println("mensamenu request at " + System.currentTimeMillis() / 1000 + "!") logger.info("mensamenu request at ${LocalDateTime.now()}!")
return Mensa(MensaMeta(mensaName, mensaLastUpdate), mensaCurrentWeek, mensaNextWeek) return Mensa(MensaMeta(mensaName, mensaLastUpdate), mensaCurrentWeek, mensaNextWeek)
} }
@RequestMapping("/timetable") @RequestMapping("/timetable")
fun timetable(@RequestParam(value = "courseName", defaultValue = "AI4") courseName: String): TimetableCourse { fun timetable(@RequestParam(value = "courseName", defaultValue = "AI4") courseName: String): TimetableCourse {
println("timetable request at " + System.currentTimeMillis() / 1000 + "!") logger.info("timetable request at ${LocalDateTime.now()}!")
updateTimetableCourse(courseName) // check if we need to update and perform the update if so checkTimetableCourse(courseName) // check if we need to update and perform the update if so
return timetableList.stream().filter { x -> x.meta.courseName == courseName }.findAny().orElse(null) return timetableList.stream().filter { x -> x.meta.courseName == courseName }.findAny().orElse(null)
} }
@RequestMapping("/status") @RequestMapping("/status")
fun status(): Status { fun status(): Status {
println("status request at " + System.currentTimeMillis() / 1000 + "!") val currentTime = System.currentTimeMillis() / 1000
val minutes = (currentTime - startTime) % 3600 / 60
val hours = (currentTime - startTime) % 86400 / 3600
val days = (currentTime - startTime) / 86400
var hsoCode = 999 var hsoCode = 999
var swfrCode = 999 var swfrCode = 999
logger.info("status request at ${LocalDateTime.now()}!")
try { try {
val hsoURL = URL("https://www.hs-offenburg.de/") val hsoURL = URL("https://www.hs-offenburg.de/")
@ -102,35 +120,55 @@ class APIController {
connection = swfrURL.openConnection() as HttpURLConnection connection = swfrURL.openConnection() as HttpURLConnection
swfrCode = connection.responseCode swfrCode = connection.responseCode
} catch (e: Exception) { } catch (e: Exception) {
// do some error handling logger.error("Error while fetching url response codes!", e)
} }
return Status(LocalDateTime.now(), Date(coursesLastUpdate * 1000), Date(mensaLastUpdate * 1000), hsoCode, swfrCode) return Status(
LocalDateTime.now(),
"$days days, $hours:$minutes",
apiVersion,
Date(coursesLastUpdate * 1000),
Date(mensaLastUpdate * 1000),
hsoCode,
swfrCode
)
} }
/** /**
* checks if we need to update the courses list and if so does it async * this function updates the courses list
* during the update process the old data will be returned for a API request * during the update process the old data will be returned for a API request
* update if the last update was 24 hours ago
*/ */
private fun asyncUpdateCourses() = GlobalScope.launch { private fun asyncUpdateCourses() = GlobalScope.launch {
coursesLinkList = CourseListParser().getCourseLinks() coursesLinkList = CourseListParser().getCourseLinks()
coursesLastUpdate = System.currentTimeMillis() / 1000 coursesLastUpdate = System.currentTimeMillis() / 1000
println("updated courses successful at $coursesLastUpdate") logger.info("updated courses successful at ${Date(coursesLastUpdate * 1000)}")
} }
/** /**
* this function checks if we need to update the mensa menu list and if so does it * this function updates the mensa menu list
* during the update process the old data will be returned for a API request * during the update process the old data will be returned for a API request
* update if the last update was 6 hours ago
*/ */
private fun asyncUpdateMensa() = GlobalScope.launch { private fun asyncUpdateMensa() = GlobalScope.launch {
mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink) mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink)
mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaLink)) mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaLink))
mensaLastUpdate = System.currentTimeMillis() / 1000 mensaLastUpdate = System.currentTimeMillis() / 1000
println("updated mensamenu successful at $mensaLastUpdate") logger.info("updated mensamenu successful at ${Date(mensaLastUpdate * 1000)}")
}
/**
* this function updates all existing timetables
* during the update process the old data will be returned for a API request
*/
private fun asyncUpdateTimetables() = GlobalScope.launch {
timetableList.forEach { timetableCourse ->
val updateURL = timetableCourse.meta.courseLink
timetableCourse.currentWeek = TimetableParser().getTimeTable(updateURL)
timetableCourse.nextWeek = TimetableParser().getTimeTable(updateURL.replace("week=0","week=1"))
timetableCourse.meta.time = System.currentTimeMillis() / 1000
}
logger.info("updated ${timetableList.size} timetables successful!")
} }
/** /**
@ -138,16 +176,16 @@ class APIController {
* during the update process the old data will be returned for a API request * during the update process the old data will be returned for a API request
* update if the last update was 6 hours ago * update if the last update was 6 hours ago
*/ */
private fun updateTimetableCourse(courseName: String) = runBlocking { private fun checkTimetableCourse(courseName: String) = runBlocking {
val currentTime = System.currentTimeMillis() / 1000 val currentTime = System.currentTimeMillis() / 1000
var currentWeek = TimetableWeek() var currentWeek = TimetableWeek()
var nextWeek = TimetableWeek() var nextWeek = TimetableWeek()
// check if the timetable already exists and is up to date // check if the timetable already exists and is up to date
val result = timetableList.stream().filter { x -> x.meta.courseName == courseName }.findAny().orElse(null) val result = timetableList.stream().filter { x -> x.meta.courseName == courseName }.findAny().orElse(null)
when { when (result) {
// there is no such course yet, create one // there is no such course yet, create one
result == null -> { null -> {
val courseLink = coursesLinkList.stream().filter { x -> x.courseName == courseName }.findFirst().orElse(null).courseLink val courseLink = coursesLinkList.stream().filter { x -> x.courseName == courseName }.findFirst().orElse(null).courseLink
val timetableMeta = TimetableMeta(courseName, courseLink, currentTime) val timetableMeta = TimetableMeta(courseName, courseLink, currentTime)
@ -163,21 +201,8 @@ class APIController {
jobNext.await() jobNext.await()
timetableList.add(TimetableCourse(timetableMeta, currentWeek, nextWeek)) timetableList.add(TimetableCourse(timetableMeta, currentWeek, nextWeek))
logger.info("added new timetable for $courseName")
} }
// update
(currentTime - result.meta.time) > 21600 -> {
val index = timetableList.indexOf(result)
println("update $courseName wit index: $index")
GlobalScope.async {
val courseLink = coursesLinkList.stream().filter { x -> x.courseName == courseName }.findFirst().orElse(null).courseLink
timetableList[index].currentWeek = TimetableParser().getTimeTable(courseLink)
timetableList[index].nextWeek = TimetableParser().getTimeTable(courseLink.replace("week=0","week=1"))
timetableList[index].meta.time = currentTime
}
}
else -> println("timetable for $courseName is up to date")
} }
} }
@ -201,6 +226,8 @@ class APIController {
jobCourseUpdate.await() jobCourseUpdate.await()
jobCurrentMensa.await() jobCurrentMensa.await()
jobNextMensa.await() jobNextMensa.await()
logger.info("init updates successful")
} }
} }

View File

@ -44,9 +44,15 @@ data class MensaMeta(val mensaName: String, val time: Long)
data class Mensa(val meta: MensaMeta, val currentWeek: MensaWeek, val nextWeek: MensaWeek) data class Mensa(val meta: MensaMeta, val currentWeek: MensaWeek, val nextWeek: MensaWeek)
// data classes for the timetable part // data classes for the timetable part
data class Lesson(val lessonID: String, val lessonSubject: String, val lessonTeacher: String, val lessonRoom:String, val lessonRemark: String) data class Lesson(
val lessonID: String,
val lessonSubject: String,
val lessonTeacher: String,
val lessonRoom: String,
val lessonRemark: String
)
data class TimetableDay( val timeslots: Array<ArrayList<Lesson>> = Array(6) { ArrayList<Lesson>()}) data class TimetableDay(val timeslots: Array<ArrayList<Lesson>> = Array(6) { ArrayList<Lesson>() })
data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() }) data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() })
@ -56,4 +62,12 @@ data class TimetableCourse(val meta: TimetableMeta, var currentWeek: TimetableWe
// data classes for the status part // data classes for the status part
data class Status(val time: LocalDateTime, val coursesLastUpdate: Date, val mensaLastUpdate: Date, val hsoResponseCode: Int, val swfrResponseCode: Int) data class Status(
val time: LocalDateTime,
val uptime: String,
val apiVersion: String,
val coursesLastUpdate: Date,
val mensaLastUpdate: Date,
val hsoResponseCode: Int,
val swfrResponseCode: Int
)

View File

@ -43,16 +43,13 @@ class MensaParser {
val heading = day.select("h4")[mealIndex].text() val heading = day.select("h4")[mealIndex].text()
val parts = ArrayList(meal.html().substringBefore("<br>\n").replace("\n", "").split("<br>")) val parts = ArrayList(meal.html().substringBefore("<br>\n").replace("\n", "").split("<br>"))
val additives = meal.select("span.show-with-allergenes").text() val additives = meal.select("span.show-with-allergenes").text()
parts.removeIf { x -> x.isEmpty()|| x.isBlank() } parts.removeIf { x -> x.isEmpty() || x.isBlank() }
mealWeekList.days[dayIndex].meals.add(Meal(strDay, heading, parts, additives)) mealWeekList.days[dayIndex].meals.add(Meal(strDay, heading, parts, additives))
} }
} }
// Mon to Sat (0 - 5)
//println(mealWeekList.days[4])
return mealWeekList return mealWeekList
} }

View File

@ -51,7 +51,7 @@ class TimetableParser {
var day = 0 var day = 0
// elements are now all lessons, including empty ones // elements are now all lessons, including empty ones
row.select("td.lastcol, td[style]").forEachIndexed {elementIndex, element -> row.select("td.lastcol, td[style]").forEachIndexed { elementIndex, element ->
// if there is a lecture with rowspan="2", we need to shift everything by one to the left. This is stupid and ugly there needs to bee an API // if there is a lecture with rowspan="2", we need to shift everything by one to the left. This is stupid and ugly there needs to bee an API
if ((sDay > -1 && sRow > -1) && (sDay == day && ((sRow + 1) == rowIndex))) { if ((sDay > -1 && sRow > -1) && (sDay == day && ((sRow + 1) == rowIndex))) {

View File

@ -0,0 +1,4 @@
# Specify the level for spring boot and hibernate's loggers
# Available levels are: TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF
logging.level.org.springframework.web = INFO
#logging.level.org.hibernate = ERROR