clean up APICOntroller & new CacheController

* added a CacheController to hold all cache variables and clean up the APIController
* more consistent naming of variables
* update mensaMenu and courseList only if the request was successful
pull/18/head
Jannik 4 years ago
parent 9efc7fe4b7
commit 754c8cb17b
Signed by: Seil0
GPG Key ID: E8459F3723C52C24

@ -38,4 +38,4 @@ compileTestKotlin {
}
group 'org.mosad'
version '1.1.1'
version '1.1.2'

@ -22,13 +22,8 @@
package org.mosad.thecitadelofricks
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.async
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import org.mosad.thecitadelofricks.hsoparser.CourseListParser
import org.mosad.thecitadelofricks.hsoparser.MensaParser
import org.mosad.thecitadelofricks.hsoparser.TimetableParser
import org.mosad.thecitadelofricks.CacheController.Companion.courseList
import org.mosad.thecitadelofricks.CacheController.Companion.mensaMenu
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.springframework.web.bind.annotation.RequestMapping
@ -40,74 +35,37 @@ import java.time.LocalDateTime
import java.util.*
import kotlin.collections.ArrayList
import kotlin.collections.HashSet
import kotlin.concurrent.scheduleAtFixedRate
@RestController
class APIController {
// TODO clean up and move stuff to a CacheController
// Controller stuff
var logger: Logger = LoggerFactory.getLogger(APIController::class.java)
private var requestCount = 0
private val startTime = System.currentTimeMillis() / 1000
private val softwareVersion = "1.1.1"
private val apiVersion = "1.1.0"
// hso parser links (hardcoded)
private val mensaLink = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
private val mensaName = "Offenburg"
// cache objects
private var coursesLinkList = ArrayList<Course>()
private var coursesLastUpdate: Long = 0
private var timetableList = ArrayList<TimetableCourseWeek>() // this list contains all timetables
private var mensaCurrentWeek = MensaWeek()
private var mensaNextWeek = MensaWeek()
private var mensaLastUpdate: Long = 0
private val logger: Logger = LoggerFactory.getLogger(APIController::class.java)
private val cache = CacheController()
init {
initUpdates() // without this 5-10 seconds after startup the response will be empty
val currentTime = System.currentTimeMillis()
val delay24h = (86400000 - ((currentTime + 3600000) % 86400000)) + 60000
val delay3h = (10800000 - ((currentTime + 3600000) % 10800000)) + 60000
// update courses every 24 hours (time in ms)
Timer().scheduleAtFixedRate(delay24h, 86400000) {
asyncUpdateCourses()
}
// update courses every 3 hours (time in ms)
Timer().scheduleAtFixedRate(delay3h, 10800000) {
asyncUpdateMensa()
}
private val softwareVersion = "1.1.2"
private val apiVersion = "1.1.1"
private val startTime = System.currentTimeMillis() / 1000
// update all already existing timetables every 3 hours (time in ms)
Timer().scheduleAtFixedRate(delay3h, 10800000) {
asyncUpdateTimetables()
}
}
private var requestCount = 0
@Deprecated("courses is replaced by courseList", replaceWith = ReplaceWith("courseList()"))
@RequestMapping("/courses")
fun courses(): CoursesList {
fun courses(): CourseList {
return courseList()
}
@RequestMapping("/courseList")
fun courseList(): CoursesList {
fun courseList(): CourseList {
logger.info("courseList request at ${LocalDateTime.now()}!")
requestCount++
return CoursesList(CoursesMeta(coursesLastUpdate, coursesLinkList.size), coursesLinkList)
return courseList
}
@RequestMapping("/mensamenu")
fun mensamenu(): Mensa {
fun mensamenu(): MensaMenu {
logger.info("mensamenu request at ${LocalDateTime.now()}!")
requestCount++
return Mensa(MensaMeta(mensaLastUpdate, mensaName), mensaCurrentWeek, mensaNextWeek)
return mensaMenu
}
@RequestMapping("/timetable")
@ -117,7 +75,7 @@ class APIController {
): TimetableCourseWeek {
logger.info("timetable request at ${LocalDateTime.now()}!")
requestCount++
return getTimetable(courseName, week)
return cache.getTimetable(courseName, week)
}
@RequestMapping("/lessonSubjectList")
@ -130,7 +88,7 @@ class APIController {
requestCount++
// get every lesson subject for the given week
val flatMap = getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() }
val flatMap = cache.getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() }
flatMap.forEach {
it.stream().filter { x -> x.lessonSubject.isNotEmpty() }.findAny().ifPresent { x -> lessonSubjectList.add(x.lessonSubject) }
}
@ -147,7 +105,7 @@ class APIController {
val lessonList = ArrayList<Lesson>()
// get all lessons from the weeks timetable
val flatMap = getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() }
val flatMap = cache.getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() }
flatMap.forEach {
it.forEach { lesson ->
if(lesson.lessonSubject.contains(lessonSubject)) {
@ -194,105 +152,11 @@ class APIController {
apiVersion,
softwareVersion,
requestCount,
Date(coursesLastUpdate * 1000),
Date(mensaLastUpdate * 1000),
Date(courseList.meta.updateTime * 1000),
Date(mensaMenu.meta.updateTime * 1000),
hsoCode,
swfrCode
)
}
/**
* this function updates the courses list
* during the update process the old data will be returned for a API request
*/
private fun asyncUpdateCourses() = GlobalScope.launch {
coursesLinkList = CourseListParser().getCourseLinks()
coursesLastUpdate = System.currentTimeMillis() / 1000
logger.info("updated courses successful at ${Date(coursesLastUpdate * 1000)}")
}
/**
* this function updates the mensa menu list
* during the update process the old data will be returned for a API request
*/
private fun asyncUpdateMensa() = GlobalScope.launch {
mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink)
mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaLink))
mensaLastUpdate = System.currentTimeMillis() / 1000
logger.info("updated mensamenu successful at ${Date(mensaLastUpdate * 1000)}")
}
/**
* this function updates all existing timetables
* during the update process the old data will be returned for a API request
*/
private fun asyncUpdateTimetables() = GlobalScope.launch {
timetableList.forEach { timetableCourse ->
val updateURL = timetableCourse.meta.link
timetableCourse.timetable = TimetableParser().getTimeTable(updateURL)
timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000
}
logger.info("updated ${timetableList.size} timetables successful!")
}
/**
* this function checks if we need to update the timetable for a given course and if so does it
* during the update process the old data will be returned for a API request
* update if the last update was 6 hours ago
*/
private fun checkTimetableCourse(courseName: String, week: Int) = runBlocking {
val currentTime = System.currentTimeMillis() / 1000
var timetable = TimetableWeek()
// check if the timetable already exists and is up to date
val result = timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null)
when (result) {
// there is no such course yet, create one
null -> {
val courseLink = coursesLinkList.stream().filter { x -> x.courseName == courseName }.findFirst().orElse(null).courseLink
val timetableMeta = TimetableCourseMeta(currentTime, courseName, week, courseLink.replace("week=0","week=$week"))
val jobTimetable = GlobalScope.async {
timetable = TimetableParser().getTimeTable(timetableMeta.link)
}
jobTimetable.await()
timetableList.add(TimetableCourseWeek(timetableMeta, timetable))
logger.info("added new timetable for $courseName, week $week")
}
}
}
private fun getTimetable(courseName: String, week: Int): TimetableCourseWeek {
checkTimetableCourse(courseName, week) // check if we need to update and perform the update if so
return timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null)
}
private fun initUpdates() = runBlocking {
// get all courses on startup
val jobCourseUpdate = GlobalScope.async{
coursesLinkList = CourseListParser().getCourseLinks()
coursesLastUpdate = System.currentTimeMillis() / 1000
}
// get the current and next weeks mensa menus
val jobCurrentMensa = GlobalScope.async{
mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink)
}
val jobNextMensa = GlobalScope.async{
mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink)
mensaLastUpdate = System.currentTimeMillis() / 1000
}
jobCourseUpdate.await()
jobCurrentMensa.await()
jobNextMensa.await()
logger.info("init updates successful")
}
}

@ -0,0 +1,180 @@
/**
* TheCitadelofRicks
*
* Copyright 2019 <seil0@mosad.xyz>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*
*/
package org.mosad.thecitadelofricks
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.async
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import org.mosad.thecitadelofricks.hsoparser.CourseListParser
import org.mosad.thecitadelofricks.hsoparser.MensaParser
import org.mosad.thecitadelofricks.hsoparser.TimetableParser
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.util.*
import kotlin.collections.ArrayList
import kotlin.concurrent.scheduleAtFixedRate
class CacheController {
private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
// hso parser links (hardcoded)
private val courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/"
private val mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
private val mensaName = "Offenburg"
// cache objects
companion object{
lateinit var courseList: CourseList
lateinit var mensaMenu: MensaMenu
var timetableList = ArrayList<TimetableCourseWeek>() // this list contains all timetables
}
init {
initUpdates()
scheduledUpdates()
}
fun getTimetable(courseName: String, week: Int): TimetableCourseWeek = runBlocking {
val currentTime = System.currentTimeMillis() / 1000
var timetable = TimetableWeek()
// check if the timetable already exists and is up to date
when (timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null)) {
// there is no such course yet, create one
null -> {
val courseLink = courseList.courses.stream().filter { x -> x.courseName == courseName }.findFirst().orElse(null).courseLink
val timetableMeta = TimetableCourseMeta(currentTime, courseName, week, courseLink.replace("week=0","week=$week"))
val jobTimetable = GlobalScope.async {
timetable = TimetableParser().getTimeTable(timetableMeta.link)
}
jobTimetable.await()
timetableList.add(TimetableCourseWeek(timetableMeta, timetable))
logger.info("added new timetable for $courseName, week $week")
}
}
return@runBlocking timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null)
}
/**
* this function updates the courses list
* during the update process the old data will be returned for a API request
*/
private fun asyncUpdateCourses() = GlobalScope.launch {
val result = CourseListParser().getCourseLinks(courseListURL)
if (result != null) {
courseList = CourseList(CourseMeta(System.currentTimeMillis() / 1000, result.size), result)
}
logger.info("updated courses successful at ${Date(courseList.meta.updateTime * 1000)}")
}
/**
* this function updates the mensa menu list
* during the update process the old data will be returned for a API request
*/
private fun asyncUpdateMensa() = GlobalScope.launch {
val mensaCurrentWeek = MensaParser().getMensaMenu(mensaMenuURL)
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaMenuURL))
// only update if we get valid data
if (mensaCurrentWeek != null && mensaNextWeek != null) {
mensaMenu = MensaMenu(MensaMeta(System.currentTimeMillis() / 1000, mensaName), mensaCurrentWeek, mensaNextWeek)
}
logger.info("updated mensamenu successful at ${Date(mensaMenu.meta.updateTime * 1000)}")
}
/**
* this function updates all existing timetables
* during the update process the old data will be returned for a API request
*/
private fun asyncUpdateTimetables() = GlobalScope.launch {
timetableList.forEach { timetableCourse ->
val updateURL = timetableCourse.meta.link
timetableCourse.timetable = TimetableParser().getTimeTable(updateURL)
timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000
}
logger.info("updated ${timetableList.size} timetables successful!")
}
/**
* before the APIController is up, get the data fist
* runBlocking: otherwise the api would return no data to requests for a few seconds after startup
*/
private fun initUpdates() = runBlocking {
// get all courses on startup
val jobCourseUpdate = GlobalScope.async {
val result = CourseListParser().getCourseLinks(courseListURL)
if (result != null) {
courseList = CourseList(CourseMeta(System.currentTimeMillis() / 1000, result.size), result)
}
}
// get the current and next weeks mensa menus
val jobMensa = GlobalScope.async{
val mensaCurrentWeek = MensaParser().getMensaMenu(mensaMenuURL)
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaMenuURL))
// only update if we get valid data
if (mensaCurrentWeek != null && mensaNextWeek != null) {
mensaMenu = MensaMenu(MensaMeta(System.currentTimeMillis() / 1000, mensaName), mensaCurrentWeek, mensaNextWeek)
}
}
jobCourseUpdate.await()
jobMensa.await()
logger.info("init updates successful")
}
// TODO rework!
private fun scheduledUpdates() {
val currentTime = System.currentTimeMillis()
val delay24h = (86400000 - ((currentTime + 3600000) % 86400000)) + 60000
val delay3h = (10800000 - ((currentTime + 3600000) % 10800000)) + 60000
// update courses every 24 hours (time in ms)
Timer().scheduleAtFixedRate(delay24h, 86400000) {
asyncUpdateCourses()
}
// update courses every 3 hours (time in ms)
Timer().scheduleAtFixedRate(delay3h, 10800000) {
asyncUpdateMensa()
}
// update all already existing timetables every 3 hours (time in ms)
Timer().scheduleAtFixedRate(delay3h, 10800000) {
asyncUpdateTimetables()
}
}
}

@ -28,9 +28,9 @@ import java.util.*
// data classes for the course part
data class Course(val courseName: String, val courseLink: String)
data class CoursesMeta(val updateTime: Long, val totalCourses: Int)
data class CourseMeta(val updateTime: Long, val totalCourses: Int)
data class CoursesList(val meta: CoursesMeta, val courses: ArrayList<Course>)
data class CourseList(val meta: CourseMeta, val courses: ArrayList<Course>)
// data classes for the Mensa part
data class Meal(val day: String, val heading: String, val parts: ArrayList<String>, val additives: String)
@ -41,7 +41,8 @@ data class MensaWeek(val days: Array<Meals> = Array(7) { Meals(ArrayList()) })
data class MensaMeta(val updateTime: Long, val mensaName: String)
data class Mensa(val meta: MensaMeta, val currentWeek: MensaWeek, val nextWeek: MensaWeek)
data class MensaMenu(val meta: MensaMeta, val currentWeek: MensaWeek, val nextWeek: MensaWeek) {
}
// data classes for the timetable part
data class Lesson(

@ -32,13 +32,14 @@ class CourseListParser {
var logger: org.slf4j.Logger = LoggerFactory.getLogger(MensaParser::class.java)
/**
* return a list of all courses at hs-offenburg.de/studium/vorlesungsplaene/
* @return a ArrayList<Course> with all courses
* return a list of all courses at courseListURL
* @param courseListURL the url to the course list page
* @return a ArrayList<Course> with all courses or null if the request was not successful
*/
fun getCourseLinks(): ArrayList<Course> {
fun getCourseLinks(courseListURL: String): ArrayList<Course>? {
val courseLinkList = ArrayList<Course>()
try {
val courseHTML = Jsoup.connect("https://www.hs-offenburg.de/studium/vorlesungsplaene/").get()
val courseHTML = Jsoup.connect(courseListURL).get()
courseHTML.select("ul.index-group").select("li.Class").select("a[href]").forEachIndexed { _, element ->
courseLinkList.add(
@ -50,8 +51,10 @@ class CourseListParser {
}
} catch (ex: SocketTimeoutException) {
logger.warn("timeout from hs-offenburg.de, updating on next attempt!")
return null
} catch (gex: Exception) {
logger.error("general CourseListParser error", gex)
return null
}
return courseLinkList

@ -34,12 +34,13 @@ class MensaParser {
/**
* returns the mensa menu for a week
* @param menuLink the url to a mensa menu (swfr)
* @param mensaMenuURL the url to a mensa menu (swfr)
* @return the menu plan found at menuURL or null if the request was not successful
*/
fun getMensaMenu(menuLink: String): MensaWeek {
fun getMensaMenu(mensaMenuURL: String): MensaWeek? {
val mealWeekList = MensaWeek()
try {
val menuHTML = Jsoup.connect(menuLink).timeout(15000).get()
val menuHTML = Jsoup.connect(mensaMenuURL).timeout(15000).get()
menuHTML.select("#speiseplan-tabs").select("div.tab-content").select("div.menu-tagesplan")
.forEachIndexed { dayIndex, day ->
@ -56,9 +57,11 @@ class MensaParser {
}
} catch (ex: SocketTimeoutException) {
logger.warn("timeout from $menuLink, updating on next attempt!")
logger.warn("timeout from $mensaMenuURL, updating on next attempt!")
return null
} catch (gex: Exception) {
logger.error("general MensaParser error", gex)
return null
}
return mealWeekList
@ -66,10 +69,10 @@ class MensaParser {
/**
* return the link of the next weeks menus
* @param menuLink the current weeks menus link
* @param mensaMenuURL the current weeks menus link
*/
fun getMenuLinkNextWeek(menuLink: String): String {
val menuHTML = Jsoup.connect(menuLink).get()
fun getMenuLinkNextWeek(mensaMenuURL: String): String {
val menuHTML = Jsoup.connect(mensaMenuURL).get()
return "https://www.swfr.de" + menuHTML.select("#speiseplan-tabs").select("a.next-week").attr("href")
}

@ -37,7 +37,7 @@ class TimetableParser {
*/
fun getTimeTable(timetableURL: String): TimetableWeek {
val timetableWeek = TimetableWeek()
val scheduleHTML = Jsoup.connect(timetableURL).get() // TODO add a tyr catch block to cover timeouts
val scheduleHTML = Jsoup.connect(timetableURL).get() // TODO add a try catch block to cover timeouts
//val week = scheduleHTML.select("h1.timetable-caption").text()
//println("$week successful!\n")

Loading…
Cancel
Save

Du besuchst diese Seite mit einem veralteten IPv4-Internetzugang. Möglicherweise treten in Zukunft Probleme mit der Erreichbarkeit und Performance auf. Bitte frage deinen Internetanbieter oder Netzwerkadministrator nach IPv6-Unterstützung.
You are visiting this site with an outdated IPv4 internet access. You may experience problems with accessibility and performance in the future. Please ask your ISP or network administrator for IPv6 support.
Weitere Infos | More Information
Klicke zum schließen | Click to close