diff --git a/build.gradle b/build.gradle index 67c0d8b..d4d059e 100644 --- a/build.gradle +++ b/build.gradle @@ -38,4 +38,4 @@ compileTestKotlin { } group 'org.mosad' -version '1.1.1' +version '1.1.2' diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/APIController.kt b/src/main/kotlin/org/mosad/thecitadelofricks/APIController.kt index feb1391..d1967af 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/APIController.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/APIController.kt @@ -22,13 +22,8 @@ package org.mosad.thecitadelofricks -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.async -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import org.mosad.thecitadelofricks.hsoparser.CourseListParser -import org.mosad.thecitadelofricks.hsoparser.MensaParser -import org.mosad.thecitadelofricks.hsoparser.TimetableParser +import org.mosad.thecitadelofricks.CacheController.Companion.courseList +import org.mosad.thecitadelofricks.CacheController.Companion.mensaMenu import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.web.bind.annotation.RequestMapping @@ -40,74 +35,37 @@ import java.time.LocalDateTime import java.util.* import kotlin.collections.ArrayList import kotlin.collections.HashSet -import kotlin.concurrent.scheduleAtFixedRate @RestController class APIController { - // TODO clean up and move stuff to a CacheController - // Controller stuff - var logger: Logger = LoggerFactory.getLogger(APIController::class.java) - private var requestCount = 0 + private val logger: Logger = LoggerFactory.getLogger(APIController::class.java) + private val cache = CacheController() + + private val softwareVersion = "1.1.2" + private val apiVersion = "1.1.1" private val startTime = System.currentTimeMillis() / 1000 - private val softwareVersion = "1.1.1" - private val apiVersion = "1.1.0" - // hso parser links (hardcoded) - private val mensaLink = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/" - private val mensaName = "Offenburg" - - // cache objects - private var coursesLinkList = ArrayList() - private var coursesLastUpdate: Long = 0 - - private var timetableList = ArrayList() // this list contains all timetables - - private var mensaCurrentWeek = MensaWeek() - private var mensaNextWeek = MensaWeek() - private var mensaLastUpdate: Long = 0 - - init { - initUpdates() // without this 5-10 seconds after startup the response will be empty - - val currentTime = System.currentTimeMillis() - val delay24h = (86400000 - ((currentTime + 3600000) % 86400000)) + 60000 - val delay3h = (10800000 - ((currentTime + 3600000) % 10800000)) + 60000 - - // update courses every 24 hours (time in ms) - Timer().scheduleAtFixedRate(delay24h, 86400000) { - asyncUpdateCourses() - } - - // update courses every 3 hours (time in ms) - Timer().scheduleAtFixedRate(delay3h, 10800000) { - asyncUpdateMensa() - } - - // update all already existing timetables every 3 hours (time in ms) - Timer().scheduleAtFixedRate(delay3h, 10800000) { - asyncUpdateTimetables() - } - } + private var requestCount = 0 @Deprecated("courses is replaced by courseList", replaceWith = ReplaceWith("courseList()")) @RequestMapping("/courses") - fun courses(): CoursesList { + fun courses(): CourseList { return courseList() } @RequestMapping("/courseList") - fun courseList(): CoursesList { + fun courseList(): CourseList { logger.info("courseList request at ${LocalDateTime.now()}!") requestCount++ - return CoursesList(CoursesMeta(coursesLastUpdate, coursesLinkList.size), coursesLinkList) + return courseList } @RequestMapping("/mensamenu") - fun mensamenu(): Mensa { + fun mensamenu(): MensaMenu { logger.info("mensamenu request at ${LocalDateTime.now()}!") requestCount++ - return Mensa(MensaMeta(mensaLastUpdate, mensaName), mensaCurrentWeek, mensaNextWeek) + return mensaMenu } @RequestMapping("/timetable") @@ -117,7 +75,7 @@ class APIController { ): TimetableCourseWeek { logger.info("timetable request at ${LocalDateTime.now()}!") requestCount++ - return getTimetable(courseName, week) + return cache.getTimetable(courseName, week) } @RequestMapping("/lessonSubjectList") @@ -130,7 +88,7 @@ class APIController { requestCount++ // get every lesson subject for the given week - val flatMap = getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() } + val flatMap = cache.getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() } flatMap.forEach { it.stream().filter { x -> x.lessonSubject.isNotEmpty() }.findAny().ifPresent { x -> lessonSubjectList.add(x.lessonSubject) } } @@ -147,7 +105,7 @@ class APIController { val lessonList = ArrayList() // get all lessons from the weeks timetable - val flatMap = getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() } + val flatMap = cache.getTimetable(courseName, week).timetable.days.flatMap { it.timeslots.asIterable() } flatMap.forEach { it.forEach { lesson -> if(lesson.lessonSubject.contains(lessonSubject)) { @@ -194,105 +152,11 @@ class APIController { apiVersion, softwareVersion, requestCount, - Date(coursesLastUpdate * 1000), - Date(mensaLastUpdate * 1000), + Date(courseList.meta.updateTime * 1000), + Date(mensaMenu.meta.updateTime * 1000), hsoCode, swfrCode ) } - /** - * this function updates the courses list - * during the update process the old data will be returned for a API request - */ - private fun asyncUpdateCourses() = GlobalScope.launch { - coursesLinkList = CourseListParser().getCourseLinks() - coursesLastUpdate = System.currentTimeMillis() / 1000 - - logger.info("updated courses successful at ${Date(coursesLastUpdate * 1000)}") - } - - /** - * this function updates the mensa menu list - * during the update process the old data will be returned for a API request - */ - private fun asyncUpdateMensa() = GlobalScope.launch { - mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink) - mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaLink)) - mensaLastUpdate = System.currentTimeMillis() / 1000 - - logger.info("updated mensamenu successful at ${Date(mensaLastUpdate * 1000)}") - } - - /** - * this function updates all existing timetables - * during the update process the old data will be returned for a API request - */ - private fun asyncUpdateTimetables() = GlobalScope.launch { - timetableList.forEach { timetableCourse -> - val updateURL = timetableCourse.meta.link - timetableCourse.timetable = TimetableParser().getTimeTable(updateURL) - timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000 - } - logger.info("updated ${timetableList.size} timetables successful!") - } - - /** - * this function checks if we need to update the timetable for a given course and if so does it - * during the update process the old data will be returned for a API request - * update if the last update was 6 hours ago - */ - private fun checkTimetableCourse(courseName: String, week: Int) = runBlocking { - val currentTime = System.currentTimeMillis() / 1000 - var timetable = TimetableWeek() - // check if the timetable already exists and is up to date - val result = timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null) - - when (result) { - // there is no such course yet, create one - null -> { - val courseLink = coursesLinkList.stream().filter { x -> x.courseName == courseName }.findFirst().orElse(null).courseLink - val timetableMeta = TimetableCourseMeta(currentTime, courseName, week, courseLink.replace("week=0","week=$week")) - - val jobTimetable = GlobalScope.async { - timetable = TimetableParser().getTimeTable(timetableMeta.link) - } - - jobTimetable.await() - - timetableList.add(TimetableCourseWeek(timetableMeta, timetable)) - logger.info("added new timetable for $courseName, week $week") - } - } - } - - private fun getTimetable(courseName: String, week: Int): TimetableCourseWeek { - checkTimetableCourse(courseName, week) // check if we need to update and perform the update if so - return timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null) - } - - private fun initUpdates() = runBlocking { - // get all courses on startup - val jobCourseUpdate = GlobalScope.async{ - coursesLinkList = CourseListParser().getCourseLinks() - coursesLastUpdate = System.currentTimeMillis() / 1000 - } - - // get the current and next weeks mensa menus - val jobCurrentMensa = GlobalScope.async{ - mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink) - } - - val jobNextMensa = GlobalScope.async{ - mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink) - mensaLastUpdate = System.currentTimeMillis() / 1000 - } - - jobCourseUpdate.await() - jobCurrentMensa.await() - jobNextMensa.await() - - logger.info("init updates successful") - } - } \ No newline at end of file diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/CacheController.kt b/src/main/kotlin/org/mosad/thecitadelofricks/CacheController.kt new file mode 100644 index 0000000..d49f4ea --- /dev/null +++ b/src/main/kotlin/org/mosad/thecitadelofricks/CacheController.kt @@ -0,0 +1,180 @@ +/** + * TheCitadelofRicks + * + * Copyright 2019 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + * + */ + +package org.mosad.thecitadelofricks + +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.async +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import org.mosad.thecitadelofricks.hsoparser.CourseListParser +import org.mosad.thecitadelofricks.hsoparser.MensaParser +import org.mosad.thecitadelofricks.hsoparser.TimetableParser +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.* +import kotlin.collections.ArrayList +import kotlin.concurrent.scheduleAtFixedRate + +class CacheController { + + private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java) + + // hso parser links (hardcoded) + private val courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/" + private val mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/" + private val mensaName = "Offenburg" + + // cache objects + companion object{ + + lateinit var courseList: CourseList + lateinit var mensaMenu: MensaMenu + + var timetableList = ArrayList() // this list contains all timetables + } + + init { + initUpdates() + scheduledUpdates() + } + + fun getTimetable(courseName: String, week: Int): TimetableCourseWeek = runBlocking { + val currentTime = System.currentTimeMillis() / 1000 + var timetable = TimetableWeek() + // check if the timetable already exists and is up to date + + when (timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null)) { + // there is no such course yet, create one + null -> { + val courseLink = courseList.courses.stream().filter { x -> x.courseName == courseName }.findFirst().orElse(null).courseLink + val timetableMeta = TimetableCourseMeta(currentTime, courseName, week, courseLink.replace("week=0","week=$week")) + + val jobTimetable = GlobalScope.async { + timetable = TimetableParser().getTimeTable(timetableMeta.link) + } + + jobTimetable.await() + + timetableList.add(TimetableCourseWeek(timetableMeta, timetable)) + logger.info("added new timetable for $courseName, week $week") + } + } + + return@runBlocking timetableList.stream().filter { x -> x.meta.courseName == courseName && x.meta.week == week }.findAny().orElse(null) + } + + /** + * this function updates the courses list + * during the update process the old data will be returned for a API request + */ + private fun asyncUpdateCourses() = GlobalScope.launch { + val result = CourseListParser().getCourseLinks(courseListURL) + if (result != null) { + courseList = CourseList(CourseMeta(System.currentTimeMillis() / 1000, result.size), result) + } + + logger.info("updated courses successful at ${Date(courseList.meta.updateTime * 1000)}") + } + + /** + * this function updates the mensa menu list + * during the update process the old data will be returned for a API request + */ + private fun asyncUpdateMensa() = GlobalScope.launch { + val mensaCurrentWeek = MensaParser().getMensaMenu(mensaMenuURL) + val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaMenuURL)) + + // only update if we get valid data + if (mensaCurrentWeek != null && mensaNextWeek != null) { + mensaMenu = MensaMenu(MensaMeta(System.currentTimeMillis() / 1000, mensaName), mensaCurrentWeek, mensaNextWeek) + } + + logger.info("updated mensamenu successful at ${Date(mensaMenu.meta.updateTime * 1000)}") + } + + /** + * this function updates all existing timetables + * during the update process the old data will be returned for a API request + */ + private fun asyncUpdateTimetables() = GlobalScope.launch { + timetableList.forEach { timetableCourse -> + val updateURL = timetableCourse.meta.link + timetableCourse.timetable = TimetableParser().getTimeTable(updateURL) + timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000 + } + logger.info("updated ${timetableList.size} timetables successful!") + } + + /** + * before the APIController is up, get the data fist + * runBlocking: otherwise the api would return no data to requests for a few seconds after startup + */ + private fun initUpdates() = runBlocking { + // get all courses on startup + val jobCourseUpdate = GlobalScope.async { + val result = CourseListParser().getCourseLinks(courseListURL) + if (result != null) { + courseList = CourseList(CourseMeta(System.currentTimeMillis() / 1000, result.size), result) + } + } + + // get the current and next weeks mensa menus + val jobMensa = GlobalScope.async{ + val mensaCurrentWeek = MensaParser().getMensaMenu(mensaMenuURL) + val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaMenuURL)) + + // only update if we get valid data + if (mensaCurrentWeek != null && mensaNextWeek != null) { + mensaMenu = MensaMenu(MensaMeta(System.currentTimeMillis() / 1000, mensaName), mensaCurrentWeek, mensaNextWeek) + } + } + + jobCourseUpdate.await() + jobMensa.await() + + logger.info("init updates successful") + } + + // TODO rework! + private fun scheduledUpdates() { + val currentTime = System.currentTimeMillis() + val delay24h = (86400000 - ((currentTime + 3600000) % 86400000)) + 60000 + val delay3h = (10800000 - ((currentTime + 3600000) % 10800000)) + 60000 + + // update courses every 24 hours (time in ms) + Timer().scheduleAtFixedRate(delay24h, 86400000) { + asyncUpdateCourses() + } + + // update courses every 3 hours (time in ms) + Timer().scheduleAtFixedRate(delay3h, 10800000) { + asyncUpdateMensa() + } + + // update all already existing timetables every 3 hours (time in ms) + Timer().scheduleAtFixedRate(delay3h, 10800000) { + asyncUpdateTimetables() + } + } + +} \ No newline at end of file diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt b/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt index 679dd7c..56a11d1 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt @@ -28,9 +28,9 @@ import java.util.* // data classes for the course part data class Course(val courseName: String, val courseLink: String) -data class CoursesMeta(val updateTime: Long, val totalCourses: Int) +data class CourseMeta(val updateTime: Long, val totalCourses: Int) -data class CoursesList(val meta: CoursesMeta, val courses: ArrayList) +data class CourseList(val meta: CourseMeta, val courses: ArrayList) // data classes for the Mensa part data class Meal(val day: String, val heading: String, val parts: ArrayList, val additives: String) @@ -41,7 +41,8 @@ data class MensaWeek(val days: Array = Array(7) { Meals(ArrayList()) }) data class MensaMeta(val updateTime: Long, val mensaName: String) -data class Mensa(val meta: MensaMeta, val currentWeek: MensaWeek, val nextWeek: MensaWeek) +data class MensaMenu(val meta: MensaMeta, val currentWeek: MensaWeek, val nextWeek: MensaWeek) { +} // data classes for the timetable part data class Lesson( diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/CourseListParser.kt b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/CourseListParser.kt index 3436d26..29d6635 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/CourseListParser.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/CourseListParser.kt @@ -32,13 +32,14 @@ class CourseListParser { var logger: org.slf4j.Logger = LoggerFactory.getLogger(MensaParser::class.java) /** - * return a list of all courses at hs-offenburg.de/studium/vorlesungsplaene/ - * @return a ArrayList with all courses + * return a list of all courses at courseListURL + * @param courseListURL the url to the course list page + * @return a ArrayList with all courses or null if the request was not successful */ - fun getCourseLinks(): ArrayList { + fun getCourseLinks(courseListURL: String): ArrayList? { val courseLinkList = ArrayList() try { - val courseHTML = Jsoup.connect("https://www.hs-offenburg.de/studium/vorlesungsplaene/").get() + val courseHTML = Jsoup.connect(courseListURL).get() courseHTML.select("ul.index-group").select("li.Class").select("a[href]").forEachIndexed { _, element -> courseLinkList.add( @@ -50,8 +51,10 @@ class CourseListParser { } } catch (ex: SocketTimeoutException) { logger.warn("timeout from hs-offenburg.de, updating on next attempt!") + return null } catch (gex: Exception) { logger.error("general CourseListParser error", gex) + return null } return courseLinkList diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/MensaParser.kt b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/MensaParser.kt index cb39fe2..5f8bed6 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/MensaParser.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/MensaParser.kt @@ -34,12 +34,13 @@ class MensaParser { /** * returns the mensa menu for a week - * @param menuLink the url to a mensa menu (swfr) + * @param mensaMenuURL the url to a mensa menu (swfr) + * @return the menu plan found at menuURL or null if the request was not successful */ - fun getMensaMenu(menuLink: String): MensaWeek { + fun getMensaMenu(mensaMenuURL: String): MensaWeek? { val mealWeekList = MensaWeek() try { - val menuHTML = Jsoup.connect(menuLink).timeout(15000).get() + val menuHTML = Jsoup.connect(mensaMenuURL).timeout(15000).get() menuHTML.select("#speiseplan-tabs").select("div.tab-content").select("div.menu-tagesplan") .forEachIndexed { dayIndex, day -> @@ -56,9 +57,11 @@ class MensaParser { } } catch (ex: SocketTimeoutException) { - logger.warn("timeout from $menuLink, updating on next attempt!") + logger.warn("timeout from $mensaMenuURL, updating on next attempt!") + return null } catch (gex: Exception) { logger.error("general MensaParser error", gex) + return null } return mealWeekList @@ -66,10 +69,10 @@ class MensaParser { /** * return the link of the next weeks menus - * @param menuLink the current weeks menus link + * @param mensaMenuURL the current weeks menus link */ - fun getMenuLinkNextWeek(menuLink: String): String { - val menuHTML = Jsoup.connect(menuLink).get() + fun getMenuLinkNextWeek(mensaMenuURL: String): String { + val menuHTML = Jsoup.connect(mensaMenuURL).get() return "https://www.swfr.de" + menuHTML.select("#speiseplan-tabs").select("a.next-week").attr("href") } diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt index 9134cec..47cc3e2 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt @@ -37,7 +37,7 @@ class TimetableParser { */ fun getTimeTable(timetableURL: String): TimetableWeek { val timetableWeek = TimetableWeek() - val scheduleHTML = Jsoup.connect(timetableURL).get() // TODO add a tyr catch block to cover timeouts + val scheduleHTML = Jsoup.connect(timetableURL).get() // TODO add a try catch block to cover timeouts //val week = scheduleHTML.select("h1.timetable-caption").text() //println("$week successful!\n")