added on disk cache for the timetables
continuous-integration/drone/push Build is passing Details

* update spring-boot 2.1.9 -> 2.1.10
This commit is contained in:
Jannik 2019-11-14 18:43:00 +01:00
parent dd064d63af
commit 697f5e3167
Signed by: Seil0
GPG Key ID: E8459F3723C52C24
5 changed files with 89 additions and 64 deletions

View File

@ -1,7 +1,7 @@
buildscript {
ext.kotlin_version = '1.3.50'
ext.spring_boot_version = '2.1.9.RELEASE'
ext.spring_boot_version = '2.1.10.RELEASE'
repositories {
jcenter()
@ -34,8 +34,9 @@ repositories {
dependencies {
implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.2"
implementation 'org.jsoup:jsoup:1.12.1'
implementation 'org.springframework.boot:spring-boot-starter-web'
implementation "org.jsoup:jsoup:1.12.1"
implementation "org.springframework.boot:spring-boot-starter-web"
implementation "com.google.code.gson:gson:2.8.6"
testImplementation("org.junit.jupiter:junit-jupiter:5.5.1")
}

View File

@ -56,9 +56,9 @@ data class TimetableDay(val timeslots: Array<ArrayList<Lesson>> = Array(6) { Arr
data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() })
data class TimetableCourseMeta(var updateTime: Long, val courseName: String, val weekIndex: Int, val weekNumberYear: Int, val link: String)
data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, val weekNumberYear: Int = 0, val link: String = "")
data class TimetableCourseWeek(val meta: TimetableCourseMeta, var timetable: TimetableWeek)
data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek())
// data classes for the status part

View File

@ -22,6 +22,7 @@
package org.mosad.thecitadelofricks.controller
import com.google.gson.Gson
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.async
import kotlinx.coroutines.launch
@ -32,6 +33,9 @@ import org.mosad.thecitadelofricks.hsoparser.MensaParser
import org.mosad.thecitadelofricks.hsoparser.TimetableParser
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.io.BufferedWriter
import java.io.File
import java.io.FileWriter
import java.util.*
import java.util.concurrent.Executors
import kotlin.collections.ArrayList
@ -59,7 +63,7 @@ class CacheController {
* get a timetable, since they may not cached, we need to make sure it's cached, otherwise download
* @param courseName the name of the course to be requested
* @param weekIndex request week number (current week = 0)
* @return the timetable of a course (courseName)
* @return timetable of the course (Type: [TimetableCourseWeek])
*/
fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking {
val currentTime = System.currentTimeMillis() / 1000
@ -81,17 +85,11 @@ class CacheController {
jobTimetable.await()
timetableList.add(
TimetableCourseWeek(
TimetableCourseMeta(
currentTime,
courseName,
weekIndex,
weekNumberYear,
timetableLink
),
TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink),
timetable
)
)
logger.info("added new timetable for $courseName, week $weekIndex")
}
}
@ -122,7 +120,7 @@ class CacheController {
* @param courseName the name of the course to be requested
* @param lessonSubject the lesson subject to be requested
* @param weekIndex request week number (current week = 0)
* @return a ArrayList of every lesson with lessonSubject for one week
* @return a ArrayList<[Lesson]> of every lesson with lessonSubject for one week
*/
fun getLesson(courseName: String, lessonSubject: String, weekIndex: Int): ArrayList<Lesson> {
val lessonList = ArrayList<Lesson>()
@ -130,14 +128,14 @@ class CacheController {
// get all lessons from the weeks timetable
val flatMap = getTimetable(courseName, weekIndex).timetable.days.flatMap { it.timeslots.asIterable() }
flatMap.forEach {
// TODO Java 11
//it.stream().filter { x -> x.lessonSubject.contains(lessonSubject) }.findAny().ifPresent { x -> println("${x.lessonSubject}, ${x.lessonTeacher}") }
it.forEach { lesson ->
if(lesson.lessonSubject.contains(lessonSubject)) {
lessonList.add(lesson)
}
}
// TODO Java 11
//it.stream().filter { x -> x.lessonSubject.contains(lessonSubject) }.findAny().ifPresent { x -> println("${x.lessonSubject}, ${x.lessonTeacher}") }
}
return lessonList
@ -149,10 +147,10 @@ class CacheController {
* during the update process the old data will be returned for a API request
*/
private fun asyncUpdateCourseList() = GlobalScope.launch {
CourseListParser().getCourseLinks(StartupController.getCourseListURL())?.let {
CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
courseList = CourseList(
CourseMeta(System.currentTimeMillis() / 1000, it.size), it
)
)
}
logger.info("updated courses successful at ${Date(courseList.meta.updateTime * 1000)}")
@ -163,13 +161,13 @@ class CacheController {
* during the update process the old data will be returned for a API request
*/
private fun asyncUpdateMensa() = GlobalScope.launch {
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.getMensaMenuURL())
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.getMensaMenuURL()))
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.mensaMenuURL))
// only update if we get valid data
if (mensaCurrentWeek != null && mensaNextWeek != null) {
mensaMenu = MensaMenu(
MensaMeta(System.currentTimeMillis() / 1000, StartupController.getMensaName()), mensaCurrentWeek, mensaNextWeek
MensaMeta(System.currentTimeMillis() / 1000, StartupController.mensaName), mensaCurrentWeek, mensaNextWeek
)
}
@ -192,6 +190,8 @@ class CacheController {
executor.execute {
timetableCourse.timetable = TimetableParser().getTimeTable(timetableCourse.meta.link)
timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000
saveTimetableToCache(timetableCourse) // save the updated timetable to the cache directory
}
}
@ -202,6 +202,20 @@ class CacheController {
}
}
/**
* save a timetable to the cache directory
* this is only call on async updates, it is NOT call when first getting the timetable
* @param timetable a timetable of the type [TimetableCourseWeek]
*/
private fun saveTimetableToCache(timetable: TimetableCourseWeek) {
println(timetable.timetable.toString())
val file = File(StartupController.dirTcorCache, "timetable-${timetable.meta.courseName}-${timetable.meta.weekIndex}.json")
val writer = BufferedWriter(FileWriter(file))
writer.write(Gson().toJson(timetable))
writer.close()
}
/**
* before the APIController is up, get the data fist
* runBlocking: otherwise the api would return no data to requests for a few seconds after startup
@ -209,7 +223,7 @@ class CacheController {
private fun initUpdates() = runBlocking {
// get all courses on startup
val jobCourseUpdate = GlobalScope.async {
CourseListParser().getCourseLinks(StartupController.getCourseListURL())?.let {
CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
courseList = CourseList(
CourseMeta(System.currentTimeMillis() / 1000, it.size), it
)
@ -218,13 +232,13 @@ class CacheController {
// get the current and next weeks mensa menus
val jobMensa = GlobalScope.async{
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.getMensaMenuURL())
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.getMensaMenuURL()))
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.mensaMenuURL))
// only update if we get valid data
if (mensaCurrentWeek != null && mensaNextWeek != null) {
mensaMenu = MensaMenu(
MensaMeta(System.currentTimeMillis() / 1000, StartupController.getMensaName()), mensaCurrentWeek, mensaNextWeek
MensaMeta(System.currentTimeMillis() / 1000, StartupController.mensaName), mensaCurrentWeek, mensaNextWeek
)
}
}
@ -236,8 +250,8 @@ class CacheController {
}
/**
* update the CourseList every 24h, the Timetables every 3h and the Mensa Menu every hour
* doesn't account the change between winter and summer time!
*
*/
private fun scheduledUpdates() {
val currentTime = System.currentTimeMillis()
@ -261,7 +275,7 @@ class CacheController {
}
// post to status.mosad.xyz every hour, if an API key is present
if (StartupController.getCachetAPIKey() != "0") {
if (StartupController.cachetAPIKey != "0") {
Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
CachetAPIController.postTotalRequests()
}

View File

@ -41,7 +41,7 @@ class CachetAPIController {
fun postTotalRequests() {
try {
val url = URL("${StartupController.getCachetBaseURL()}/api/v1/metrics/1/points")
val url = URL("${StartupController.cachetBaseURL}/api/v1/metrics/1/points")
val jsonInputString = "{\"value\": ${getTotalRequests() -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
oldTotalRequests = getTotalRequests()
@ -49,7 +49,7 @@ class CachetAPIController {
con.requestMethod = "POST"
con.setRequestProperty("Content-Type", "application/json; utf-8")
con.setRequestProperty("Accept", "application/json")
con.setRequestProperty("X-Cachet-Token", StartupController.getCachetAPIKey())
con.setRequestProperty("X-Cachet-Token", StartupController.cachetAPIKey)
con.doOutput = true
val os = con.outputStream

View File

@ -22,22 +22,31 @@
package org.mosad.thecitadelofricks.controller
import com.google.gson.Gson
import com.google.gson.JsonParser
import org.mosad.thecitadelofricks.TimetableCourseWeek
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.io.*
import java.util.*
class StartupController {
private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
private val userHome = System.getProperty("user.home")
private val tcorHome = "$userHome/.tcor"
private val dirTcorHome = File(tcorHome)
private val dirTcorCache = File("$tcorHome/cache")
private val fileConfig = File("$tcorHome/config.xml")
companion object {
val userHome: String = System.getProperty("user.home")
val tcorHome = "$userHome/.tcor"
val dirTcorHome = File(tcorHome)
val dirTcorCache = File("$tcorHome/cache")
val fileConfig = File("$tcorHome/config.xml")
var cachetAPIKey = "0"
var cachetBaseURL = "https://status.mosad.xyz"
var courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/"
var mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
var mensaName = "Offenburg"
}
init {
// if the tcor directory doesn't exist, create it
@ -56,8 +65,14 @@ class StartupController {
} else {
createConfig()
}
// TODO read cached timetable files, as they are not initially cached
readCachedTimetables()
}
/**
* load the config stored in the config.xml file
*/
private fun loadConfig() = try {
val properties = Properties()
properties.loadFromXML(FileInputStream(fileConfig))
@ -90,6 +105,9 @@ class StartupController {
logger.error("error while loading config", ex)
}
/**
* create an initial config file
*/
private fun createConfig() = try {
val properties = Properties()
@ -105,32 +123,24 @@ class StartupController {
logger.error("error while creating config", ex)
}
companion object {
private var cachetAPIKey = "0"
private var cachetBaseURL = "https://status.mosad.xyz"
private var courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/"
private var mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
private var mensaName = "Offenburg"
/**
* read all previously cached timetables
*/
private fun readCachedTimetables() {
dirTcorCache.walkTopDown().forEach {
if (it.isFile && it.name.endsWith(".json")) {
try {
val fileReader = FileReader(it)
val bufferedReader = BufferedReader(fileReader)
val timetableObject = JsonParser.parseString(bufferedReader.readLine()).asJsonObject
fun getCachetAPIKey(): String {
return cachetAPIKey
CacheController.timetableList.add(Gson().fromJson(timetableObject, TimetableCourseWeek().javaClass))
bufferedReader.close()
fileReader.close()
} catch (ex: Exception) {
logger.error("error while reading cache", ex)
}
}
}
fun getCachetBaseURL(): String {
return cachetBaseURL
}
fun getCourseListURL(): String {
return courseListURL
}
fun getMensaMenuURL(): String {
return mensaMenuURL
}
fun getMensaName(): String {
return mensaName
}
}
}