Browse Source

added on disk cache for the timetables

* update spring-boot 2.1.9 -> 2.1.10
pull/18/head
Jannik 3 years ago
parent
commit
697f5e3167
Signed by: Seil0
GPG Key ID: E8459F3723C52C24
  1. 7
      build.gradle
  2. 4
      src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt
  3. 62
      src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt
  4. 4
      src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt
  5. 78
      src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt

7
build.gradle

@ -1,7 +1,7 @@
buildscript {
ext.kotlin_version = '1.3.50'
ext.spring_boot_version = '2.1.9.RELEASE'
ext.spring_boot_version = '2.1.10.RELEASE'
repositories {
jcenter()
@ -34,8 +34,9 @@ repositories {
dependencies {
implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.2"
implementation 'org.jsoup:jsoup:1.12.1'
implementation 'org.springframework.boot:spring-boot-starter-web'
implementation "org.jsoup:jsoup:1.12.1"
implementation "org.springframework.boot:spring-boot-starter-web"
implementation "com.google.code.gson:gson:2.8.6"
testImplementation("org.junit.jupiter:junit-jupiter:5.5.1")
}

4
src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt

@ -56,9 +56,9 @@ data class TimetableDay(val timeslots: Array<ArrayList<Lesson>> = Array(6) { Arr
data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() })
data class TimetableCourseMeta(var updateTime: Long, val courseName: String, val weekIndex: Int, val weekNumberYear: Int, val link: String)
data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, val weekNumberYear: Int = 0, val link: String = "")
data class TimetableCourseWeek(val meta: TimetableCourseMeta, var timetable: TimetableWeek)
data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek())
// data classes for the status part

62
src/main/kotlin/org/mosad/thecitadelofricks/controller/CacheController.kt

@ -22,6 +22,7 @@
package org.mosad.thecitadelofricks.controller
import com.google.gson.Gson
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.async
import kotlinx.coroutines.launch
@ -32,6 +33,9 @@ import org.mosad.thecitadelofricks.hsoparser.MensaParser
import org.mosad.thecitadelofricks.hsoparser.TimetableParser
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.io.BufferedWriter
import java.io.File
import java.io.FileWriter
import java.util.*
import java.util.concurrent.Executors
import kotlin.collections.ArrayList
@ -59,7 +63,7 @@ class CacheController {
* get a timetable, since they may not cached, we need to make sure it's cached, otherwise download
* @param courseName the name of the course to be requested
* @param weekIndex request week number (current week = 0)
* @return the timetable of a course (courseName)
* @return timetable of the course (Type: [TimetableCourseWeek])
*/
fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking {
val currentTime = System.currentTimeMillis() / 1000
@ -81,17 +85,11 @@ class CacheController {
jobTimetable.await()
timetableList.add(
TimetableCourseWeek(
TimetableCourseMeta(
currentTime,
courseName,
weekIndex,
weekNumberYear,
timetableLink
),
TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink),
timetable
)
)
logger.info("added new timetable for $courseName, week $weekIndex")
}
}
@ -122,7 +120,7 @@ class CacheController {
* @param courseName the name of the course to be requested
* @param lessonSubject the lesson subject to be requested
* @param weekIndex request week number (current week = 0)
* @return a ArrayList of every lesson with lessonSubject for one week
* @return a ArrayList<[Lesson]> of every lesson with lessonSubject for one week
*/
fun getLesson(courseName: String, lessonSubject: String, weekIndex: Int): ArrayList<Lesson> {
val lessonList = ArrayList<Lesson>()
@ -130,14 +128,14 @@ class CacheController {
// get all lessons from the weeks timetable
val flatMap = getTimetable(courseName, weekIndex).timetable.days.flatMap { it.timeslots.asIterable() }
flatMap.forEach {
// TODO Java 11
//it.stream().filter { x -> x.lessonSubject.contains(lessonSubject) }.findAny().ifPresent { x -> println("${x.lessonSubject}, ${x.lessonTeacher}") }
it.forEach { lesson ->
if(lesson.lessonSubject.contains(lessonSubject)) {
lessonList.add(lesson)
}
}
// TODO Java 11
//it.stream().filter { x -> x.lessonSubject.contains(lessonSubject) }.findAny().ifPresent { x -> println("${x.lessonSubject}, ${x.lessonTeacher}") }
}
return lessonList
@ -149,10 +147,10 @@ class CacheController {
* during the update process the old data will be returned for a API request
*/
private fun asyncUpdateCourseList() = GlobalScope.launch {
CourseListParser().getCourseLinks(StartupController.getCourseListURL())?.let {
CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
courseList = CourseList(
CourseMeta(System.currentTimeMillis() / 1000, it.size), it
)
)
}
logger.info("updated courses successful at ${Date(courseList.meta.updateTime * 1000)}")
@ -163,13 +161,13 @@ class CacheController {
* during the update process the old data will be returned for a API request
*/
private fun asyncUpdateMensa() = GlobalScope.launch {
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.getMensaMenuURL())
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.getMensaMenuURL()))
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.mensaMenuURL))
// only update if we get valid data
if (mensaCurrentWeek != null && mensaNextWeek != null) {
mensaMenu = MensaMenu(
MensaMeta(System.currentTimeMillis() / 1000, StartupController.getMensaName()), mensaCurrentWeek, mensaNextWeek
MensaMeta(System.currentTimeMillis() / 1000, StartupController.mensaName), mensaCurrentWeek, mensaNextWeek
)
}
@ -192,6 +190,8 @@ class CacheController {
executor.execute {
timetableCourse.timetable = TimetableParser().getTimeTable(timetableCourse.meta.link)
timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000
saveTimetableToCache(timetableCourse) // save the updated timetable to the cache directory
}
}
@ -202,6 +202,20 @@ class CacheController {
}
}
/**
* save a timetable to the cache directory
* this is only call on async updates, it is NOT call when first getting the timetable
* @param timetable a timetable of the type [TimetableCourseWeek]
*/
private fun saveTimetableToCache(timetable: TimetableCourseWeek) {
println(timetable.timetable.toString())
val file = File(StartupController.dirTcorCache, "timetable-${timetable.meta.courseName}-${timetable.meta.weekIndex}.json")
val writer = BufferedWriter(FileWriter(file))
writer.write(Gson().toJson(timetable))
writer.close()
}
/**
* before the APIController is up, get the data fist
* runBlocking: otherwise the api would return no data to requests for a few seconds after startup
@ -209,7 +223,7 @@ class CacheController {
private fun initUpdates() = runBlocking {
// get all courses on startup
val jobCourseUpdate = GlobalScope.async {
CourseListParser().getCourseLinks(StartupController.getCourseListURL())?.let {
CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
courseList = CourseList(
CourseMeta(System.currentTimeMillis() / 1000, it.size), it
)
@ -218,13 +232,13 @@ class CacheController {
// get the current and next weeks mensa menus
val jobMensa = GlobalScope.async{
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.getMensaMenuURL())
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.getMensaMenuURL()))
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.mensaMenuURL))
// only update if we get valid data
if (mensaCurrentWeek != null && mensaNextWeek != null) {
mensaMenu = MensaMenu(
MensaMeta(System.currentTimeMillis() / 1000, StartupController.getMensaName()), mensaCurrentWeek, mensaNextWeek
MensaMeta(System.currentTimeMillis() / 1000, StartupController.mensaName), mensaCurrentWeek, mensaNextWeek
)
}
}
@ -236,8 +250,8 @@ class CacheController {
}
/**
* update the CourseList every 24h, the Timetables every 3h and the Mensa Menu every hour
* doesn't account the change between winter and summer time!
*
*/
private fun scheduledUpdates() {
val currentTime = System.currentTimeMillis()
@ -261,7 +275,7 @@ class CacheController {
}
// post to status.mosad.xyz every hour, if an API key is present
if (StartupController.getCachetAPIKey() != "0") {
if (StartupController.cachetAPIKey != "0") {
Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
CachetAPIController.postTotalRequests()
}

4
src/main/kotlin/org/mosad/thecitadelofricks/controller/CachetAPIController.kt

@ -41,7 +41,7 @@ class CachetAPIController {
fun postTotalRequests() {
try {
val url = URL("${StartupController.getCachetBaseURL()}/api/v1/metrics/1/points")
val url = URL("${StartupController.cachetBaseURL}/api/v1/metrics/1/points")
val jsonInputString = "{\"value\": ${getTotalRequests() -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
oldTotalRequests = getTotalRequests()
@ -49,7 +49,7 @@ class CachetAPIController {
con.requestMethod = "POST"
con.setRequestProperty("Content-Type", "application/json; utf-8")
con.setRequestProperty("Accept", "application/json")
con.setRequestProperty("X-Cachet-Token", StartupController.getCachetAPIKey())
con.setRequestProperty("X-Cachet-Token", StartupController.cachetAPIKey)
con.doOutput = true
val os = con.outputStream

78
src/main/kotlin/org/mosad/thecitadelofricks/controller/StartupController.kt

@ -22,22 +22,31 @@
package org.mosad.thecitadelofricks.controller
import com.google.gson.Gson
import com.google.gson.JsonParser
import org.mosad.thecitadelofricks.TimetableCourseWeek
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.io.*
import java.util.*
class StartupController {
private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
private val userHome = System.getProperty("user.home")
private val tcorHome = "$userHome/.tcor"
private val dirTcorHome = File(tcorHome)
private val dirTcorCache = File("$tcorHome/cache")
private val fileConfig = File("$tcorHome/config.xml")
companion object {
val userHome: String = System.getProperty("user.home")
val tcorHome = "$userHome/.tcor"
val dirTcorHome = File(tcorHome)
val dirTcorCache = File("$tcorHome/cache")
val fileConfig = File("$tcorHome/config.xml")
var cachetAPIKey = "0"
var cachetBaseURL = "https://status.mosad.xyz"
var courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/"
var mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
var mensaName = "Offenburg"
}
init {
// if the tcor directory doesn't exist, create it
@ -56,8 +65,14 @@ class StartupController {
} else {
createConfig()
}
// TODO read cached timetable files, as they are not initially cached
readCachedTimetables()
}
/**
* load the config stored in the config.xml file
*/
private fun loadConfig() = try {
val properties = Properties()
properties.loadFromXML(FileInputStream(fileConfig))
@ -90,6 +105,9 @@ class StartupController {
logger.error("error while loading config", ex)
}
/**
* create an initial config file
*/
private fun createConfig() = try {
val properties = Properties()
@ -105,32 +123,24 @@ class StartupController {
logger.error("error while creating config", ex)
}
companion object {
private var cachetAPIKey = "0"
private var cachetBaseURL = "https://status.mosad.xyz"
private var courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/"
private var mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
private var mensaName = "Offenburg"
fun getCachetAPIKey(): String {
return cachetAPIKey
/**
* read all previously cached timetables
*/
private fun readCachedTimetables() {
dirTcorCache.walkTopDown().forEach {
if (it.isFile && it.name.endsWith(".json")) {
try {
val fileReader = FileReader(it)
val bufferedReader = BufferedReader(fileReader)
val timetableObject = JsonParser.parseString(bufferedReader.readLine()).asJsonObject
CacheController.timetableList.add(Gson().fromJson(timetableObject, TimetableCourseWeek().javaClass))
bufferedReader.close()
fileReader.close()
} catch (ex: Exception) {
logger.error("error while reading cache", ex)
}
}
}
fun getCachetBaseURL(): String {
return cachetBaseURL
}
fun getCourseListURL(): String {
return courseListURL
}
fun getMensaMenuURL(): String {
return mensaMenuURL
}
fun getMensaName(): String {
return mensaName
}
}
}
Loading…
Cancel
Save

Du besuchst diese Seite mit einem veralteten IPv4-Internetzugang. Möglicherweise treten in Zukunft Probleme mit der Erreichbarkeit und Performance auf. Bitte frage deinen Internetanbieter oder Netzwerkadministrator nach IPv6-Unterstützung.
You are visiting this site with an outdated IPv4 internet access. You may experience problems with accessibility and performance in the future. Please ask your ISP or network administrator for IPv6 support.
Weitere Infos | More Information
Klicke zum schließen | Click to close