added on disk cache for the timetables
continuous-integration/drone/push Build is passing
Details
continuous-integration/drone/push Build is passing
Details
* update spring-boot 2.1.9 -> 2.1.10
This commit is contained in:
parent
dd064d63af
commit
697f5e3167
|
@ -1,7 +1,7 @@
|
||||||
|
|
||||||
buildscript {
|
buildscript {
|
||||||
ext.kotlin_version = '1.3.50'
|
ext.kotlin_version = '1.3.50'
|
||||||
ext.spring_boot_version = '2.1.9.RELEASE'
|
ext.spring_boot_version = '2.1.10.RELEASE'
|
||||||
|
|
||||||
repositories {
|
repositories {
|
||||||
jcenter()
|
jcenter()
|
||||||
|
@ -34,8 +34,9 @@ repositories {
|
||||||
dependencies {
|
dependencies {
|
||||||
implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
|
implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
|
||||||
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.2"
|
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.2"
|
||||||
implementation 'org.jsoup:jsoup:1.12.1'
|
implementation "org.jsoup:jsoup:1.12.1"
|
||||||
implementation 'org.springframework.boot:spring-boot-starter-web'
|
implementation "org.springframework.boot:spring-boot-starter-web"
|
||||||
|
implementation "com.google.code.gson:gson:2.8.6"
|
||||||
|
|
||||||
testImplementation("org.junit.jupiter:junit-jupiter:5.5.1")
|
testImplementation("org.junit.jupiter:junit-jupiter:5.5.1")
|
||||||
}
|
}
|
||||||
|
|
|
@ -56,9 +56,9 @@ data class TimetableDay(val timeslots: Array<ArrayList<Lesson>> = Array(6) { Arr
|
||||||
|
|
||||||
data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() })
|
data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() })
|
||||||
|
|
||||||
data class TimetableCourseMeta(var updateTime: Long, val courseName: String, val weekIndex: Int, val weekNumberYear: Int, val link: String)
|
data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, val weekNumberYear: Int = 0, val link: String = "")
|
||||||
|
|
||||||
data class TimetableCourseWeek(val meta: TimetableCourseMeta, var timetable: TimetableWeek)
|
data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek())
|
||||||
|
|
||||||
|
|
||||||
// data classes for the status part
|
// data classes for the status part
|
||||||
|
|
|
@ -22,6 +22,7 @@
|
||||||
|
|
||||||
package org.mosad.thecitadelofricks.controller
|
package org.mosad.thecitadelofricks.controller
|
||||||
|
|
||||||
|
import com.google.gson.Gson
|
||||||
import kotlinx.coroutines.GlobalScope
|
import kotlinx.coroutines.GlobalScope
|
||||||
import kotlinx.coroutines.async
|
import kotlinx.coroutines.async
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
|
@ -32,6 +33,9 @@ import org.mosad.thecitadelofricks.hsoparser.MensaParser
|
||||||
import org.mosad.thecitadelofricks.hsoparser.TimetableParser
|
import org.mosad.thecitadelofricks.hsoparser.TimetableParser
|
||||||
import org.slf4j.Logger
|
import org.slf4j.Logger
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
|
import java.io.BufferedWriter
|
||||||
|
import java.io.File
|
||||||
|
import java.io.FileWriter
|
||||||
import java.util.*
|
import java.util.*
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import kotlin.collections.ArrayList
|
import kotlin.collections.ArrayList
|
||||||
|
@ -59,7 +63,7 @@ class CacheController {
|
||||||
* get a timetable, since they may not cached, we need to make sure it's cached, otherwise download
|
* get a timetable, since they may not cached, we need to make sure it's cached, otherwise download
|
||||||
* @param courseName the name of the course to be requested
|
* @param courseName the name of the course to be requested
|
||||||
* @param weekIndex request week number (current week = 0)
|
* @param weekIndex request week number (current week = 0)
|
||||||
* @return the timetable of a course (courseName)
|
* @return timetable of the course (Type: [TimetableCourseWeek])
|
||||||
*/
|
*/
|
||||||
fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking {
|
fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking {
|
||||||
val currentTime = System.currentTimeMillis() / 1000
|
val currentTime = System.currentTimeMillis() / 1000
|
||||||
|
@ -81,17 +85,11 @@ class CacheController {
|
||||||
jobTimetable.await()
|
jobTimetable.await()
|
||||||
|
|
||||||
timetableList.add(
|
timetableList.add(
|
||||||
TimetableCourseWeek(
|
TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink),
|
||||||
TimetableCourseMeta(
|
|
||||||
currentTime,
|
|
||||||
courseName,
|
|
||||||
weekIndex,
|
|
||||||
weekNumberYear,
|
|
||||||
timetableLink
|
|
||||||
),
|
|
||||||
timetable
|
timetable
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("added new timetable for $courseName, week $weekIndex")
|
logger.info("added new timetable for $courseName, week $weekIndex")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -122,7 +120,7 @@ class CacheController {
|
||||||
* @param courseName the name of the course to be requested
|
* @param courseName the name of the course to be requested
|
||||||
* @param lessonSubject the lesson subject to be requested
|
* @param lessonSubject the lesson subject to be requested
|
||||||
* @param weekIndex request week number (current week = 0)
|
* @param weekIndex request week number (current week = 0)
|
||||||
* @return a ArrayList of every lesson with lessonSubject for one week
|
* @return a ArrayList<[Lesson]> of every lesson with lessonSubject for one week
|
||||||
*/
|
*/
|
||||||
fun getLesson(courseName: String, lessonSubject: String, weekIndex: Int): ArrayList<Lesson> {
|
fun getLesson(courseName: String, lessonSubject: String, weekIndex: Int): ArrayList<Lesson> {
|
||||||
val lessonList = ArrayList<Lesson>()
|
val lessonList = ArrayList<Lesson>()
|
||||||
|
@ -130,14 +128,14 @@ class CacheController {
|
||||||
// get all lessons from the weeks timetable
|
// get all lessons from the weeks timetable
|
||||||
val flatMap = getTimetable(courseName, weekIndex).timetable.days.flatMap { it.timeslots.asIterable() }
|
val flatMap = getTimetable(courseName, weekIndex).timetable.days.flatMap { it.timeslots.asIterable() }
|
||||||
flatMap.forEach {
|
flatMap.forEach {
|
||||||
|
// TODO Java 11
|
||||||
|
//it.stream().filter { x -> x.lessonSubject.contains(lessonSubject) }.findAny().ifPresent { x -> println("${x.lessonSubject}, ${x.lessonTeacher}") }
|
||||||
|
|
||||||
it.forEach { lesson ->
|
it.forEach { lesson ->
|
||||||
if(lesson.lessonSubject.contains(lessonSubject)) {
|
if(lesson.lessonSubject.contains(lessonSubject)) {
|
||||||
lessonList.add(lesson)
|
lessonList.add(lesson)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO Java 11
|
|
||||||
//it.stream().filter { x -> x.lessonSubject.contains(lessonSubject) }.findAny().ifPresent { x -> println("${x.lessonSubject}, ${x.lessonTeacher}") }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return lessonList
|
return lessonList
|
||||||
|
@ -149,10 +147,10 @@ class CacheController {
|
||||||
* during the update process the old data will be returned for a API request
|
* during the update process the old data will be returned for a API request
|
||||||
*/
|
*/
|
||||||
private fun asyncUpdateCourseList() = GlobalScope.launch {
|
private fun asyncUpdateCourseList() = GlobalScope.launch {
|
||||||
CourseListParser().getCourseLinks(StartupController.getCourseListURL())?.let {
|
CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
|
||||||
courseList = CourseList(
|
courseList = CourseList(
|
||||||
CourseMeta(System.currentTimeMillis() / 1000, it.size), it
|
CourseMeta(System.currentTimeMillis() / 1000, it.size), it
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.info("updated courses successful at ${Date(courseList.meta.updateTime * 1000)}")
|
logger.info("updated courses successful at ${Date(courseList.meta.updateTime * 1000)}")
|
||||||
|
@ -163,13 +161,13 @@ class CacheController {
|
||||||
* during the update process the old data will be returned for a API request
|
* during the update process the old data will be returned for a API request
|
||||||
*/
|
*/
|
||||||
private fun asyncUpdateMensa() = GlobalScope.launch {
|
private fun asyncUpdateMensa() = GlobalScope.launch {
|
||||||
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.getMensaMenuURL())
|
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
|
||||||
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.getMensaMenuURL()))
|
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.mensaMenuURL))
|
||||||
|
|
||||||
// only update if we get valid data
|
// only update if we get valid data
|
||||||
if (mensaCurrentWeek != null && mensaNextWeek != null) {
|
if (mensaCurrentWeek != null && mensaNextWeek != null) {
|
||||||
mensaMenu = MensaMenu(
|
mensaMenu = MensaMenu(
|
||||||
MensaMeta(System.currentTimeMillis() / 1000, StartupController.getMensaName()), mensaCurrentWeek, mensaNextWeek
|
MensaMeta(System.currentTimeMillis() / 1000, StartupController.mensaName), mensaCurrentWeek, mensaNextWeek
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -192,6 +190,8 @@ class CacheController {
|
||||||
executor.execute {
|
executor.execute {
|
||||||
timetableCourse.timetable = TimetableParser().getTimeTable(timetableCourse.meta.link)
|
timetableCourse.timetable = TimetableParser().getTimeTable(timetableCourse.meta.link)
|
||||||
timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000
|
timetableCourse.meta.updateTime = System.currentTimeMillis() / 1000
|
||||||
|
|
||||||
|
saveTimetableToCache(timetableCourse) // save the updated timetable to the cache directory
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -202,6 +202,20 @@ class CacheController {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* save a timetable to the cache directory
|
||||||
|
* this is only call on async updates, it is NOT call when first getting the timetable
|
||||||
|
* @param timetable a timetable of the type [TimetableCourseWeek]
|
||||||
|
*/
|
||||||
|
private fun saveTimetableToCache(timetable: TimetableCourseWeek) {
|
||||||
|
println(timetable.timetable.toString())
|
||||||
|
|
||||||
|
val file = File(StartupController.dirTcorCache, "timetable-${timetable.meta.courseName}-${timetable.meta.weekIndex}.json")
|
||||||
|
val writer = BufferedWriter(FileWriter(file))
|
||||||
|
writer.write(Gson().toJson(timetable))
|
||||||
|
writer.close()
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* before the APIController is up, get the data fist
|
* before the APIController is up, get the data fist
|
||||||
* runBlocking: otherwise the api would return no data to requests for a few seconds after startup
|
* runBlocking: otherwise the api would return no data to requests for a few seconds after startup
|
||||||
|
@ -209,7 +223,7 @@ class CacheController {
|
||||||
private fun initUpdates() = runBlocking {
|
private fun initUpdates() = runBlocking {
|
||||||
// get all courses on startup
|
// get all courses on startup
|
||||||
val jobCourseUpdate = GlobalScope.async {
|
val jobCourseUpdate = GlobalScope.async {
|
||||||
CourseListParser().getCourseLinks(StartupController.getCourseListURL())?.let {
|
CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
|
||||||
courseList = CourseList(
|
courseList = CourseList(
|
||||||
CourseMeta(System.currentTimeMillis() / 1000, it.size), it
|
CourseMeta(System.currentTimeMillis() / 1000, it.size), it
|
||||||
)
|
)
|
||||||
|
@ -218,13 +232,13 @@ class CacheController {
|
||||||
|
|
||||||
// get the current and next weeks mensa menus
|
// get the current and next weeks mensa menus
|
||||||
val jobMensa = GlobalScope.async{
|
val jobMensa = GlobalScope.async{
|
||||||
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.getMensaMenuURL())
|
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
|
||||||
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.getMensaMenuURL()))
|
val mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(StartupController.mensaMenuURL))
|
||||||
|
|
||||||
// only update if we get valid data
|
// only update if we get valid data
|
||||||
if (mensaCurrentWeek != null && mensaNextWeek != null) {
|
if (mensaCurrentWeek != null && mensaNextWeek != null) {
|
||||||
mensaMenu = MensaMenu(
|
mensaMenu = MensaMenu(
|
||||||
MensaMeta(System.currentTimeMillis() / 1000, StartupController.getMensaName()), mensaCurrentWeek, mensaNextWeek
|
MensaMeta(System.currentTimeMillis() / 1000, StartupController.mensaName), mensaCurrentWeek, mensaNextWeek
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -236,8 +250,8 @@ class CacheController {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* update the CourseList every 24h, the Timetables every 3h and the Mensa Menu every hour
|
||||||
* doesn't account the change between winter and summer time!
|
* doesn't account the change between winter and summer time!
|
||||||
*
|
|
||||||
*/
|
*/
|
||||||
private fun scheduledUpdates() {
|
private fun scheduledUpdates() {
|
||||||
val currentTime = System.currentTimeMillis()
|
val currentTime = System.currentTimeMillis()
|
||||||
|
@ -261,7 +275,7 @@ class CacheController {
|
||||||
}
|
}
|
||||||
|
|
||||||
// post to status.mosad.xyz every hour, if an API key is present
|
// post to status.mosad.xyz every hour, if an API key is present
|
||||||
if (StartupController.getCachetAPIKey() != "0") {
|
if (StartupController.cachetAPIKey != "0") {
|
||||||
Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
|
Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
|
||||||
CachetAPIController.postTotalRequests()
|
CachetAPIController.postTotalRequests()
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,7 +41,7 @@ class CachetAPIController {
|
||||||
|
|
||||||
fun postTotalRequests() {
|
fun postTotalRequests() {
|
||||||
try {
|
try {
|
||||||
val url = URL("${StartupController.getCachetBaseURL()}/api/v1/metrics/1/points")
|
val url = URL("${StartupController.cachetBaseURL}/api/v1/metrics/1/points")
|
||||||
val jsonInputString = "{\"value\": ${getTotalRequests() -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
|
val jsonInputString = "{\"value\": ${getTotalRequests() -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
|
||||||
oldTotalRequests = getTotalRequests()
|
oldTotalRequests = getTotalRequests()
|
||||||
|
|
||||||
|
@ -49,7 +49,7 @@ class CachetAPIController {
|
||||||
con.requestMethod = "POST"
|
con.requestMethod = "POST"
|
||||||
con.setRequestProperty("Content-Type", "application/json; utf-8")
|
con.setRequestProperty("Content-Type", "application/json; utf-8")
|
||||||
con.setRequestProperty("Accept", "application/json")
|
con.setRequestProperty("Accept", "application/json")
|
||||||
con.setRequestProperty("X-Cachet-Token", StartupController.getCachetAPIKey())
|
con.setRequestProperty("X-Cachet-Token", StartupController.cachetAPIKey)
|
||||||
con.doOutput = true
|
con.doOutput = true
|
||||||
|
|
||||||
val os = con.outputStream
|
val os = con.outputStream
|
||||||
|
|
|
@ -22,22 +22,31 @@
|
||||||
|
|
||||||
package org.mosad.thecitadelofricks.controller
|
package org.mosad.thecitadelofricks.controller
|
||||||
|
|
||||||
|
import com.google.gson.Gson
|
||||||
|
import com.google.gson.JsonParser
|
||||||
|
import org.mosad.thecitadelofricks.TimetableCourseWeek
|
||||||
import org.slf4j.Logger
|
import org.slf4j.Logger
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import java.io.File
|
import java.io.*
|
||||||
import java.io.FileInputStream
|
|
||||||
import java.io.FileOutputStream
|
|
||||||
import java.util.*
|
import java.util.*
|
||||||
|
|
||||||
class StartupController {
|
class StartupController {
|
||||||
|
|
||||||
private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
|
private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
|
||||||
|
|
||||||
private val userHome = System.getProperty("user.home")
|
companion object {
|
||||||
private val tcorHome = "$userHome/.tcor"
|
val userHome: String = System.getProperty("user.home")
|
||||||
private val dirTcorHome = File(tcorHome)
|
val tcorHome = "$userHome/.tcor"
|
||||||
private val dirTcorCache = File("$tcorHome/cache")
|
val dirTcorHome = File(tcorHome)
|
||||||
private val fileConfig = File("$tcorHome/config.xml")
|
val dirTcorCache = File("$tcorHome/cache")
|
||||||
|
val fileConfig = File("$tcorHome/config.xml")
|
||||||
|
|
||||||
|
var cachetAPIKey = "0"
|
||||||
|
var cachetBaseURL = "https://status.mosad.xyz"
|
||||||
|
var courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/"
|
||||||
|
var mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
|
||||||
|
var mensaName = "Offenburg"
|
||||||
|
}
|
||||||
|
|
||||||
init {
|
init {
|
||||||
// if the tcor directory doesn't exist, create it
|
// if the tcor directory doesn't exist, create it
|
||||||
|
@ -56,8 +65,14 @@ class StartupController {
|
||||||
} else {
|
} else {
|
||||||
createConfig()
|
createConfig()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO read cached timetable files, as they are not initially cached
|
||||||
|
readCachedTimetables()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* load the config stored in the config.xml file
|
||||||
|
*/
|
||||||
private fun loadConfig() = try {
|
private fun loadConfig() = try {
|
||||||
val properties = Properties()
|
val properties = Properties()
|
||||||
properties.loadFromXML(FileInputStream(fileConfig))
|
properties.loadFromXML(FileInputStream(fileConfig))
|
||||||
|
@ -90,6 +105,9 @@ class StartupController {
|
||||||
logger.error("error while loading config", ex)
|
logger.error("error while loading config", ex)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* create an initial config file
|
||||||
|
*/
|
||||||
private fun createConfig() = try {
|
private fun createConfig() = try {
|
||||||
val properties = Properties()
|
val properties = Properties()
|
||||||
|
|
||||||
|
@ -105,32 +123,24 @@ class StartupController {
|
||||||
logger.error("error while creating config", ex)
|
logger.error("error while creating config", ex)
|
||||||
}
|
}
|
||||||
|
|
||||||
companion object {
|
/**
|
||||||
private var cachetAPIKey = "0"
|
* read all previously cached timetables
|
||||||
private var cachetBaseURL = "https://status.mosad.xyz"
|
*/
|
||||||
private var courseListURL = "https://www.hs-offenburg.de/studium/vorlesungsplaene/"
|
private fun readCachedTimetables() {
|
||||||
private var mensaMenuURL = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
|
dirTcorCache.walkTopDown().forEach {
|
||||||
private var mensaName = "Offenburg"
|
if (it.isFile && it.name.endsWith(".json")) {
|
||||||
|
try {
|
||||||
|
val fileReader = FileReader(it)
|
||||||
|
val bufferedReader = BufferedReader(fileReader)
|
||||||
|
val timetableObject = JsonParser.parseString(bufferedReader.readLine()).asJsonObject
|
||||||
|
|
||||||
fun getCachetAPIKey(): String {
|
CacheController.timetableList.add(Gson().fromJson(timetableObject, TimetableCourseWeek().javaClass))
|
||||||
return cachetAPIKey
|
bufferedReader.close()
|
||||||
|
fileReader.close()
|
||||||
|
} catch (ex: Exception) {
|
||||||
|
logger.error("error while reading cache", ex)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun getCachetBaseURL(): String {
|
|
||||||
return cachetBaseURL
|
|
||||||
}
|
|
||||||
|
|
||||||
fun getCourseListURL(): String {
|
|
||||||
return courseListURL
|
|
||||||
}
|
|
||||||
|
|
||||||
fun getMensaMenuURL(): String {
|
|
||||||
return mensaMenuURL
|
|
||||||
}
|
|
||||||
fun getMensaName(): String {
|
|
||||||
return mensaName
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
Loading…
Reference in New Issue