@ -23,82 +23,99 @@
package org.mosad.thecitadelofricks.controller
import com.google.gson.Gson
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.async
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.*
import org.jsoup.Jsoup
import org.mosad.thecitadelofricks.*
import org.mosad.thecitadelofricks.hsoparser.CourseListParser
import org.mosad.thecitadelofricks.hsoparser.MensaParser
import org.mosad.thecitadelofricks.hsoparser.TimetableParser
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.io.BufferedWriter
import java.io.File
import java.io.FileWriter
import java.io.*
import java.util.*
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.Executors
import kotlin.collections.ArrayList
import kotlin.collections.HashSet
import kotlin.concurrent.scheduleAtFixedRate
import kotlin.time.Duration.Companion.hours
import kotlin.time.Duration.Companion.minutes
import kotlin.time.ExperimentalTime
class CacheController {
private val logger : Logger = LoggerFactory . getLogger ( CacheController :: class . java )
init {
initUpdates ( )
scheduledUpdates ( )
}
// cache objects
companion object {
companion object {
private val logger : Logger = LoggerFactory . getLogger ( CacheController :: class . java )
lateinit var courseList : CourseList
lateinit var mensaMenu : MensaMenu
var timetableList = ArrayList < TimetableCourseWeek > ( ) // this list contains all timetables
var courseList = CoursesList ( CoursesMeta ( ) , sortedMapOf ( ) )
var mensaMenu = MensaMenu ( MensaMeta ( 0 , " " ) , MensaWeek ( ) , MensaWeek ( ) )
var timetableList = ConcurrentHashMap < String , TimetableCourseWeek > ( ) // this list contains all timetables
/ * *
* get a timetable , since they may not cached , we need to make sure it ' s cached , otherwise download
* get a timetable , since they may not be cached , we need to make sure it ' s cached , otherwise download
* @param courseName the name of the course to be requested
* @param weekIndex request week number ( current week = 0 )
* @return timetable of the course ( Type : [ TimetableCourseWeek ] )
* /
fun getTimetable ( courseName : String , weekIndex : Int ) : TimetableCourseWeek = runBlocking {
val currentTime = System . currentTimeMillis ( ) / 1000
var timetable = TimetableWeek ( )
var weekNumberYear = 0
// check if the timetable already exists and is up to date
when ( timetableList . stream ( ) . filter { x -> x . meta . courseName == courseName && x . meta . weekIndex == weekIndex } . findAny ( ) . orElse ( null ) ) {
// there is no such course yet, create one
null -> {
val courseLink = courseList . courses . stream ( ) . filter { x -> x . courseName == courseName } . findFirst ( ) . orElse ( null ) . courseLink
val timetableLink = courseLink . replace ( " week=0 " , " week= $weekIndex " )
val jobTimetable = GlobalScope . async {
timetable = TimetableParser ( ) . getTimeTable ( timetableLink )
weekNumberYear = TimetableParser ( ) . getWeekNumberYear ( timetableLink )
}
jobTimetable . await ( )
timetableList . add (
TimetableCourseWeek ( TimetableCourseMeta ( currentTime , courseName , weekIndex , weekNumberYear , timetableLink ) ,
timetable
)
)
logger . info ( " added new timetable for $courseName , week $weekIndex " )
}
fun getTimetable ( courseName : String , weekIndex : Int ) : TimetableCourseWeek {
// TODO just for testing
if ( courseName == " TEST_A " || courseName == " TEST_B " ) {
val currentTime = System . currentTimeMillis ( ) / 1000
val timetableLink = " https://mosad.xyz "
val weekNumberYear = 0
val year = 0
val instr = CacheController :: class . java . getResourceAsStream ( " /html/Timetable_normal-week.html " )
val timetableParser =
TimetableParser ( htmlDoc = Jsoup . parse ( instr !! , " UTF-8 " , " https://www.hs-offenburg.de/ " ) )
val timetableTest = timetableParser . parseTimeTable ( )
return TimetableCourseWeek (
TimetableCourseMeta (
currentTime ,
courseName ,
weekIndex ,
weekNumberYear ,
year ,
timetableLink
) , timetableTest ?: TimetableWeek ( )
)
}
return @runBlocking timetableList . stream ( ) . filter { x -> x . meta . courseName == courseName && x . meta . weekIndex == weekIndex } . findAny ( ) . orElse ( null )
val key = " $courseName - $weekIndex "
return if ( timetableList . containsKey ( key ) ) {
timetableList [ key ] !!
} else {
val timetableLink = courseList . courses [ courseName ]
?. courseLink
?. replace ( " week=0 " , " week= $weekIndex " ) ?: " "
val currentTime = System . currentTimeMillis ( ) / 1000
val timetableParser = TimetableParser ( timetableLink )
val calendarWeek = timetableParser . parseCalendarWeek ( )
val timetable = timetableParser . parseTimeTable ( )
TimetableCourseWeek (
TimetableCourseMeta (
currentTime ,
courseName ,
weekIndex ,
calendarWeek ?. week ?: 0 ,
calendarWeek ?. year ?: 0 ,
timetableLink
) , timetable ?: TimetableWeek ( )
) . also { if ( timetable != null ) timetableList [ key ] = it }
}
}
/ * *
* get every explicit lesson in a week
* get every explicit lesson in a week for a selected course
* @param courseName the name of the course to be requested
* @param weekIndex request week number ( current week = 0 )
* @return a HashSet of explicit lessons for one week
@ -133,146 +150,146 @@ class CacheController {
return lessonList
}
}
/ * *
* this function updates the courseList
* during the update process the old data will be returned for a API request
* /
private fun asyncUpdateCourseList ( ) = GlobalScope . launch {
CourseListParser ( ) . getCourseLinks ( StartupController . courseListURL ) ?. let {
courseList = CourseList (
CourseMeta ( System . currentTimeMillis ( ) / 1000 , it . size ) , it
)
}
logger . info ( " updated courses successful at ${Date(courseList.meta.updateTime * 1000)} " )
}
/ * *
* this function updates the mensa menu list
* during the update process the old data will be returned for a API request
* /
private fun asyncUpdateMensa ( ) = GlobalScope . launch {
val mensaCurrentWeek = MensaParser ( ) . getMensaMenu ( StartupController . mensaMenuURL )
val mensaNextWeek = MensaParser ( ) . getMensaMenu ( MensaParser ( ) . getMenuLinkNextWeek ( StartupController . mensaMenuURL ) )
// only update if we get valid data
if ( mensaCurrentWeek != null && mensaNextWeek != null ) {
mensaMenu = MensaMenu (
MensaMeta ( System . currentTimeMillis ( ) / 1000 , StartupController . mensaName ) , mensaCurrentWeek , mensaNextWeek
)
}
logger . info ( " updated mensamenu successful at ${Date(mensaMenu.meta.updateTime * 1000)} " )
}
/ * *
* this function updates all existing timetables
* during the update process the old data will be returned for a API request
* a FixedThreadPool is used to make parallel requests for faster updates
* /
private fun asyncUpdateTimetables ( ) = GlobalScope . launch {
logger . info ( " updating ${timetableList.size} timetables ... " )
// create a new ThreadPool with 5 threads
val executor = Executors . newFixedThreadPool ( 5 )
try {
timetableList . forEach { timetableCourse ->
executor . execute {
timetableCourse . timetable = TimetableParser ( ) . getTimeTable ( timetableCourse . meta . link )
timetableCourse . meta . updateTime = System . currentTimeMillis ( ) / 1000
saveTimetableToCache ( timetableCourse ) // save the updated timetable to the cache directory
}
// private cache functions
/ * *
* this function updates the courseList
* during the update process the old data will be returned for an API request
* /
private fun asyncUpdateCourseList ( ) = GlobalScope . launch {
CourseListParser ( ) . getCourseLinks ( StartupController . courseListURL ) ?. let {
courseList = CoursesList ( CoursesMeta ( System . currentTimeMillis ( ) / 1000 , it . size ) , it . toSortedMap ( ) )
}
} catch ( ex : Exception ) {
logger . error ( " error while updating the timetables " , ex )
} finally {
executor . shutdown ( )
}
}
/ * *
* save a timetable to the cache directory
* this is only call on async updates , it is NOT call when first getting the timetable
* @param timetable a timetable of the type [ TimetableCourseWeek ]
* /
private fun saveTimetableToCache ( timetable : TimetableCourseWeek ) {
println ( timetable . timetable . toString ( ) )
val file = File ( StartupController . dirTcorCache , " timetable- ${timetable.meta.courseName} - ${timetable.meta.weekIndex} .json " )
val writer = BufferedWriter ( FileWriter ( file ) )
writer . write ( Gson ( ) . toJson ( timetable ) )
writer . close ( )
}
// TODO just for testing
courseList . courses [ " TEST_A " ] = Course ( " TEST_A " , " https://mosad.xyz " )
courseList . courses [ " TEST_B " ] = Course ( " TEST_B " , " https://mosad.xyz " )
/ * *
* before the APIController is up , get the data fist
* runBlocking : otherwise the api would return no data to requests for a few seconds after startup
* /
private fun initUpdates ( ) = runBlocking {
// get all courses on startup
val jobCourseUpdate = GlobalScope . async {
CourseListParser ( ) . getCourseLinks ( StartupController . courseListURL ) ?. let {
courseList = CourseList (
CourseMeta ( System . currentTimeMillis ( ) / 1000 , it . size ) , it
)
}
logger . info ( " Updated courses successful at ${Date(courseList.meta.updateTime * 1000)} " )
}
// get the current and next weeks mensa menus
val jobMensa = GlobalScope . async {
/ * *
* this function updates the mensa menu list
* during the update process the old data will be returned for an API request
* /
private fun asyncUpdateMensa ( ) = GlobalScope . launch {
val mensaCurrentWeek = MensaParser ( ) . getMensaMenu ( StartupController . mensaMenuURL )
val mensaNextWeek = MensaParser ( ) . getMensaMenu ( MensaParser ( ) . getMenuLinkNextWeek ( StartupController . mensaMenuURL ) )
// only update if we get valid data
if ( mensaCurrentWeek != null && mensaNextWeek != null ) {
mensaMenu = MensaMenu (
MensaMeta ( System . currentTimeMillis ( ) / 1000 , StartupController . mensaName ) , mensaCurrentWeek , mensaNextWeek
)
mensaMenu = MensaMenu ( MensaMeta ( System . currentTimeMillis ( ) / 1000 , StartupController . mensaName ) , mensaCurrentWeek , mensaNextWeek )
}
}
jobCourseUpdate . await ( )
jobMensa . await ( )
logger . info ( " Updated mensamenu successful at ${Date(mensaMenu.meta.updateTime * 1000)} " )
}
logger . info ( " init updates successful " )
}
/ * *
* this function updates all existing timetables
* during the update process the old data will be returned for an API request
* a FixedThreadPool is used to make parallel requests for faster updates
* /
private fun asyncUpdateTimetables ( ) = GlobalScope . launch {
logger . info ( " Updating ${timetableList.size} timetables ... " )
// create a new ThreadPool with 5 threads
val executor = Executors . newFixedThreadPool ( 5 )
try {
timetableList . forEach { timetableCourse ->
executor . execute {
val timetableParser = TimetableParser ( timetableCourse . value . meta . link )
timetableCourse . value . timetable = timetableParser . parseTimeTable ( ) ?: return @execute
timetableCourse . value . meta . weekNumberYear =
timetableParser . parseCalendarWeek ( ) ?. week ?: return @execute
timetableCourse . value . meta . updateTime = System . currentTimeMillis ( ) / 1000
saveTimetableToCache ( timetableCourse . value ) // save the updated timetable to the cache directory
}
/ * *
* update the CourseList every 24 h , the Timetables every 3 h and the Mensa Menu every hour
* doesn ' t account the change between winter and summer time !
* /
private fun scheduledUpdates ( ) {
val currentTime = System . currentTimeMillis ( )
val initDelay24h = ( 86400000 - ( ( currentTime + 3600000 ) % 86400000 ) ) + 60000
val initDelay3h = ( 10800000 - ( ( currentTime + 3600000 ) % 10800000 ) ) + 60000
val initDelay1h = ( 3600000 - ( ( currentTime + 3600000 ) % 3600000 ) ) + 60000
// update courseList every 24 hours (time in ms)
Timer ( ) . scheduleAtFixedRate ( initDelay24h , 86400000 ) {
asyncUpdateCourseList ( )
}
} catch ( ex : Exception ) {
logger . error ( " Error while updating the timetables " , ex )
} finally {
executor . shutdown ( )
}
}
// update all already existing timetables every 3 hours (time in ms)
Timer ( ) . scheduleAtFixedRate ( initDelay3h , 10800000 ) {
asyncUpdateTimetables ( )
/ * *
* save a timetable to the cache directory
* this is only call on async updates , it is NOT call when first getting the timetable
* @param timetable a timetable of the type [ TimetableCourseWeek ]
* /
private fun saveTimetableToCache ( timetable : TimetableCourseWeek ) {
val file = File ( StartupController . dirTcorCache , " timetable- ${timetable.meta.courseName} - ${timetable.meta.weekIndex} .json " )
val writer = BufferedWriter ( FileWriter ( file ) )
try {
writer . write ( Gson ( ) . toJson ( timetable ) )
} catch ( e : Exception ) {
logger . error ( " something went wrong while trying to write a cache file " , e )
} finally {
writer . close ( )
}
}
// update courses every hour (time in ms)
Timer ( ) . scheduleAtFixedRate ( initDelay1h , 3600000 ) {
asyncUpdateMensa ( )
/ * *
* before the APIController is up , get the data fist
* runBlocking : otherwise the api would return no data to requests for a few seconds after startup
* /
private fun initUpdates ( ) = runBlocking {
// get all course links on startup, make sure there are course links
val jobCourseUpdate = asyncUpdateCourseList ( )
val jobMensa = asyncUpdateMensa ( )
jobCourseUpdate . join ( )
jobMensa . join ( )
logger . info ( " Initial updates successful " )
}
// post to status.mosad.xyz every hour, if an API key is present
if ( StartupController . cachetAPIKey != " 0 " ) {
Timer ( ) . scheduleAtFixedRate ( initDelay1h , 3600000 ) {
CachetAPIController . postTotalRequests ( )
/ * *
* update the CourseList every 24 h , the Timetables every 3 h and the Mensa Menu every hour
* doesn ' t account the change between winter and summer time !
* /
@OptIn ( ExperimentalTime :: class )
private fun scheduledUpdates ( ) {
val currentTime = System . currentTimeMillis ( )
val duration24h = 24. hours . inWholeMilliseconds
val duration3h = 3. hours . inWholeMilliseconds
val duration1h = 1. hours . inWholeMilliseconds
val duration1m = 1. minutes . inWholeMilliseconds
// Calculate the initial delay to make the update time independent of the start time
fun calcInitDelay ( period : Long ) = ( period - ( ( currentTime + duration1h ) % period ) ) + duration1m
val initDelay24h = calcInitDelay ( duration24h )
val initDelay3h = calcInitDelay ( duration3h )
val initDelay1h = calcInitDelay ( duration1h )
// update courseList every 24 hours (time in ms)
Timer ( ) . scheduleAtFixedRate ( initDelay24h , duration24h ) {
asyncUpdateCourseList ( )
}
// update all already existing timetables every 3 hours (time in ms)
Timer ( ) . scheduleAtFixedRate ( initDelay3h , duration3h ) {
asyncUpdateTimetables ( )
}
// update courses every hour (time in ms)
Timer ( ) . scheduleAtFixedRate ( initDelay1h , duration1h ) {
asyncUpdateMensa ( )
}
// post to status.mosad.xyz every hour, if an API key is present
if ( StartupController . cachetAPIKey != " 0 " ) {
Timer ( ) . scheduleAtFixedRate ( initDelay1h , duration1h ) {
CachetAPIController . postTotalRequests ( )
}
}
}
}
}
}
