Various improvements #17
30
build.gradle
30
build.gradle
|
@ -1,25 +1,25 @@
|
||||||
plugins {
|
plugins {
|
||||||
id 'org.jetbrains.kotlin.jvm' version '1.4.10'
|
id 'org.jetbrains.kotlin.jvm' version '1.5.31'
|
||||||
id 'org.jetbrains.kotlin.plugin.spring' version '1.4.10'
|
id 'org.jetbrains.kotlin.plugin.spring' version '1.5.31'
|
||||||
id 'org.springframework.boot' version '2.3.3.RELEASE'
|
id 'org.springframework.boot' version '2.5.5'
|
||||||
id 'io.spring.dependency-management' version '1.0.9.RELEASE'
|
id 'io.spring.dependency-management' version '1.0.11.RELEASE'
|
||||||
}
|
}
|
||||||
|
|
||||||
group 'org.mosad'
|
group 'org.mosad'
|
||||||
version '1.2.8'
|
version '1.2.8'
|
||||||
|
|
||||||
repositories {
|
repositories {
|
||||||
jcenter()
|
mavenCentral()
|
||||||
}
|
}
|
||||||
|
|
||||||
dependencies {
|
dependencies {
|
||||||
implementation 'org.springframework.boot:spring-boot-starter-web'
|
implementation 'org.springframework.boot:spring-boot-starter-web'
|
||||||
implementation 'org.jetbrains.kotlin:kotlin-stdlib'
|
implementation 'org.jetbrains.kotlin:kotlin-stdlib'
|
||||||
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.8'
|
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2'
|
||||||
implementation 'org.jsoup:jsoup:1.13.1'
|
implementation 'org.jsoup:jsoup:1.14.3'
|
||||||
implementation 'com.google.code.gson:gson:2.8.6'
|
implementation 'com.google.code.gson:gson:2.8.8'
|
||||||
|
|
||||||
testImplementation 'org.junit.jupiter:junit-jupiter:5.6.2'
|
testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1'
|
||||||
}
|
}
|
||||||
|
|
||||||
test {
|
test {
|
||||||
|
@ -30,9 +30,17 @@ test {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def jvmTargetVersion = "11"
|
||||||
compileKotlin {
|
compileKotlin {
|
||||||
kotlinOptions.jvmTarget = "11"
|
kotlinOptions.jvmTarget = jvmTargetVersion
|
||||||
|
kotlinOptions.freeCompilerArgs += "-Xopt-in=kotlin.RequiresOptIn"
|
||||||
|
}
|
||||||
|
compileJava {
|
||||||
|
targetCompatibility = jvmTargetVersion
|
||||||
}
|
}
|
||||||
compileTestKotlin {
|
compileTestKotlin {
|
||||||
kotlinOptions.jvmTarget = "11"
|
kotlinOptions.jvmTarget = jvmTargetVersion
|
||||||
|
}
|
||||||
|
compileTestJava {
|
||||||
|
targetCompatibility = jvmTargetVersion
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
distributionBase=GRADLE_USER_HOME
|
distributionBase=GRADLE_USER_HOME
|
||||||
distributionPath=wrapper/dists
|
distributionPath=wrapper/dists
|
||||||
distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip
|
distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
|
||||||
zipStoreBase=GRADLE_USER_HOME
|
zipStoreBase=GRADLE_USER_HOME
|
||||||
zipStorePath=wrapper/dists
|
zipStorePath=wrapper/dists
|
||||||
|
|
|
@ -58,7 +58,7 @@ data class TimetableDay(val timeslots: Array<ArrayList<Lesson>> = Array(6) { Arr
|
||||||
|
|
||||||
data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() })
|
data class TimetableWeek(val days: Array<TimetableDay> = Array(6) { TimetableDay() })
|
||||||
|
|
||||||
data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, val weekNumberYear: Int = 0, val link: String = "")
|
data class TimetableCourseMeta(var updateTime: Long = 0, val courseName: String = "", val weekIndex: Int = 0, var weekNumberYear: Int = 0, val link: String = "")
|
||||||
|
|
||||||
data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek())
|
data class TimetableCourseWeek(val meta: TimetableCourseMeta = TimetableCourseMeta(), var timetable: TimetableWeek = TimetableWeek())
|
||||||
|
|
||||||
|
|
|
@ -33,11 +33,13 @@ import org.slf4j.Logger
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import java.io.*
|
import java.io.*
|
||||||
import java.util.*
|
import java.util.*
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import kotlin.collections.ArrayList
|
import kotlin.collections.ArrayList
|
||||||
import kotlin.collections.HashMap
|
|
||||||
import kotlin.collections.HashSet
|
import kotlin.collections.HashSet
|
||||||
import kotlin.concurrent.scheduleAtFixedRate
|
import kotlin.concurrent.scheduleAtFixedRate
|
||||||
|
import kotlin.time.Duration
|
||||||
|
import kotlin.time.ExperimentalTime
|
||||||
|
|
||||||
class CacheController {
|
class CacheController {
|
||||||
|
|
||||||
|
@ -46,12 +48,12 @@ class CacheController {
|
||||||
scheduledUpdates()
|
scheduledUpdates()
|
||||||
}
|
}
|
||||||
|
|
||||||
companion object{
|
companion object {
|
||||||
private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
|
private val logger: Logger = LoggerFactory.getLogger(CacheController::class.java)
|
||||||
|
|
||||||
var courseList = CoursesList(CoursesMeta(), sortedMapOf())
|
var courseList = CoursesList(CoursesMeta(), sortedMapOf())
|
||||||
var mensaMenu = MensaMenu(MensaMeta(0,""), MensaWeek(), MensaWeek())
|
var mensaMenu = MensaMenu(MensaMeta(0, ""), MensaWeek(), MensaWeek())
|
||||||
var timetableList = HashMap<String, TimetableCourseWeek>() // this list contains all timetables
|
var timetableList = ConcurrentHashMap<String, TimetableCourseWeek>() // this list contains all timetables
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* get a timetable, since they may not be cached, we need to make sure it's cached, otherwise download
|
* get a timetable, since they may not be cached, we need to make sure it's cached, otherwise download
|
||||||
|
@ -59,36 +61,52 @@ class CacheController {
|
||||||
* @param weekIndex request week number (current week = 0)
|
* @param weekIndex request week number (current week = 0)
|
||||||
* @return timetable of the course (Type: [TimetableCourseWeek])
|
* @return timetable of the course (Type: [TimetableCourseWeek])
|
||||||
*/
|
*/
|
||||||
fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek = runBlocking {
|
fun getTimetable(courseName: String, weekIndex: Int): TimetableCourseWeek {
|
||||||
|
|
||||||
// TODO just for testing
|
// TODO just for testing
|
||||||
if (courseName == "TEST_A" || courseName == "TEST_B") {
|
if (courseName == "TEST_A" || courseName == "TEST_B") {
|
||||||
val currentTime = System.currentTimeMillis() / 1000
|
val currentTime = System.currentTimeMillis() / 1000
|
||||||
val timetableLink = "https://mosad.xyz"
|
val timetableLink = "https://mosad.xyz"
|
||||||
val weekNumberYear = 0
|
val weekNumberYear = 0
|
||||||
val instr = javaClass.getResourceAsStream("/html/Timetable_normal-week.html")
|
val instr = CacheController::class.java.getResourceAsStream("/html/Timetable_normal-week.html")
|
||||||
|
|
||||||
val htmlDoc = Jsoup.parse(instr,"UTF-8", "https://www.hs-offenburg.de/")
|
val timetableParser =
|
||||||
val timetableTest = TimetableParser().parseTimeTable(htmlDoc)
|
TimetableParser(htmlDoc = Jsoup.parse(instr, "UTF-8", "https://www.hs-offenburg.de/"))
|
||||||
|
val timetableTest = timetableParser.parseTimeTable()
|
||||||
|
|
||||||
return@runBlocking TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetableTest)
|
return TimetableCourseWeek(
|
||||||
|
TimetableCourseMeta(
|
||||||
|
currentTime,
|
||||||
|
courseName,
|
||||||
|
weekIndex,
|
||||||
|
weekNumberYear,
|
||||||
|
timetableLink
|
||||||
|
), timetableTest ?: TimetableWeek()
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return@runBlocking timetableList.getOrPut("$courseName-$weekIndex") {
|
val key = "$courseName-$weekIndex"
|
||||||
|
return if (timetableList.contains(key)) {
|
||||||
|
timetableList[key]!!
|
||||||
|
} else {
|
||||||
val timetableLink = courseList.courses[courseName]
|
val timetableLink = courseList.courses[courseName]
|
||||||
?.courseLink
|
?.courseLink
|
||||||
?.replace("week=0", "week=$weekIndex") ?: ""
|
?.replace("week=0", "week=$weekIndex") ?: ""
|
||||||
val currentTime = System.currentTimeMillis() / 1000
|
val currentTime = System.currentTimeMillis() / 1000
|
||||||
var timetable = TimetableWeek()
|
|
||||||
var weekNumberYear = 0
|
|
||||||
|
|
||||||
val timetableJobs = listOf(
|
val timetableParser = TimetableParser(timetableLink)
|
||||||
async { timetable = TimetableParser().getTimeTable(timetableLink) },
|
val weekNumberYear = timetableParser.parseWeekNumberYear()
|
||||||
async { weekNumberYear = TimetableParser().getWeekNumberYear(timetableLink) }
|
val timetable = timetableParser.parseTimeTable()
|
||||||
)
|
|
||||||
|
|
||||||
timetableJobs.awaitAll()
|
TimetableCourseWeek(
|
||||||
TimetableCourseWeek(TimetableCourseMeta(currentTime, courseName, weekIndex, weekNumberYear, timetableLink), timetable)
|
TimetableCourseMeta(
|
||||||
|
currentTime,
|
||||||
|
courseName,
|
||||||
|
weekIndex,
|
||||||
|
weekNumberYear ?: 0,
|
||||||
|
timetableLink
|
||||||
|
), timetable ?: TimetableWeek()
|
||||||
|
).also { if (timetable != null) timetableList[key] = it }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -133,7 +151,7 @@ class CacheController {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* this function updates the courseList
|
* this function updates the courseList
|
||||||
* during the update process the old data will be returned for a API request
|
* during the update process the old data will be returned for an API request
|
||||||
*/
|
*/
|
||||||
private fun asyncUpdateCourseList() = GlobalScope.launch {
|
private fun asyncUpdateCourseList() = GlobalScope.launch {
|
||||||
CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
|
CourseListParser().getCourseLinks(StartupController.courseListURL)?.let {
|
||||||
|
@ -149,7 +167,7 @@ class CacheController {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* this function updates the mensa menu list
|
* this function updates the mensa menu list
|
||||||
* during the update process the old data will be returned for a API request
|
* during the update process the old data will be returned for an API request
|
||||||
*/
|
*/
|
||||||
private fun asyncUpdateMensa() = GlobalScope.launch {
|
private fun asyncUpdateMensa() = GlobalScope.launch {
|
||||||
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
|
val mensaCurrentWeek = MensaParser().getMensaMenu(StartupController.mensaMenuURL)
|
||||||
|
@ -165,7 +183,7 @@ class CacheController {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* this function updates all existing timetables
|
* this function updates all existing timetables
|
||||||
* during the update process the old data will be returned for a API request
|
* during the update process the old data will be returned for an API request
|
||||||
* a FixedThreadPool is used to make parallel requests for faster updates
|
* a FixedThreadPool is used to make parallel requests for faster updates
|
||||||
*/
|
*/
|
||||||
private fun asyncUpdateTimetables() = GlobalScope.launch {
|
private fun asyncUpdateTimetables() = GlobalScope.launch {
|
||||||
|
@ -177,7 +195,10 @@ class CacheController {
|
||||||
try {
|
try {
|
||||||
timetableList.forEach { timetableCourse ->
|
timetableList.forEach { timetableCourse ->
|
||||||
executor.execute {
|
executor.execute {
|
||||||
timetableCourse.value.timetable = TimetableParser().getTimeTable(timetableCourse.value.meta.link)
|
val timetableParser = TimetableParser(timetableCourse.value.meta.link)
|
||||||
|
timetableCourse.value.timetable = timetableParser.parseTimeTable() ?: return@execute
|
||||||
|
timetableCourse.value.meta.weekNumberYear =
|
||||||
|
timetableParser.parseWeekNumberYear() ?: return@execute
|
||||||
timetableCourse.value.meta.updateTime = System.currentTimeMillis() / 1000
|
timetableCourse.value.meta.updateTime = System.currentTimeMillis() / 1000
|
||||||
|
|
||||||
saveTimetableToCache(timetableCourse.value) // save the updated timetable to the cache directory
|
saveTimetableToCache(timetableCourse.value) // save the updated timetable to the cache directory
|
||||||
|
@ -228,30 +249,39 @@ class CacheController {
|
||||||
* update the CourseList every 24h, the Timetables every 3h and the Mensa Menu every hour
|
* update the CourseList every 24h, the Timetables every 3h and the Mensa Menu every hour
|
||||||
* doesn't account the change between winter and summer time!
|
* doesn't account the change between winter and summer time!
|
||||||
*/
|
*/
|
||||||
|
@OptIn(ExperimentalTime::class)
|
||||||
private fun scheduledUpdates() {
|
private fun scheduledUpdates() {
|
||||||
val currentTime = System.currentTimeMillis()
|
val currentTime = System.currentTimeMillis()
|
||||||
val initDelay24h = (86400000 - ((currentTime + 3600000) % 86400000)) + 60000
|
|
||||||
val initDelay3h = (10800000 - ((currentTime + 3600000) % 10800000)) + 60000
|
val duration24h = Duration.hours(24).inWholeMilliseconds
|
||||||
val initDelay1h = (3600000 - ((currentTime + 3600000) % 3600000)) + 60000
|
val duration3h = Duration.hours(3).inWholeMilliseconds
|
||||||
|
val duration1h = Duration.hours(1).inWholeMilliseconds
|
||||||
|
val duration1m = Duration.minutes(1).inWholeMilliseconds
|
||||||
|
|
||||||
|
// Calculate the initial delay to make the update time independent of the start time
|
||||||
|
fun calcInitDelay(period: Long) = (period - ((currentTime + duration1h) % period)) + duration1m
|
||||||
|
val initDelay24h = calcInitDelay(duration24h)
|
||||||
|
val initDelay3h = calcInitDelay(duration3h)
|
||||||
|
val initDelay1h = calcInitDelay(duration1h)
|
||||||
|
|
||||||
// update courseList every 24 hours (time in ms)
|
// update courseList every 24 hours (time in ms)
|
||||||
Timer().scheduleAtFixedRate(initDelay24h, 86400000) {
|
Timer().scheduleAtFixedRate(initDelay24h, duration24h) {
|
||||||
asyncUpdateCourseList()
|
asyncUpdateCourseList()
|
||||||
}
|
}
|
||||||
|
|
||||||
// update all already existing timetables every 3 hours (time in ms)
|
// update all already existing timetables every 3 hours (time in ms)
|
||||||
Timer().scheduleAtFixedRate(initDelay3h, 10800000) {
|
Timer().scheduleAtFixedRate(initDelay3h, duration3h) {
|
||||||
asyncUpdateTimetables()
|
asyncUpdateTimetables()
|
||||||
}
|
}
|
||||||
|
|
||||||
// update courses every hour (time in ms)
|
// update courses every hour (time in ms)
|
||||||
Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
|
Timer().scheduleAtFixedRate(initDelay1h, duration1h) {
|
||||||
asyncUpdateMensa()
|
asyncUpdateMensa()
|
||||||
}
|
}
|
||||||
|
|
||||||
// post to status.mosad.xyz every hour, if an API key is present
|
// post to status.mosad.xyz every hour, if an API key is present
|
||||||
if (StartupController.cachetAPIKey != "0") {
|
if (StartupController.cachetAPIKey != "0") {
|
||||||
Timer().scheduleAtFixedRate(initDelay1h, 3600000) {
|
Timer().scheduleAtFixedRate(initDelay1h, duration1h) {
|
||||||
CachetAPIController.postTotalRequests()
|
CachetAPIController.postTotalRequests()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,7 +22,7 @@
|
||||||
|
|
||||||
package org.mosad.thecitadelofricks.controller
|
package org.mosad.thecitadelofricks.controller
|
||||||
|
|
||||||
import org.mosad.thecitadelofricks.controller.StatusController.Companion.getTotalRequests
|
import org.mosad.thecitadelofricks.controller.StatusController.Companion.totalRequests
|
||||||
import org.slf4j.Logger
|
import org.slf4j.Logger
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import java.io.BufferedReader
|
import java.io.BufferedReader
|
||||||
|
@ -42,8 +42,8 @@ class CachetAPIController {
|
||||||
fun postTotalRequests() {
|
fun postTotalRequests() {
|
||||||
try {
|
try {
|
||||||
val url = URL("${StartupController.cachetBaseURL}/api/v1/metrics/1/points")
|
val url = URL("${StartupController.cachetBaseURL}/api/v1/metrics/1/points")
|
||||||
val jsonInputString = "{\"value\": ${getTotalRequests() -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
|
val jsonInputString = "{\"value\": ${totalRequests -oldTotalRequests}, \"timestamp\": \"${(System.currentTimeMillis() / 1000)}\"}"
|
||||||
oldTotalRequests = getTotalRequests()
|
oldTotalRequests = totalRequests
|
||||||
|
|
||||||
val con = url.openConnection() as HttpURLConnection
|
val con = url.openConnection() as HttpURLConnection
|
||||||
con.requestMethod = "POST"
|
con.requestMethod = "POST"
|
||||||
|
|
|
@ -135,7 +135,8 @@ class StartupController {
|
||||||
try {
|
try {
|
||||||
val timetableObject = JsonParser.parseString(bufferedReader.readLine()).asJsonObject
|
val timetableObject = JsonParser.parseString(bufferedReader.readLine()).asJsonObject
|
||||||
val timetable = Gson().fromJson(timetableObject, TimetableCourseWeek().javaClass)
|
val timetable = Gson().fromJson(timetableObject, TimetableCourseWeek().javaClass)
|
||||||
CacheController.timetableList.put("${timetable.meta.courseName}-${timetable.meta.weekIndex}", timetable)
|
CacheController.timetableList["${timetable.meta.courseName}-${timetable.meta.weekIndex}"] =
|
||||||
|
timetable
|
||||||
} catch (ex: Exception) {
|
} catch (ex: Exception) {
|
||||||
logger.error("error while reading cache", ex)
|
logger.error("error while reading cache", ex)
|
||||||
} finally {
|
} finally {
|
||||||
|
|
|
@ -39,10 +39,14 @@ class StatusController {
|
||||||
companion object {
|
companion object {
|
||||||
private val logger: Logger = LoggerFactory.getLogger(StatusController::class.java)
|
private val logger: Logger = LoggerFactory.getLogger(StatusController::class.java)
|
||||||
|
|
||||||
private var totalRequests = 0
|
var totalRequests = 0
|
||||||
private var mensaMenuRequests = 0
|
private set
|
||||||
private var courseListRequests = 0
|
var mensaMenuRequests = 0
|
||||||
private var timetableRequests = HashMap<String, Int>()
|
private set
|
||||||
|
var courseListRequests = 0
|
||||||
|
private set
|
||||||
|
var timetableRequests = HashMap<String, Int>()
|
||||||
|
private set
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* if a mensamenu/courseList/timetable is requested update the specific and total request count
|
* if a mensamenu/courseList/timetable is requested update the specific and total request count
|
||||||
|
@ -58,29 +62,10 @@ class StatusController {
|
||||||
}
|
}
|
||||||
|
|
||||||
fun updateTimetableRequests(courseName: String) {
|
fun updateTimetableRequests(courseName: String) {
|
||||||
timetableRequests[courseName] = timetableRequests.getOrPut(courseName) {0} + 1
|
timetableRequests[courseName] = (timetableRequests[courseName] ?: 0) + 1
|
||||||
totalRequests++
|
totalRequests++
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* getters and setters
|
|
||||||
*/
|
|
||||||
fun getTotalRequests(): Int {
|
|
||||||
return totalRequests
|
|
||||||
}
|
|
||||||
|
|
||||||
fun getMensaMenuRequests(): Int {
|
|
||||||
return mensaMenuRequests
|
|
||||||
}
|
|
||||||
|
|
||||||
fun getCourseListRequests(): Int {
|
|
||||||
return courseListRequests
|
|
||||||
}
|
|
||||||
|
|
||||||
fun getTimetableRequests(): HashMap<String, Int> {
|
|
||||||
return timetableRequests
|
|
||||||
}
|
|
||||||
|
|
||||||
fun getStatus(): Status {
|
fun getStatus(): Status {
|
||||||
val currentTime = System.currentTimeMillis() / 1000
|
val currentTime = System.currentTimeMillis() / 1000
|
||||||
val minutes = (currentTime - startTime) % 3600 / 60
|
val minutes = (currentTime - startTime) % 3600 / 60
|
||||||
|
@ -112,10 +97,10 @@ class StatusController {
|
||||||
"$days days, $hours:$minutes",
|
"$days days, $hours:$minutes",
|
||||||
apiVersion,
|
apiVersion,
|
||||||
softwareVersion,
|
softwareVersion,
|
||||||
getTotalRequests(),
|
totalRequests,
|
||||||
getMensaMenuRequests(),
|
mensaMenuRequests,
|
||||||
getCourseListRequests(),
|
courseListRequests,
|
||||||
getTimetableRequests(),
|
timetableRequests,
|
||||||
CacheController.timetableList.size,
|
CacheController.timetableList.size,
|
||||||
Date(CacheController.courseList.meta.updateTime * 1000),
|
Date(CacheController.courseList.meta.updateTime * 1000),
|
||||||
Date(CacheController.mensaMenu.meta.updateTime * 1000),
|
Date(CacheController.mensaMenu.meta.updateTime * 1000),
|
||||||
|
|
|
@ -22,34 +22,49 @@
|
||||||
|
|
||||||
package org.mosad.thecitadelofricks.hsoparser
|
package org.mosad.thecitadelofricks.hsoparser
|
||||||
|
|
||||||
|
import kotlinx.coroutines.runBlocking
|
||||||
|
import kotlinx.coroutines.sync.Semaphore
|
||||||
import org.jsoup.Jsoup
|
import org.jsoup.Jsoup
|
||||||
import org.jsoup.nodes.Document
|
import org.jsoup.nodes.Document
|
||||||
import org.mosad.thecitadelofricks.Lesson
|
import org.mosad.thecitadelofricks.Lesson
|
||||||
import org.mosad.thecitadelofricks.TimetableWeek
|
import org.mosad.thecitadelofricks.TimetableWeek
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
|
|
||||||
class TimetableParser {
|
/**
|
||||||
|
* @param timetableURL the URL of the timetable you want to get
|
||||||
|
* @param htmlDoc the html document to use (the timetableURL will be ignored if this value is present)
|
||||||
|
*/
|
||||||
|
class TimetableParser(timetableURL: String? = null, htmlDoc: Document? = null) {
|
||||||
private var logger: org.slf4j.Logger = LoggerFactory.getLogger(TimetableParser::class.java)
|
private var logger: org.slf4j.Logger = LoggerFactory.getLogger(TimetableParser::class.java)
|
||||||
private val days = arrayOf("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday")
|
private val days = arrayOf("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday")
|
||||||
|
|
||||||
/**
|
companion object {
|
||||||
* get the timetable from the given url
|
val semaphore = Semaphore(3, 0)
|
||||||
* the timetable is organised per row not per column;
|
}
|
||||||
* Mon 1, Tue 1, Wed 1, Thur 1, Fri 1, Sat 1, Mon 2 and so on
|
|
||||||
* @param timetableURL the URL of the timetable you want to get
|
private val htmlDoc: Document? = htmlDoc ?: timetableURL?.let {
|
||||||
*/
|
runBlocking {
|
||||||
fun getTimeTable(timetableURL: String): TimetableWeek {
|
try {
|
||||||
return try {
|
// Only allow sending a limited amount of requests at the same time
|
||||||
parseTimeTable(Jsoup.connect(timetableURL).get())
|
semaphore.acquire()
|
||||||
} catch (gex: Exception) {
|
Jsoup.connect(timetableURL).get()
|
||||||
logger.error("general TimetableParser error", gex)
|
} catch (gex: Exception) {
|
||||||
TimetableWeek()
|
logger.error("general TimetableParser error", gex)
|
||||||
|
null
|
||||||
|
} finally {
|
||||||
|
semaphore.release()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fun parseTimeTable(htmlDoc: Document): TimetableWeek {
|
/**
|
||||||
|
* parse the timetable from the previously given url
|
||||||
|
* the timetable is organised per row not per column;
|
||||||
|
* Mon 1, Tue 1, Wed 1, Thur 1, Fri 1, Sat 1, Mon 2 and so on
|
||||||
|
*/
|
||||||
|
fun parseTimeTable(): TimetableWeek? = htmlDoc?.let {
|
||||||
val timetableWeek = TimetableWeek()
|
val timetableWeek = TimetableWeek()
|
||||||
val rows = htmlDoc.select("table.timetable").select("tr[scope=\"row\"]")
|
val rows = it.select("table.timetable").select("tr[scope=\"row\"]")
|
||||||
|
|
||||||
var sDay = -1
|
var sDay = -1
|
||||||
var sRow = -1
|
var sRow = -1
|
||||||
|
@ -61,7 +76,7 @@ class TimetableParser {
|
||||||
var lessonIndexDay = 0 // the index of the lesson per timeslot
|
var lessonIndexDay = 0 // the index of the lesson per timeslot
|
||||||
|
|
||||||
// elements are now all lessons, including empty ones
|
// elements are now all lessons, including empty ones
|
||||||
row.select("td.lastcol, td[style]").forEach {element ->
|
row.select("td.lastcol, td[style]").forEach { element ->
|
||||||
|
|
||||||
// if there is a lecture with rowspan="2", we need to shift everything by one to the left. This is stupid and ugly there needs to bee an API
|
// if there is a lecture with rowspan="2", we need to shift everything by one to the left. This is stupid and ugly there needs to bee an API
|
||||||
if ((sDay > -1 && sRow > -1) && (sDay == day && ((sRow + 1) == rowIndex))) {
|
if ((sDay > -1 && sRow > -1) && (sDay == day && ((sRow + 1) == rowIndex))) {
|
||||||
|
@ -104,8 +119,7 @@ class TimetableParser {
|
||||||
|
|
||||||
lessonIndexDay++
|
lessonIndexDay++
|
||||||
|
|
||||||
if (element.hasClass("lastcol"))
|
if (element.hasClass("lastcol")) {
|
||||||
{
|
|
||||||
day++
|
day++
|
||||||
lessonIndexDay = 0
|
lessonIndexDay = 0
|
||||||
}
|
}
|
||||||
|
@ -117,20 +131,10 @@ class TimetableParser {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* get the week number of the year for the timetable
|
* parse the week number of the year for the timetable
|
||||||
* @param timetableURL the URL of the timetable you want to get
|
|
||||||
*/
|
*/
|
||||||
fun getWeekNumberYear(timetableURL: String): Int {
|
fun parseWeekNumberYear(): Int? = htmlDoc?.let {
|
||||||
return try {
|
it.select("h1.timetable-caption").text().substringAfter("- ")
|
||||||
parseWeekNumberYear(Jsoup.connect(timetableURL).get())
|
|
||||||
} catch (gex: Exception) {
|
|
||||||
logger.error("general TimetableParser error", gex)
|
|
||||||
0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fun parseWeekNumberYear(htmlDoc: Document): Int {
|
|
||||||
return htmlDoc.select("h1.timetable-caption").text().substringAfter("- ")
|
|
||||||
.substringBefore(".").replace(" ", "").toInt()
|
.substringBefore(".").replace(" ", "").toInt()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -32,8 +32,8 @@ class TimetableParserTest {
|
||||||
@Test
|
@Test
|
||||||
fun parseTimetableNormalWeek() {
|
fun parseTimetableNormalWeek() {
|
||||||
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path)
|
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path)
|
||||||
val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
|
val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
|
||||||
val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim()
|
val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim()
|
||||||
val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_normal-week.txt").readText().trim()
|
val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_normal-week.txt").readText().trim()
|
||||||
|
|
||||||
Assertions.assertEquals(expectedTimetable, actualTimetable)
|
Assertions.assertEquals(expectedTimetable, actualTimetable)
|
||||||
|
@ -42,8 +42,8 @@ class TimetableParserTest {
|
||||||
@Test
|
@Test
|
||||||
fun parseTimetableEmptyWeek() {
|
fun parseTimetableEmptyWeek() {
|
||||||
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_empty-week.html").path)
|
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_empty-week.html").path)
|
||||||
val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
|
val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
|
||||||
val actualTimetable = TimetableParser().parseTimeTable(htmlDoc).toString().trim()
|
val actualTimetable = TimetableParser(htmlDoc = htmlDoc).parseTimeTable().toString().trim()
|
||||||
val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_empty-week.txt").readText().trim()
|
val expectedTimetable = TimetableParserTest::class.java.getResource("/expected/Timetable_empty-week.txt").readText().trim()
|
||||||
|
|
||||||
Assertions.assertEquals(expectedTimetable, actualTimetable)
|
Assertions.assertEquals(expectedTimetable, actualTimetable)
|
||||||
|
@ -52,8 +52,8 @@ class TimetableParserTest {
|
||||||
@Test
|
@Test
|
||||||
fun parseWeekNumberYear() {
|
fun parseWeekNumberYear() {
|
||||||
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path)
|
val htmlFile = File(TimetableParserTest::class.java.getResource("/html/Timetable_normal-week.html").path)
|
||||||
val htmlDoc = Jsoup.parse(htmlFile,"UTF-8", "https://www.hs-offenburg.de/")
|
val htmlDoc = Jsoup.parse(htmlFile, "UTF-8", "https://www.hs-offenburg.de/")
|
||||||
val actualWeekNumberYear = TimetableParser().parseWeekNumberYear(htmlDoc)
|
val actualWeekNumberYear = TimetableParser(htmlDoc = htmlDoc).parseWeekNumberYear()
|
||||||
|
|
||||||
Assertions.assertEquals(42, actualWeekNumberYear)
|
Assertions.assertEquals(42, actualWeekNumberYear)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue