From 6f2bed65abe6402bc45425e77a231cbe4d65ec4a Mon Sep 17 00:00:00 2001 From: Seil0 Date: Tue, 12 Mar 2019 22:06:04 +0100 Subject: [PATCH] added spring and a API specific stuff this commit adds a fully working API wit /courses, /timetable?courseName=[course] and /mensamenu --- build.gradle | 25 ++- .../mosad/thecitadelofricks/APIController.kt | 178 ++++++++++++++++++ .../{Main.kt => Application.kt} | 21 +-- .../org/mosad/thecitadelofricks/DataTypes.kt | 26 ++- .../hsoparser/CourseListParser.kt | 11 +- .../hsoparser/MensaParser.kt | 34 +--- ...{TimeTableParser.kt => TimetableParser.kt} | 29 +-- 7 files changed, 247 insertions(+), 77 deletions(-) create mode 100644 src/main/kotlin/org/mosad/thecitadelofricks/APIController.kt rename src/main/kotlin/org/mosad/thecitadelofricks/{Main.kt => Application.kt} (57%) rename src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/{TimeTableParser.kt => TimetableParser.kt} (85%) diff --git a/build.gradle b/build.gradle index 2c6467e..d1d3e27 100644 --- a/build.gradle +++ b/build.gradle @@ -1,35 +1,34 @@ buildscript { ext.kotlin_version = '1.3.21' + ext.spring_boot_version = '2.1.0.RELEASE' repositories { - mavenCentral() + jcenter() } dependencies { classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" - classpath 'org.jsoup:jsoup:1.11.3' + classpath "org.jetbrains.kotlin:kotlin-allopen:$kotlin_version" + classpath "org.jsoup:jsoup:1.11.3" + classpath "org.springframework.boot:spring-boot-gradle-plugin:$spring_boot_version" } } -plugins { - id 'org.jetbrains.kotlin.jvm' version '1.3.21' -} - apply plugin: 'kotlin' -apply plugin: 'application' - -application { - mainClassName = "org.mosad.thecitadelofricks.MainKt" -} +apply plugin: 'kotlin-spring' +apply plugin: 'org.springframework.boot' +apply plugin: 'io.spring.dependency-management' repositories { - mavenCentral() + jcenter() } dependencies { - implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" + implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.1.1" implementation 'org.jsoup:jsoup:1.11.3' + implementation 'org.springframework.boot:spring-boot-starter-web' } compileKotlin { diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/APIController.kt b/src/main/kotlin/org/mosad/thecitadelofricks/APIController.kt new file mode 100644 index 0000000..d169846 --- /dev/null +++ b/src/main/kotlin/org/mosad/thecitadelofricks/APIController.kt @@ -0,0 +1,178 @@ +/** + * TheCitadelofRicks + * + * Copyright 2019 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + * + */ + +package org.mosad.thecitadelofricks + +import kotlinx.coroutines.* +import org.mosad.thecitadelofricks.hsoparser.CourseListParser +import org.mosad.thecitadelofricks.hsoparser.MensaParser +import org.mosad.thecitadelofricks.hsoparser.TimetableParser +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController +import java.util.ArrayList + + +@RestController +class APIController { + + private val mensaLink = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/" + private val mensaName = "Offenburg" + + private var coursesLinkList = ArrayList() + private var coursesLastUpdate: Long = 0 + + private var timetableList = ArrayList() // this list contains all timetables + + private var mensaCurrentWeek = MensaWeek() + private var mensaNextWeek = MensaWeek() + private var mensaLastUpdate: Long = 0 + + init { + initUpdates() + } + + @RequestMapping("/courses") + fun courses(): CoursesList { + println("courses request at " + System.currentTimeMillis() / 1000 + "!") + updateCoursesAsync() // check if we need to update and perform the update if so + return CoursesList(CoursesMeta(coursesLinkList.size, coursesLastUpdate), coursesLinkList) + } + + @RequestMapping("/mensamenu") + fun mensamenu(): Mensa { + println("mensamenu request at " + System.currentTimeMillis() / 1000 + "!") + updateMensa() // check if we need to update and perform the update if so + return Mensa(MensaMeta(mensaName, mensaLastUpdate), mensaCurrentWeek, mensaNextWeek) + } + + @RequestMapping("/timetable") + fun timetable(@RequestParam(value = "courseName", defaultValue = "AI4") courseName: String): TimetableCourse { + println("timetable request at " + System.currentTimeMillis() / 1000 + "!") + updateTimetableCourse(courseName) // check if we need to update and perform the update if so + return timetableList.stream().filter { x -> x.meta.courseName == courseName }.findAny().orElse(null) + } + + /** + * checks if we need to update the courses list and if so does it async + * during the update process the old data will be returned for a API request + * update if the last update was 24 hours ago + */ + private fun updateCoursesAsync() = GlobalScope.launch { + val currentTime = System.currentTimeMillis() / 1000 + if ((currentTime - coursesLastUpdate) > 86400) { + coursesLinkList = CourseListParser().getCourseLinks() + coursesLastUpdate = currentTime + println("updated courses successful at " + System.currentTimeMillis() / 1000) + } else { + println("courses are up to date!") + } + } + + /** + * this function checks if we need to update the mensa menu list and if so does it + * during the update process the old data will be returned for a API request + * update if the last update was 6 hours ago + */ + private fun updateMensa() = GlobalScope.launch { + val currentTime = System.currentTimeMillis() / 1000 + if ((currentTime - coursesLastUpdate) > 21600) { + mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink) + mensaNextWeek = MensaParser().getMensaMenu(MensaParser().getMenuLinkNextWeek(mensaLink)) + mensaLastUpdate = currentTime + println("updated mensamenu successful at " + System.currentTimeMillis() / 1000) + } else { + println("mensamenu is up to date!") + } + } + + /** + * this function checks if we need to update the timetable for a given course and if so does it + * during the update process the old data will be returned for a API request + * update if the last update was 6 hours ago + */ + private fun updateTimetableCourse(courseName: String) = runBlocking { + val currentTime = System.currentTimeMillis() / 1000 + var currentWeek = TimetableWeek() + var nextWeek = TimetableWeek() + // check if the timetable already exists and is up to date + val result = timetableList.stream().filter { x -> x.meta.courseName == courseName }.findAny().orElse(null) + + when { + // there is no such course yet, create one + result == null -> { + val courseLink = coursesLinkList.stream().filter { x -> x.courseName == courseName }.findFirst().orElse(null).courseLink + val timetableMeta = TimetableMeta(courseName, courseLink, currentTime) + + val jobCurrent = GlobalScope.async { + currentWeek = TimetableParser().getTimeTable(courseLink) + } + + val jobNext = GlobalScope.async { + nextWeek = TimetableParser().getTimeTable(courseLink.replace("week=0","week=1")) + } + + jobCurrent.await() + jobNext.await() + + timetableList.add(TimetableCourse(timetableMeta, currentWeek, nextWeek)) + } + // update + (currentTime - result.meta.time) > 21600 -> { + val index = timetableList.indexOf(result) + println("update $courseName wit index: $index") + + GlobalScope.async { + val courseLink = coursesLinkList.stream().filter { x -> x.courseName == courseName }.findFirst().orElse(null).courseLink + + timetableList[index].currentWeek = TimetableParser().getTimeTable(courseLink) + timetableList[index].nextWeek = TimetableParser().getTimeTable(courseLink.replace("week=0","week=1")) + timetableList[index].meta.time = currentTime + } + } + else -> println("timetable for $courseName is up to date") + } + } + + private fun initUpdates() = runBlocking { + // get all courses on startup + val jobCourseUpdate = GlobalScope.async{ + coursesLinkList = CourseListParser().getCourseLinks() + coursesLastUpdate = System.currentTimeMillis() / 1000 + } + + // get the current and next weeks mensa menus + val jobCurrentMensa = GlobalScope.async{ + mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink) + } + + val jobNextMensa = GlobalScope.async{ + mensaCurrentWeek = MensaParser().getMensaMenu(mensaLink) + mensaLastUpdate = System.currentTimeMillis() / 1000 + } + + jobCourseUpdate.await() + jobCurrentMensa.await() + jobNextMensa.await() + } + +} \ No newline at end of file diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/Main.kt b/src/main/kotlin/org/mosad/thecitadelofricks/Application.kt similarity index 57% rename from src/main/kotlin/org/mosad/thecitadelofricks/Main.kt rename to src/main/kotlin/org/mosad/thecitadelofricks/Application.kt index 4a6059d..1175b98 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/Main.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/Application.kt @@ -22,19 +22,12 @@ package org.mosad.thecitadelofricks -import org.mosad.thecitadelofricks.hsoparser.CourseListParser -import org.mosad.thecitadelofricks.hsoparser.TimeTableParser - - fun main() { - - // TESTING AREA - val courseLinks = CourseListParser().getCourseLinks() - println(courseLinks) - - val timeTableWeek0 = TimeTableParser().getTimeTable("https://www.hs-offenburg.de/index.php?id=6627&class=class&iddV=5D255C23-BC03-4AA0-9F36-DC6767F3E05D&week=0") - - val timeTableWeek1 = TimeTableParser().getTimeTable("https://www.hs-offenburg.de/index.php?id=6627&class=class&iddV=5D255C23-BC03-4AA0-9F36-DC6767F3E05D&week=1") - } - +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +@SpringBootApplication +class Application +fun main(args: Array) { + SpringApplication.run(Application::class.java, *args) +} \ No newline at end of file diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt b/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt index 63d90f2..464b0e4 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/DataTypes.kt @@ -22,15 +22,31 @@ package org.mosad.thecitadelofricks +// data classes for the course part +data class Course(val courseName: String, val courseLink: String) -data class Course(val courseLink: String, val courseName: String) +data class CoursesMeta(val totalCourses: Int, val time: Long) +data class CoursesList(val meta: CoursesMeta, val courses: ArrayList) + +// data classes for the Mensa part data class Meal(val day: String, val heading: String, val parts: ArrayList, val additives: String) -data class MealWeek(val day: Array> = Array(7) { ArrayList() }) +data class Meals(val meals: ArrayList) -data class Lesson(val lessonSubject: String, val lessonTeacher: String, val lessonRoom:String, val lessonRemark: String) +data class MensaWeek(val days: Array = Array(7) { Meals(ArrayList()) }) -data class TimeTableDay( val timeslots: Array> = Array(6) { ArrayList()}) +data class MensaMeta(val mensaName: String, val time: Long) -data class TimeTable(val days: Array = Array(6) { TimeTableDay() }) \ No newline at end of file +data class Mensa(val meta: MensaMeta, val currentWeek: MensaWeek, val nextWeek: MensaWeek) + +// data classes for the timetable part +data class Lesson(val lessonID: String, val lessonSubject: String, val lessonTeacher: String, val lessonRoom:String, val lessonRemark: String) + +data class TimetableDay( val timeslots: Array> = Array(6) { ArrayList()}) + +data class TimetableWeek(val days: Array = Array(6) { TimetableDay() }) + +data class TimetableMeta(val courseName: String, val courseLink: String, var time: Long) + +data class TimetableCourse(val meta: TimetableMeta, var currentWeek: TimetableWeek, var nextWeek: TimetableWeek) \ No newline at end of file diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/CourseListParser.kt b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/CourseListParser.kt index 1bbd94b..ab6cc47 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/CourseListParser.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/CourseListParser.kt @@ -20,7 +20,6 @@ * */ - package org.mosad.thecitadelofricks.hsoparser import org.jsoup.Jsoup @@ -29,17 +28,17 @@ import org.mosad.thecitadelofricks.Course class CourseListParser { fun getCourseLinks(): ArrayList { - val courseTTLinkList = ArrayList() // TODO val may cause bugs! + val courseLinkList = ArrayList() val courseHTML = Jsoup.connect("https://www.hs-offenburg.de/studium/vorlesungsplaene/").get() courseHTML.select("ul.index-group").select("li.Class").select("a[href]").forEachIndexed { _, element -> - courseTTLinkList.add( + courseLinkList.add( Course( - element.attr("href").replace("http", "https"), - element.text() + element.text(), + element.attr("href").replace("http", "https") ) ) } - return courseTTLinkList + return courseLinkList } } \ No newline at end of file diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/MensaParser.kt b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/MensaParser.kt index 87b64b8..8136604 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/MensaParser.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/MensaParser.kt @@ -22,54 +22,36 @@ package org.mosad.thecitadelofricks.hsoparser -import org.jsoup.Jsoup +import org.jsoup.Jsoup import org.mosad.thecitadelofricks.Meal -import org.mosad.thecitadelofricks.MealWeek +import org.mosad.thecitadelofricks.MensaWeek class MensaParser { /** * returns the mensa menu for the a week */ - fun getMensaMenu(menuLink: String): MealWeek { - val mealList = ArrayList() - val mealWeekList = MealWeek() + fun getMensaMenu(menuLink: String): MensaWeek { + val mealWeekList = MensaWeek() val menuHTML = Jsoup.connect(menuLink).get() menuHTML.select("#speiseplan-tabs").select("div.tab-content").select("div.menu-tagesplan") .forEachIndexed { dayIndex, day -> val strDay = day.select("h3").text() - day.select("div.menu-info").forEachIndexed { mealIndex, meal -> val heading = day.select("h4")[mealIndex].text() - val parts = ArrayList(meal.html().substringBefore("
\n").replace("
", " ").split("\n")) + val parts = ArrayList(meal.html().substringBefore("
\n").replace("\n", "").split("
")) val additives = meal.select("span.show-with-allergenes").text() + parts.removeIf { x -> x.isEmpty()|| x.isBlank() } - mealWeekList.day[dayIndex].add(Meal(strDay, heading, parts, additives)) + mealWeekList.days[dayIndex].meals.add(Meal(strDay, heading, parts, additives)) } - for (i in 0..(day.select("div.row h4").size - 1)) { - try { - val heading = day.select("div.row h4")[i].text() - val parts = ArrayList( - day.select("div.row").select("div.menu-info")[i].html().substringBefore("", - " " - ).split("\n") - ) - val additives = - day.select("div.row").select("div.menu-info")[i].select("span.show-with-allergenes").text() - - mealList.add(Meal(strDay, heading, parts, additives)) - } catch (e: Exception) { - //println("Oooups! Something went wrong: ${e.printStackTrace()}") - } - } } // Mon to Sat (0 - 5) - println(mealWeekList.day[4]) + //println(mealWeekList.days[4]) return mealWeekList } diff --git a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimeTableParser.kt b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt similarity index 85% rename from src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimeTableParser.kt rename to src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt index 766c7cd..a57ee13 100644 --- a/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimeTableParser.kt +++ b/src/main/kotlin/org/mosad/thecitadelofricks/hsoparser/TimetableParser.kt @@ -24,9 +24,9 @@ package org.mosad.thecitadelofricks.hsoparser import org.jsoup.Jsoup import org.mosad.thecitadelofricks.Lesson -import org.mosad.thecitadelofricks.TimeTable +import org.mosad.thecitadelofricks.TimetableWeek -class TimeTableParser { +class TimetableParser { private val days = arrayOf("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday") /** @@ -34,8 +34,8 @@ class TimeTableParser { * the timetable is organised per row not per column; * Mon 1, Tue 1, Wed 1, Thur 1, Fri 1, Sat 1, Mon 2 and so on */ - fun getTimeTable(courseTTURL: String): TimeTable { - val timeTableWeek = TimeTable() + fun getTimeTable(courseTTURL: String): TimetableWeek { + val timetableWeek = TimetableWeek() val scheduleHTML = Jsoup.connect(courseTTURL).get() //val week = scheduleHTML.select("h1.timetable-caption").text() @@ -44,23 +44,24 @@ class TimeTableParser { val rows = scheduleHTML.select("table.timetable").select("tr[scope=\"row\"]") var sDay = -1 var sRow = -1 - var sLesson = Lesson("", "", "", "") + var sLesson = Lesson("", "", "", "", "") // get each row with index, reflects 1 timeslot per day for ((rowIndex, row) in rows.withIndex()) { var day = 0 // elements are now all lessons, including empty ones - row.select("td.lastcol, td[style]").forEach { element -> + row.select("td.lastcol, td[style]").forEachIndexed {elementIndex, element -> // if there is a lecture with rowspan="2", we need to shift everything by one to the left. This is stupid and ugly there needs to bee an API if ((sDay > -1 && sRow > -1) && (sDay == day && ((sRow + 1) == rowIndex))) { // we found a lecture that is longer than 1 lesson - timeTableWeek.days[day].timeslots[rowIndex].add(sLesson) // this just works if there is one lecture per slot + timetableWeek.days[day].timeslots[rowIndex].add(sLesson) // this just works if there is one lecture per slot // adjust the following slot sDay++ sLesson = Lesson( + "$day.$rowIndex.$elementIndex", element.select("div.lesson-subject").text(), element.select("div.lesson-teacher").text(), element.select("div.lesson-room").text(), @@ -69,12 +70,13 @@ class TimeTableParser { // adjust the slot directly as we don't get there anymore if (sDay == 5) { - timeTableWeek.days[day + 1].timeslots[rowIndex].add(sLesson) + timetableWeek.days[day + 1].timeslots[rowIndex].add(sLesson) } } else { - timeTableWeek.days[day].timeslots[rowIndex].add( + timetableWeek.days[day].timeslots[rowIndex].add( Lesson( + "$day.$rowIndex.$elementIndex", element.select("div.lesson-subject").text(), element.select("div.lesson-teacher").text(), element.select("div.lesson-room").text(), @@ -87,7 +89,7 @@ class TimeTableParser { if (element.toString().contains("rowspan=\"2\"")) { sDay = day sRow = rowIndex - sLesson = timeTableWeek.days[day].timeslots[rowIndex].get(index = 0) + sLesson = timetableWeek.days[day].timeslots[rowIndex].get(index = 0) } if (element.hasClass("lastcol")) day++ @@ -95,12 +97,13 @@ class TimeTableParser { } - printTimeTableWeek(timeTableWeek) + //printTimetableWeek(timetableWeek) - return timeTableWeek + return timetableWeek } - fun printTimeTableWeek(timetable: TimeTable) { + @Suppress("unused") + fun printTimetableWeek(timetable: TimetableWeek) { for (j in 0..5) print(days[j].padEnd(75, ' ') + " | ") println() for (j in 0..5) print("-".padEnd(76 + (j.toFloat().div(j).toInt()), '-') + "+")