cover some timeouts

This commit is contained in:
Jannik 2019-05-25 19:37:04 +02:00
parent 62ad7a3b36
commit e804774970
Signed by: Seil0
GPG Key ID: E8459F3723C52C24
4 changed files with 53 additions and 25 deletions

View File

@ -45,12 +45,13 @@ import kotlin.concurrent.scheduleAtFixedRate
@RestController @RestController
class APIController { class APIController {
// TODO clean up and move stuff to a CacheController
// Controller stuff // Controller stuff
var logger: Logger = LoggerFactory.getLogger(APIController::class.java) var logger: Logger = LoggerFactory.getLogger(APIController::class.java)
private var requestCount = 0 private var requestCount = 0
private val startTime = System.currentTimeMillis() / 1000 private val startTime = System.currentTimeMillis() / 1000
private val softwareVersion = "1.0.3" private val softwareVersion = "1.1.0"
private val apiVersion = "1.0.2" private val apiVersion = "1.1.0"
// hso parser links (hardcoded) // hso parser links (hardcoded)
private val mensaLink = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/" private val mensaLink = "https://www.swfr.de/de/essen-trinken/speiseplaene/mensa-offenburg/"
@ -89,7 +90,7 @@ class APIController {
} }
} }
@Deprecated("courses is replaced by courseList") @Deprecated("courses is replaced by courseList", replaceWith = ReplaceWith("courseList()"))
@RequestMapping("/courses") @RequestMapping("/courses")
fun courses(): CoursesList { fun courses(): CoursesList {
return courseList() return courseList()
@ -172,11 +173,16 @@ class APIController {
try { try {
val hsoURL = URL("https://www.hs-offenburg.de/") val hsoURL = URL("https://www.hs-offenburg.de/")
val swfrURL = URL("https://www.swfr.de/")
var connection = hsoURL.openConnection() as HttpURLConnection var connection = hsoURL.openConnection() as HttpURLConnection
connection.requestMethod = "HEAD" connection.requestMethod = "HEAD"
connection.connectTimeout = 15000
hsoCode = connection.responseCode hsoCode = connection.responseCode
val swfrURL = URL("https://www.swfr.de/")
connection = swfrURL.openConnection() as HttpURLConnection connection = swfrURL.openConnection() as HttpURLConnection
connection.connectTimeout = 15000
swfrCode = connection.responseCode swfrCode = connection.responseCode
} catch (e: Exception) { } catch (e: Exception) {
logger.error("Error while fetching url response codes!", e) logger.error("Error while fetching url response codes!", e)

View File

@ -24,21 +24,34 @@ package org.mosad.thecitadelofricks.hsoparser
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.mosad.thecitadelofricks.Course import org.mosad.thecitadelofricks.Course
import org.slf4j.LoggerFactory
import java.net.SocketTimeoutException
class CourseListParser { class CourseListParser {
var logger: org.slf4j.Logger = LoggerFactory.getLogger(MensaParser::class.java)
/**
* return a list of all courses at hs-offenburg.de/studium/vorlesungsplaene/
* @return a ArrayList<Course> with all courses
*/
fun getCourseLinks(): ArrayList<Course> { fun getCourseLinks(): ArrayList<Course> {
val courseLinkList = ArrayList<Course>() val courseLinkList = ArrayList<Course>()
val courseHTML = Jsoup.connect("https://www.hs-offenburg.de/studium/vorlesungsplaene/").get() try {
val courseHTML = Jsoup.connect("https://www.hs-offenburg.de/studium/vorlesungsplaene/").get()
courseHTML.select("ul.index-group").select("li.Class").select("a[href]").forEachIndexed { _, element -> courseHTML.select("ul.index-group").select("li.Class").select("a[href]").forEachIndexed { _, element ->
courseLinkList.add( courseLinkList.add(
Course( Course(
element.text(), element.text(),
element.attr("href").replace("http", "https") element.attr("href").replace("http", "https")
)
) )
) }
} catch (ex: SocketTimeoutException) {
logger.warn("timeout from hs-offenburg.de, updating on next attempt!")
} }
return courseLinkList return courseLinkList
} }
} }

View File

@ -25,30 +25,39 @@ package org.mosad.thecitadelofricks.hsoparser
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.mosad.thecitadelofricks.Meal import org.mosad.thecitadelofricks.Meal
import org.mosad.thecitadelofricks.MensaWeek import org.mosad.thecitadelofricks.MensaWeek
import org.slf4j.LoggerFactory
import java.net.SocketTimeoutException
class MensaParser { class MensaParser {
var logger: org.slf4j.Logger = LoggerFactory.getLogger(MensaParser::class.java)
/** /**
* returns the mensa menu for the a week * returns the mensa menu for a week
* @param menuLink the url to a mensa menu (swfr)
*/ */
fun getMensaMenu(menuLink: String): MensaWeek { fun getMensaMenu(menuLink: String): MensaWeek {
val mealWeekList = MensaWeek() val mealWeekList = MensaWeek()
val menuHTML = Jsoup.connect(menuLink).get() try {
val menuHTML = Jsoup.connect(menuLink).timeout(15000).get()
menuHTML.select("#speiseplan-tabs").select("div.tab-content").select("div.menu-tagesplan") menuHTML.select("#speiseplan-tabs").select("div.tab-content").select("div.menu-tagesplan")
.forEachIndexed { dayIndex, day -> .forEachIndexed { dayIndex, day ->
val strDay = day.select("h3").text() val strDay = day.select("h3").text()
day.select("div.menu-info").forEachIndexed { mealIndex, meal -> day.select("div.menu-info").forEachIndexed { mealIndex, meal ->
val heading = day.select("h4")[mealIndex].text() val heading = day.select("h4")[mealIndex].text()
val parts = ArrayList(meal.html().substringBefore("<br>\n").replace("\n", "").split("<br>")) val parts = ArrayList(meal.html().substringBefore("<br>\n").replace("\n", "").split("<br>"))
val additives = meal.select("span.show-with-allergenes").text() val additives = meal.select("span.show-with-allergenes").text()
parts.removeIf { x -> x.isEmpty() || x.isBlank() } parts.removeIf { x -> x.isEmpty() || x.isBlank() }
mealWeekList.days[dayIndex].meals.add(Meal(strDay, heading, parts, additives))
}
mealWeekList.days[dayIndex].meals.add(Meal(strDay, heading, parts, additives))
} }
} catch (ex: SocketTimeoutException) {
} logger.warn("timeout from $menuLink, updating on next attempt!")
}
return mealWeekList return mealWeekList
} }

View File

@ -37,7 +37,7 @@ class TimetableParser {
*/ */
fun getTimeTable(timetableURL: String): TimetableWeek { fun getTimeTable(timetableURL: String): TimetableWeek {
val timetableWeek = TimetableWeek() val timetableWeek = TimetableWeek()
val scheduleHTML = Jsoup.connect(timetableURL).get() val scheduleHTML = Jsoup.connect(timetableURL).get() // TODO add a tyr catch block to cover timeouts
//val week = scheduleHTML.select("h1.timetable-caption").text() //val week = scheduleHTML.select("h1.timetable-caption").text()
//println("$week successful!\n") //println("$week successful!\n")