2020-10-08 22:20:20 +02:00
package org.mosad.teapod.parser
2020-10-11 23:16:47 +02:00
import android.util.Log
2020-10-11 10:02:00 +02:00
import com.google.gson.JsonParser
2020-10-09 15:18:52 +02:00
import kotlinx.coroutines.*
2020-10-08 22:20:20 +02:00
import org.jsoup.Connection
import org.jsoup.Jsoup
2020-10-11 13:18:20 +02:00
import org.mosad.teapod.preferences.EncryptedPreferences
2020-10-11 23:16:47 +02:00
import org.mosad.teapod.util.DataTypes.MediaType
2020-10-12 17:52:24 +02:00
import org.mosad.teapod.util.Episode
import org.mosad.teapod.util.Media
2020-10-15 16:23:52 +02:00
import java.io.IOException
2020-10-12 17:52:24 +02:00
import java.util.*
2020-10-11 23:16:47 +02:00
import kotlin.collections.ArrayList
2020-10-08 22:20:20 +02:00
2020-10-13 23:47:48 +02:00
/ * *
* maybe AoDParser as object would be useful
* /
2020-10-08 22:20:20 +02:00
class AoDParser {
2020-10-12 20:30:45 +02:00
private val baseUrl = " https://www.anime-on-demand.de "
2020-10-08 22:20:20 +02:00
private val loginPath = " /users/sign_in "
2020-10-11 13:18:20 +02:00
private val libraryPath = " /animes "
2020-10-08 22:20:20 +02:00
2020-10-15 18:51:29 +02:00
private val userAgent = " Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0 "
2020-10-11 10:02:00 +02:00
companion object {
2020-10-13 23:47:48 +02:00
private var csrfToken : String = " "
2020-10-11 10:02:00 +02:00
private var sessionCookies = mutableMapOf < String , String > ( )
private var loginSuccess = false
2020-10-12 17:52:24 +02:00
val mediaList = arrayListOf < Media > ( )
2020-10-11 10:02:00 +02:00
}
2020-10-08 22:20:20 +02:00
2020-10-12 23:26:32 +02:00
fun login ( ) : Boolean = runBlocking {
2020-10-08 22:20:20 +02:00
withContext ( Dispatchers . Default ) {
// get the authenticity token
2020-10-12 20:30:45 +02:00
val resAuth = Jsoup . connect ( baseUrl + loginPath )
2020-10-08 22:20:20 +02:00
. header ( " User-Agent " , userAgent )
. execute ( )
val authenticityToken = resAuth . parse ( ) . select ( " meta[name=csrf-token] " ) . attr ( " content " )
2020-10-13 15:56:07 +02:00
val authCookies = resAuth . cookies ( )
2020-10-08 22:20:20 +02:00
2020-10-13 15:56:07 +02:00
Log . i ( javaClass . name , " Received authenticity token: $authenticityToken " )
Log . i ( javaClass . name , " Received authenticity cookies: $authCookies " )
2020-10-08 22:20:20 +02:00
val data = mapOf (
2020-10-11 13:18:20 +02:00
Pair ( " user[login] " , EncryptedPreferences . login ) ,
Pair ( " user[password] " , EncryptedPreferences . password ) ,
2020-10-08 22:20:20 +02:00
Pair ( " user[remember_me] " , " 1 " ) ,
Pair ( " commit " , " Einloggen " ) ,
Pair ( " authenticity_token " , authenticityToken )
)
2020-10-12 20:30:45 +02:00
val resLogin = Jsoup . connect ( baseUrl + loginPath )
2020-10-08 22:20:20 +02:00
. method ( Connection . Method . POST )
. data ( data )
. postDataCharset ( " UTF-8 " )
2020-10-13 15:56:07 +02:00
. cookies ( authCookies )
2020-10-08 22:20:20 +02:00
. execute ( )
//println(resLogin.body())
2020-10-12 23:26:32 +02:00
sessionCookies = resLogin . cookies ( )
2020-10-09 15:18:52 +02:00
loginSuccess = resLogin . body ( ) . contains ( " Hallo, du bist jetzt angemeldet. " )
2020-10-12 23:26:32 +02:00
Log . i ( javaClass . name , " Status: ${resLogin.statusCode()} ( ${resLogin.statusMessage()} ), login successful: $loginSuccess " )
loginSuccess
2020-10-08 22:20:20 +02:00
}
}
2020-10-11 23:16:47 +02:00
/ * *
* list all animes from the website
* /
2020-10-12 17:52:24 +02:00
fun listAnimes ( ) : ArrayList < Media > = runBlocking {
2020-10-09 15:18:52 +02:00
if ( sessionCookies . isEmpty ( ) ) login ( )
2020-10-08 22:20:20 +02:00
2020-10-09 15:18:52 +02:00
withContext ( Dispatchers . Default ) {
2020-10-12 20:30:45 +02:00
val resAnimes = Jsoup . connect ( baseUrl + libraryPath )
2020-10-09 15:18:52 +02:00
. cookies ( sessionCookies )
2020-10-08 22:20:20 +02:00
. get ( )
2020-10-11 10:02:00 +02:00
//println(resAnimes)
2020-10-08 22:20:20 +02:00
2020-10-11 10:02:00 +02:00
mediaList . clear ( )
resAnimes . select ( " div.animebox " ) . forEach {
2020-10-12 17:52:24 +02:00
val type = if ( it . select ( " p.animebox-link " ) . select ( " a " ) . text ( ) . toLowerCase ( Locale . ROOT ) == " zur serie " ) {
MediaType . TVSHOW
} else {
MediaType . MOVIE
}
val media = Media (
2020-10-08 22:20:20 +02:00
it . select ( " h3.animebox-title " ) . text ( ) ,
2020-10-12 17:52:24 +02:00
it . select ( " p.animebox-link " ) . select ( " a " ) . attr ( " href " ) ,
2020-10-13 16:30:23 +02:00
type
2020-10-08 22:20:20 +02:00
)
2020-10-13 16:30:23 +02:00
media . info . posterLink = it . select ( " p.animebox-image " ) . select ( " img " ) . attr ( " src " )
media . info . shortDesc = it . select ( " p.animebox-shorttext " ) . text ( )
2020-10-11 10:02:00 +02:00
mediaList . add ( media )
2020-10-08 22:20:20 +02:00
}
2020-10-12 23:26:32 +02:00
Log . i ( javaClass . name , " Total library size is: ${mediaList.size} " )
2020-10-09 15:18:52 +02:00
2020-10-11 10:02:00 +02:00
return @withContext mediaList
2020-10-09 15:18:52 +02:00
}
}
2020-10-08 22:20:20 +02:00
2020-10-11 10:02:00 +02:00
/ * *
* load streams for the media path
* /
2020-10-12 17:52:24 +02:00
fun loadStreams ( media : Media ) : List < Episode > = runBlocking {
2020-10-09 15:18:52 +02:00
if ( sessionCookies . isEmpty ( ) ) login ( )
if ( ! loginSuccess ) {
2020-10-12 23:26:32 +02:00
Log . w ( javaClass . name , " Login, was not successful. " )
2020-10-12 17:52:24 +02:00
return @runBlocking listOf ( )
2020-10-09 15:18:52 +02:00
}
2020-10-15 18:57:58 +02:00
// if the episodes list is not empty it was loaded before
if ( media . episodes . isNotEmpty ( ) ) {
return @runBlocking media . episodes
}
2020-10-09 15:18:52 +02:00
withContext ( Dispatchers . Default ) {
2020-10-12 20:30:45 +02:00
val res = Jsoup . connect ( baseUrl + media . link )
2020-10-09 15:18:52 +02:00
. cookies ( sessionCookies )
. get ( )
//println(res)
2020-10-13 15:56:07 +02:00
// parse additional info from the media page
2020-10-15 16:23:52 +02:00
res . select ( " table.vertical-table " ) . select ( " tr " ) . forEach { row ->
when ( row . select ( " th " ) . text ( ) . toLowerCase ( Locale . ROOT ) ) {
" produktionsjahr " -> media . info . year = row . select ( " td " ) . text ( ) . toInt ( )
" fsk " -> media . info . age = row . select ( " td " ) . text ( ) . toInt ( )
" episodenanzahl " -> {
media . info . episodesCount = row . select ( " td " ) . text ( )
. substringBefore ( " / " )
. filter { it . isDigit ( ) }
. toInt ( )
}
2020-10-13 15:56:07 +02:00
}
}
2020-10-13 20:23:55 +02:00
2020-10-13 23:47:48 +02:00
// parse additional information for tv shows
2020-10-13 20:23:55 +02:00
val episodes = if ( media . type == MediaType . TVSHOW ) {
res . select ( " div.three-box-container > div.episodebox " ) . map { episodebox ->
val episodeId = episodebox . select ( " div.flip-front " ) . attr ( " id " ) . substringAfter ( " - " ) . toInt ( )
val episodeShortDesc = episodebox . select ( " p.episodebox-shorttext " ) . text ( )
2020-10-13 23:47:48 +02:00
val episodeWatched = episodebox . select ( " div.episodebox-icons > div " ) . hasClass ( " status-icon-orange " )
val episodeWatchedCallback = episodebox . select ( " input.streamstarter_html5 " ) . eachAttr ( " data-playlist " ) . first ( )
Episode (
id = episodeId ,
shortDesc = episodeShortDesc ,
watched = episodeWatched ,
watchedCallback = episodeWatchedCallback
)
2020-10-13 20:23:55 +02:00
}
} else {
listOf ( Episode ( ) )
}
2020-10-13 15:56:07 +02:00
2020-10-13 23:47:48 +02:00
if ( csrfToken . isEmpty ( ) ) {
csrfToken = res . select ( " meta[name=csrf-token] " ) . attr ( " content " )
}
2020-10-13 20:23:55 +02:00
// has attr data-lag (ger or jap)
2020-10-09 15:18:52 +02:00
val playlists = res . select ( " input.streamstarter_html5 " ) . eachAttr ( " data-playlist " )
2020-10-11 10:02:00 +02:00
//println("first entry: ${playlists.first()}")
//println("csrf token is: $csrfToken")
2020-10-12 23:26:32 +02:00
return @withContext if ( playlists . size > 0 ) {
2020-10-13 20:23:55 +02:00
loadStreamInfo ( playlists . first ( ) , csrfToken , media . type , episodes )
2020-10-12 23:26:32 +02:00
} else {
listOf ( )
}
2020-10-09 15:18:52 +02:00
}
}
2020-10-11 10:02:00 +02:00
/ * *
* load the playlist path and parse it , read the stream info from json
2020-10-13 20:23:55 +02:00
* @param episodes is used as call ba reference , additionally it is passed a return value
2020-10-11 10:02:00 +02:00
* /
2020-10-13 20:23:55 +02:00
private fun loadStreamInfo ( playlistPath : String , csrfToken : String , type : MediaType , episodes : List < Episode > ) : List < Episode > = runBlocking {
2020-10-09 15:18:52 +02:00
withContext ( Dispatchers . Default ) {
val headers = mutableMapOf (
Pair ( " Accept " , " application/json, text/javascript, */*; q=0.01 " ) ,
Pair ( " Accept-Language " , " de,en-US;q=0.7,en;q=0.3 " ) ,
Pair ( " Accept-Encoding " , " gzip, deflate, br " ) ,
Pair ( " X-CSRF-Token " , csrfToken ) ,
Pair ( " X-Requested-With " , " XMLHttpRequest " ) ,
)
2020-10-13 23:47:48 +02:00
println ( " loading streaminfo with cstf: $csrfToken " )
2020-10-12 20:30:45 +02:00
val res = Jsoup . connect ( baseUrl + playlistPath )
2020-10-09 15:18:52 +02:00
. ignoreContentType ( true )
. cookies ( sessionCookies )
. headers ( headers )
. execute ( )
//println(res.body())
2020-10-13 20:23:55 +02:00
when ( type ) {
2020-10-11 23:16:47 +02:00
MediaType . MOVIE -> {
val movie = JsonParser . parseString ( res . body ( ) ) . asJsonObject
. get ( " playlist " ) . asJsonArray
2020-10-15 18:51:29 +02:00
. first ( ) . asJsonObject
2020-10-11 23:16:47 +02:00
2020-10-15 18:51:29 +02:00
movie . get ( " sources " ) . asJsonArray . first ( ) . apply {
episodes . first ( ) . streamUrl = this . asJsonObject . get ( " file " ) . asString
2020-10-11 23:16:47 +02:00
}
}
2020-10-13 20:23:55 +02:00
2020-10-11 23:16:47 +02:00
MediaType . TVSHOW -> {
2020-10-12 17:52:24 +02:00
val episodesJson = JsonParser . parseString ( res . body ( ) ) . asJsonObject
2020-10-11 23:16:47 +02:00
. get ( " playlist " ) . asJsonArray
2020-10-13 20:23:55 +02:00
episodesJson . forEach { jsonElement ->
val episodeId = jsonElement . asJsonObject . get ( " mediaid " )
val episodeStream = jsonElement . asJsonObject . get ( " sources " ) . asJsonArray
2020-10-11 23:16:47 +02:00
. first ( ) . asJsonObject
. get ( " file " ) . asString
2020-10-13 20:23:55 +02:00
val episodeTitle = jsonElement . asJsonObject . get ( " title " ) . asString
val episodePoster = jsonElement . asJsonObject . get ( " image " ) . asString
val episodeDescription = jsonElement . asJsonObject . get ( " description " ) . asString
2020-10-13 15:56:07 +02:00
val episodeNumber = episodeTitle . substringAfter ( " , Ep. " ) . toInt ( )
2020-10-12 17:52:24 +02:00
2020-10-13 20:23:55 +02:00
episodes . first { it . id == episodeId . asInt } . apply {
this . title = episodeTitle
this . posterLink = episodePoster
this . streamUrl = episodeStream
this . description = episodeDescription
this . number = episodeNumber
}
2020-10-11 23:16:47 +02:00
}
}
2020-10-13 20:23:55 +02:00
2020-10-11 23:16:47 +02:00
else -> {
Log . e ( javaClass . name , " Wrong Type, please report this issue. " )
}
2020-10-09 15:18:52 +02:00
}
2020-10-13 20:23:55 +02:00
return @withContext episodes
2020-10-08 22:20:20 +02:00
}
}
2020-10-13 23:47:48 +02:00
fun sendCallback ( callbackPath : String ) = GlobalScope . launch {
val headers = mutableMapOf (
Pair ( " Accept " , " application/json, text/javascript, */*; q=0.01 " ) ,
Pair ( " Accept-Language " , " de,en-US;q=0.7,en;q=0.3 " ) ,
Pair ( " Accept-Encoding " , " gzip, deflate, br " ) ,
Pair ( " X-CSRF-Token " , csrfToken ) ,
Pair ( " X-Requested-With " , " XMLHttpRequest " ) ,
)
2020-10-15 16:23:52 +02:00
try {
Jsoup . connect ( baseUrl + callbackPath )
. ignoreContentType ( true )
. cookies ( sessionCookies )
. headers ( headers )
. execute ( )
} catch ( ex : IOException ) {
Log . e ( javaClass . name , " Callback for $callbackPath failed. " )
}
2020-10-13 23:47:48 +02:00
}
2020-10-11 10:02:00 +02:00
}