2020-10-19 19:59:53 +02:00
/ * *
* Teapod
*
* Copyright 2020 < seil0 @mosad . xyz >
*
* This program is free software ; you can redistribute it and / or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation ; either version 3 of the License , or
* ( at your option ) any later version .
*
* This program is distributed in the hope that it will be useful ,
* but WITHOUT ANY WARRANTY ; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
* GNU General Public License for more details .
*
* You should have received a copy of the GNU General Public License
* along with this program ; if not , write to the Free Software
* Foundation , Inc . , 51 Franklin Street , Fifth Floor , Boston ,
* MA 02110 - 1301 , USA .
*
* /
2020-10-08 22:20:20 +02:00
package org.mosad.teapod.parser
2020-10-11 23:16:47 +02:00
import android.util.Log
2020-10-20 20:07:59 +02:00
import com.google.gson.JsonParser
2020-10-09 15:18:52 +02:00
import kotlinx.coroutines.*
2020-10-08 22:20:20 +02:00
import org.jsoup.Connection
import org.jsoup.Jsoup
2020-10-11 13:18:20 +02:00
import org.mosad.teapod.preferences.EncryptedPreferences
2020-10-20 20:07:59 +02:00
import org.mosad.teapod.util.*
2020-10-11 23:16:47 +02:00
import org.mosad.teapod.util.DataTypes.MediaType
2020-10-15 16:23:52 +02:00
import java.io.IOException
2020-10-12 17:52:24 +02:00
import java.util.*
2020-10-08 22:20:20 +02:00
2020-10-19 19:59:53 +02:00
object AoDParser {
2020-10-08 22:20:20 +02:00
2020-10-19 19:59:53 +02:00
private const val baseUrl = " https://www.anime-on-demand.de "
private const val loginPath = " /users/sign_in "
private const val libraryPath = " /animes "
2020-10-08 22:20:20 +02:00
2020-10-19 19:59:53 +02:00
private const val userAgent = " Mozilla/5.0 (X11; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0 "
2020-10-15 18:51:29 +02:00
2020-10-19 19:59:53 +02:00
private var sessionCookies = mutableMapOf < String , String > ( )
private var csrfToken : String = " "
private var loginSuccess = false
2020-10-11 10:02:00 +02:00
2020-12-11 10:54:40 +01:00
private val mediaList = arrayListOf < Media > ( ) // actual media (data)
val itemMediaList = arrayListOf < ItemMedia > ( ) // gui media
2020-12-06 15:18:15 +01:00
val highlightsList = arrayListOf < ItemMedia > ( )
2020-10-19 19:59:53 +02:00
val newEpisodesList = arrayListOf < ItemMedia > ( )
2020-12-11 10:54:40 +01:00
val newSimulcastsList = arrayListOf < ItemMedia > ( )
val newTitlesList = arrayListOf < ItemMedia > ( )
2020-10-08 22:20:20 +02:00
2020-10-12 23:26:32 +02:00
fun login ( ) : Boolean = runBlocking {
2020-10-08 22:20:20 +02:00
withContext ( Dispatchers . Default ) {
// get the authenticity token
2020-10-12 20:30:45 +02:00
val resAuth = Jsoup . connect ( baseUrl + loginPath )
2020-10-08 22:20:20 +02:00
. header ( " User-Agent " , userAgent )
. execute ( )
val authenticityToken = resAuth . parse ( ) . select ( " meta[name=csrf-token] " ) . attr ( " content " )
2020-10-13 15:56:07 +02:00
val authCookies = resAuth . cookies ( )
2020-10-08 22:20:20 +02:00
2020-10-16 18:24:34 +02:00
//Log.d(javaClass.name, "Received authenticity token: $authenticityToken")
//Log.d(javaClass.name, "Received authenticity cookies: $authCookies")
2020-10-08 22:20:20 +02:00
val data = mapOf (
2020-10-11 13:18:20 +02:00
Pair ( " user[login] " , EncryptedPreferences . login ) ,
Pair ( " user[password] " , EncryptedPreferences . password ) ,
2020-10-08 22:20:20 +02:00
Pair ( " user[remember_me] " , " 1 " ) ,
Pair ( " commit " , " Einloggen " ) ,
Pair ( " authenticity_token " , authenticityToken )
)
2020-10-12 20:30:45 +02:00
val resLogin = Jsoup . connect ( baseUrl + loginPath )
2020-10-08 22:20:20 +02:00
. method ( Connection . Method . POST )
2020-11-13 15:36:12 +01:00
. timeout ( 60000 ) // login can take some time
2020-10-08 22:20:20 +02:00
. data ( data )
. postDataCharset ( " UTF-8 " )
2020-10-13 15:56:07 +02:00
. cookies ( authCookies )
2020-10-08 22:20:20 +02:00
. execute ( )
//println(resLogin.body())
2020-10-16 18:24:34 +02:00
2020-10-12 23:26:32 +02:00
sessionCookies = resLogin . cookies ( )
2020-10-09 15:18:52 +02:00
loginSuccess = resLogin . body ( ) . contains ( " Hallo, du bist jetzt angemeldet. " )
2020-10-12 23:26:32 +02:00
Log . i ( javaClass . name , " Status: ${resLogin.statusCode()} ( ${resLogin.statusMessage()} ), login successful: $loginSuccess " )
loginSuccess
2020-10-08 22:20:20 +02:00
}
}
2020-10-11 23:16:47 +02:00
/ * *
2020-10-19 19:59:53 +02:00
* initially load all media and home screen data
* -> blocking
* /
fun initialLoading ( ) = runBlocking {
2020-12-11 10:54:40 +01:00
val loadHomeJob = GlobalScope . async {
2020-12-06 15:18:15 +01:00
loadHome ( )
2020-10-19 19:59:53 +02:00
}
val listJob = GlobalScope . async {
listAnimes ( )
}
2020-12-11 10:54:40 +01:00
loadHomeJob . await ( )
2020-10-19 19:59:53 +02:00
listJob . await ( )
}
/ * *
* get a media by it ' s ID ( int )
* @return Media
2020-10-11 23:16:47 +02:00
* /
2020-11-27 11:06:16 +01:00
suspend fun getMediaById ( mediaId : Int ) : Media {
2020-10-19 19:59:53 +02:00
val media = mediaList . first { it . id == mediaId }
if ( media . episodes . isEmpty ( ) ) {
2020-11-27 11:06:16 +01:00
loadStreams ( media ) . join ( )
2020-10-19 19:59:53 +02:00
}
return media
}
// TODO don't use jsoup here
2020-11-27 11:06:16 +01:00
fun sendCallback ( callbackPath : String ) = GlobalScope . launch ( Dispatchers . IO ) {
2020-10-19 19:59:53 +02:00
val headers = mutableMapOf (
Pair ( " Accept " , " application/json, text/javascript, */*; q=0.01 " ) ,
Pair ( " Accept-Language " , " de,en-US;q=0.7,en;q=0.3 " ) ,
Pair ( " Accept-Encoding " , " gzip, deflate, br " ) ,
Pair ( " X-CSRF-Token " , csrfToken ) ,
Pair ( " X-Requested-With " , " XMLHttpRequest " ) ,
)
try {
2020-11-27 11:06:16 +01:00
Jsoup . connect ( baseUrl + callbackPath )
. ignoreContentType ( true )
. cookies ( sessionCookies )
. headers ( headers )
. execute ( )
2020-10-19 19:59:53 +02:00
} catch ( ex : IOException ) {
Log . e ( javaClass . name , " Callback for $callbackPath failed. " , ex )
}
}
/ * *
* load all media from aod into itemMediaList and mediaList
* /
private fun listAnimes ( ) = runBlocking {
2020-10-09 15:18:52 +02:00
if ( sessionCookies . isEmpty ( ) ) login ( )
2020-10-08 22:20:20 +02:00
2020-10-09 15:18:52 +02:00
withContext ( Dispatchers . Default ) {
2020-10-12 20:30:45 +02:00
val resAnimes = Jsoup . connect ( baseUrl + libraryPath )
2020-10-09 15:18:52 +02:00
. cookies ( sessionCookies )
2020-10-08 22:20:20 +02:00
. get ( )
2020-10-11 10:02:00 +02:00
//println(resAnimes)
2020-10-08 22:20:20 +02:00
2020-10-19 17:34:41 +02:00
itemMediaList . clear ( )
2020-10-11 10:02:00 +02:00
mediaList . clear ( )
resAnimes . select ( " div.animebox " ) . forEach {
2020-10-12 17:52:24 +02:00
val type = if ( it . select ( " p.animebox-link " ) . select ( " a " ) . text ( ) . toLowerCase ( Locale . ROOT ) == " zur serie " ) {
MediaType . TVSHOW
} else {
MediaType . MOVIE
}
2020-10-16 19:56:08 +02:00
val mediaTitle = it . select ( " h3.animebox-title " ) . text ( )
val mediaLink = it . select ( " p.animebox-link " ) . select ( " a " ) . attr ( " href " )
val mediaImage = it . select ( " p.animebox-image " ) . select ( " img " ) . attr ( " src " )
val mediaShortText = it . select ( " p.animebox-shorttext " ) . text ( )
val mediaId = mediaLink . substringAfterLast ( " / " ) . toInt ( )
itemMediaList . add ( ItemMedia ( mediaId , mediaTitle , mediaImage ) )
mediaList . add ( Media ( mediaId , mediaLink , type ) . apply {
info . title = mediaTitle
info . posterUrl = mediaImage
info . shortDesc = mediaShortText
} )
2020-10-08 22:20:20 +02:00
}
2020-10-12 23:26:32 +02:00
Log . i ( javaClass . name , " Total library size is: ${mediaList.size} " )
2020-10-09 15:18:52 +02:00
}
}
2020-10-08 22:20:20 +02:00
2020-10-19 19:59:53 +02:00
/ * *
2020-12-11 10:54:40 +01:00
* load new episodes , titles and highlights
2020-10-19 19:59:53 +02:00
* /
2020-12-06 15:18:15 +01:00
private fun loadHome ( ) = runBlocking {
2020-10-19 17:34:41 +02:00
if ( sessionCookies . isEmpty ( ) ) login ( )
withContext ( Dispatchers . Default ) {
val resHome = Jsoup . connect ( baseUrl )
. cookies ( sessionCookies )
. get ( )
2020-12-06 15:18:15 +01:00
// get highlights from AoD
highlightsList . clear ( )
resHome . select ( " #aod-highlights " ) . select ( " div.news-item " ) . forEach {
val mediaId = it . select ( " div.news-item-text " ) . select ( " a.serienlink " )
2020-12-07 17:50:10 +01:00
. attr ( " href " ) . substringAfterLast ( " / " ) . toIntOrNull ( )
2020-12-06 15:18:15 +01:00
val mediaTitle = it . select ( " div.news-title " ) . select ( " h2 " ) . text ( )
val mediaImage = it . select ( " img " ) . attr ( " src " )
2020-12-07 17:50:10 +01:00
if ( mediaId != null ) {
highlightsList . add ( ItemMedia ( mediaId , mediaTitle , mediaImage ) )
}
2020-12-06 15:18:15 +01:00
}
2020-12-11 10:54:40 +01:00
// get all new episodes from AoD
newEpisodesList . clear ( )
resHome . select ( " h2:contains(Neue Episoden) " ) . next ( ) . select ( " li " ) . forEach {
val mediaId = it . select ( " a.thumbs " ) . attr ( " href " )
. substringAfterLast ( " / " ) . toIntOrNull ( )
val mediaImage = it . select ( " a.thumbs > img " ) . attr ( " src " )
val mediaTitle = " ${it.select("a").text()} - ${it.select("span.neweps").text()} "
if ( mediaId != null ) {
newEpisodesList . add ( ItemMedia ( mediaId , mediaTitle , mediaImage ) )
}
}
// get new simulcasts from AoD
newSimulcastsList . clear ( )
resHome . select ( " h2:contains(Neue Simulcasts) " ) . next ( ) . select ( " li " ) . forEach {
val mediaId = it . select ( " a.thumbs " ) . attr ( " href " )
. substringAfterLast ( " / " ) . toIntOrNull ( )
val mediaImage = it . select ( " a.thumbs > img " ) . attr ( " src " )
val mediaTitle = it . select ( " a " ) . text ( )
if ( mediaId != null ) {
newSimulcastsList . add ( ItemMedia ( mediaId , mediaTitle , mediaImage ) )
}
}
// get new titles from AoD
newTitlesList . clear ( )
resHome . select ( " h2:contains(Neue Anime-Titel) " ) . next ( ) . select ( " li " ) . forEach {
val mediaId = it . select ( " a.thumbs " ) . attr ( " href " )
. substringAfterLast ( " / " ) . toIntOrNull ( )
val mediaImage = it . select ( " a.thumbs > img " ) . attr ( " src " )
val mediaTitle = it . select ( " a " ) . text ( )
if ( mediaId != null ) {
newTitlesList . add ( ItemMedia ( mediaId , mediaTitle , mediaImage ) )
}
}
2020-10-19 17:34:41 +02:00
}
}
2020-10-11 10:02:00 +02:00
/ * *
2020-10-16 19:56:08 +02:00
* load streams for the media path , movies have one episode
* @param media is used as call ba reference
2020-10-11 10:02:00 +02:00
* /
2020-11-27 11:06:16 +01:00
private suspend fun loadStreams ( media : Media ) = GlobalScope . launch ( Dispatchers . IO ) {
2020-10-09 15:18:52 +02:00
if ( sessionCookies . isEmpty ( ) ) login ( )
if ( ! loginSuccess ) {
2020-10-12 23:26:32 +02:00
Log . w ( javaClass . name , " Login, was not successful. " )
2020-11-27 11:06:16 +01:00
return @launch
2020-10-15 18:57:58 +02:00
}
2020-11-27 11:06:16 +01:00
// get the media page
val res = Jsoup . connect ( baseUrl + media . link )
. cookies ( sessionCookies )
. get ( )
2020-10-09 15:18:52 +02:00
2020-11-27 11:06:16 +01:00
//println(res)
2020-10-09 15:18:52 +02:00
2020-11-27 11:06:16 +01:00
if ( csrfToken . isEmpty ( ) ) {
csrfToken = res . select ( " meta[name=csrf-token] " ) . attr ( " content " )
//Log.i(javaClass.name, "New csrf token is $csrfToken")
}
val pl = res . select ( " input.streamstarter_html5 " ) . first ( )
val primary = pl . attr ( " data-playlist " )
val secondary = pl . attr ( " data-otherplaylist " )
val secondaryIsOmU = secondary . contains ( " OmU " , true )
// load primary and secondary playlist
val primaryPlaylist = parsePlaylistAsync ( primary )
val secondaryPlaylist = parsePlaylistAsync ( secondary )
2020-10-09 15:18:52 +02:00
2020-11-27 11:06:16 +01:00
primaryPlaylist . await ( ) . playlist . forEach { ep ->
val epNumber = if ( media . type == MediaType . TVSHOW ) {
ep . title . substringAfter ( " , Ep. " ) . toInt ( )
} else {
0
2020-10-19 21:57:02 +02:00
}
2020-11-27 11:06:16 +01:00
media . episodes . add (
Episode (
id = ep . mediaid ,
priStreamUrl = ep . sources . first ( ) . file ,
posterUrl = ep . image ,
title = ep . title ,
description = ep . description ,
number = epNumber
)
)
}
Log . i ( javaClass . name , " Loading primary playlist finished " )
2020-10-19 21:57:02 +02:00
2020-11-27 11:06:16 +01:00
secondaryPlaylist . await ( ) . playlist . forEach { ep ->
val episode = media . episodes . firstOrNull { it . id == ep . mediaid }
2020-10-19 21:57:02 +02:00
2020-11-27 11:06:16 +01:00
if ( episode != null ) {
episode . secStreamUrl = ep . sources . first ( ) . file
episode . secStreamOmU = secondaryIsOmU
} else {
2020-10-19 22:07:55 +02:00
val epNumber = if ( media . type == MediaType . TVSHOW ) {
ep . title . substringAfter ( " , Ep. " ) . toInt ( )
} else {
0
}
2020-10-19 21:57:02 +02:00
media . episodes . add (
Episode (
id = ep . mediaid ,
2020-11-27 11:06:16 +01:00
secStreamUrl = ep . sources . first ( ) . file ,
secStreamOmU = secondaryIsOmU ,
2020-10-19 21:57:02 +02:00
posterUrl = ep . image ,
title = ep . title ,
description = ep . description ,
2020-10-19 22:07:55 +02:00
number = epNumber
2020-10-19 21:57:02 +02:00
)
)
}
2020-11-27 11:06:16 +01:00
}
Log . i ( javaClass . name , " Loading secondary playlist finished " )
// parse additional info from the media page
res . select ( " table.vertical-table " ) . select ( " tr " ) . forEach { row ->
when ( row . select ( " th " ) . text ( ) . toLowerCase ( Locale . ROOT ) ) {
" produktionsjahr " -> media . info . year = row . select ( " td " ) . text ( ) . toInt ( )
" fsk " -> media . info . age = row . select ( " td " ) . text ( ) . toInt ( )
" episodenanzahl " -> {
media . info . episodesCount = row . select ( " td " ) . text ( )
. substringBefore ( " / " )
. filter { it . isDigit ( ) }
. toInt ( )
2020-10-13 15:56:07 +02:00
}
}
2020-11-27 11:06:16 +01:00
}
2020-10-13 20:23:55 +02:00
2020-12-02 11:14:09 +01:00
// parse additional information for tv shows the episode title (description) is loaded from the "api"
2020-11-27 11:06:16 +01:00
if ( media . type == MediaType . TVSHOW ) {
res . select ( " div.three-box-container > div.episodebox " ) . forEach { episodebox ->
// make sure the episode has a streaming link
if ( episodebox . select ( " input.streamstarter_html5 " ) . isNotEmpty ( ) ) {
val episodeId = episodebox . select ( " div.flip-front " ) . attr ( " id " ) . substringAfter ( " - " ) . toInt ( )
val episodeShortDesc = episodebox . select ( " p.episodebox-shorttext " ) . text ( )
val episodeWatched = episodebox . select ( " div.episodebox-icons > div " ) . hasClass ( " status-icon-orange " )
val episodeWatchedCallback = episodebox . select ( " input.streamstarter_html5 " ) . eachAttr ( " data-playlist " ) . first ( )
media . episodes . firstOrNull { it . id == episodeId } ?. apply {
shortDesc = episodeShortDesc
watched = episodeWatched
watchedCallback = episodeWatchedCallback
2020-10-16 19:56:08 +02:00
}
2020-10-13 20:23:55 +02:00
}
2020-10-13 23:47:48 +02:00
}
2020-10-09 15:18:52 +02:00
}
}
2020-10-20 20:07:59 +02:00
/ * *
* don ' t use Gson ( ) . fromJson ( ) as we don ' t have any control over the api and it may change
* /
2020-10-19 21:57:02 +02:00
private fun parsePlaylistAsync ( playlistPath : String ) : Deferred < AoDObject > {
if ( playlistPath == " [] " ) {
return CompletableDeferred ( AoDObject ( listOf ( ) ) )
}
2020-11-27 11:06:16 +01:00
return GlobalScope . async ( Dispatchers . IO ) {
2020-10-09 15:18:52 +02:00
val headers = mutableMapOf (
Pair ( " Accept " , " application/json, text/javascript, */*; q=0.01 " ) ,
Pair ( " Accept-Language " , " de,en-US;q=0.7,en;q=0.3 " ) ,
Pair ( " Accept-Encoding " , " gzip, deflate, br " ) ,
Pair ( " X-CSRF-Token " , csrfToken ) ,
Pair ( " X-Requested-With " , " XMLHttpRequest " ) ,
)
2020-10-16 18:24:34 +02:00
//println("loading streaminfo with cstf: $csrfToken")
2020-10-13 23:47:48 +02:00
2020-10-12 20:30:45 +02:00
val res = Jsoup . connect ( baseUrl + playlistPath )
2020-10-09 15:18:52 +02:00
. ignoreContentType ( true )
. cookies ( sessionCookies )
. headers ( headers )
. execute ( )
2020-10-20 20:07:59 +02:00
//Gson().fromJson(res.body(), AoDObject::class.java)
return @async AoDObject ( JsonParser . parseString ( res . body ( ) ) . asJsonObject
. get ( " playlist " ) . asJsonArray . map {
Playlist (
sources = it . asJsonObject . get ( " sources " ) . asJsonArray . map { source ->
Source ( source . asJsonObject . get ( " file " ) . asString )
} ,
image = it . asJsonObject . get ( " image " ) . asString ,
title = it . asJsonObject . get ( " title " ) . asString ,
description = it . asJsonObject . get ( " description " ) . asString ,
mediaid = it . asJsonObject . get ( " mediaid " ) . asInt
)
} )
2020-10-08 22:20:20 +02:00
}
}
2020-10-11 10:02:00 +02:00
}