/** * Teapod * * Copyright 2020-2021 * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, * MA 02110-1301, USA. * */ package org.mosad.teapod.parser import android.util.Log import com.google.gson.JsonParser import kotlinx.coroutines.* import org.jsoup.Connection import org.jsoup.Jsoup import org.mosad.teapod.preferences.EncryptedPreferences import org.mosad.teapod.util.* import org.mosad.teapod.util.DataTypes.MediaType import java.io.IOException import java.lang.NumberFormatException import java.util.* import kotlin.random.Random object AoDParser { private const val baseUrl = "https://www.anime-on-demand.de" private const val loginPath = "/users/sign_in" private const val libraryPath = "/animes" private const val subscriptionPath = "/mypools" private const val userAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0" private var sessionCookies = mutableMapOf() private var csrfToken: String = "" private var loginSuccess = false private val mediaList = arrayListOf() // actual media (data) TODO remove private val aodMediaList = arrayListOf() // actual media (data) // gui media val guiMediaList = arrayListOf() val highlightsList = arrayListOf() val newEpisodesList = arrayListOf() val newSimulcastsList = arrayListOf() val newTitlesList = arrayListOf() val topTenList = arrayListOf() fun login(): Boolean = runBlocking { withContext(Dispatchers.IO) { // get the authenticity token val resAuth = Jsoup.connect(baseUrl + loginPath) .header("User-Agent", userAgent) .execute() val authenticityToken = resAuth.parse().select("meta[name=csrf-token]").attr("content") val authCookies = resAuth.cookies() //Log.d(javaClass.name, "Received authenticity token: $authenticityToken") //Log.d(javaClass.name, "Received authenticity cookies: $authCookies") val data = mapOf( Pair("user[login]", EncryptedPreferences.login), Pair("user[password]", EncryptedPreferences.password), Pair("user[remember_me]", "1"), Pair("commit", "Einloggen"), Pair("authenticity_token", authenticityToken) ) val resLogin = Jsoup.connect(baseUrl + loginPath) .method(Connection.Method.POST) .timeout(60000) // login can take some time default is 60000 (60 sec) .data(data) .postDataCharset("UTF-8") .cookies(authCookies) .execute() //println(resLogin.body()) sessionCookies = resLogin.cookies() loginSuccess = resLogin.body().contains("Hallo, du bist jetzt angemeldet.") Log.i(javaClass.name, "Status: ${resLogin.statusCode()} (${resLogin.statusMessage()}), login successful: $loginSuccess") loginSuccess } } /** * initially load all media and home screen data */ suspend fun initialLoading() { coroutineScope { launch { loadHome() } launch { listAnimes() } } } /** * get a media by it's ID (int) * @return Media */ @Deprecated(message = "Use getMediaById2() instead") suspend fun getMediaById(aodId: Int): Media { val media = mediaList.first { it.id == aodId } if (media.episodes.isEmpty()) { loadStreams(media).join() loadMediaAsync(media.id).await() } return media } /** * get a media by it's ID (int) * @param aodId The AoD ID of the requested media * @return returns a AoDMedia of type Movie or TVShow if found, else return AoDMediaNone */ suspend fun getMediaById2(aodId: Int): AoDMedia { return aodMediaList.firstOrNull { it.aodId == aodId } ?: try { loadMediaAsync(aodId).await().apply { aodMediaList.add(this) } } catch (exn:NullPointerException) { Log.e(javaClass.name, "Error while loading media $aodId", exn) AoDMediaNone } } /** * get subscription info from aod website, remove "Anime-Abo" Prefix and trim */ suspend fun getSubscriptionInfoAsync(): Deferred { return coroutineScope { async(Dispatchers.IO) { val res = Jsoup.connect(baseUrl + subscriptionPath) .cookies(sessionCookies) .get() return@async res.select("a:contains(Anime-Abo)").text() .removePrefix("Anime-Abo").trim() } } } fun getSubscriptionUrl(): String { return baseUrl + subscriptionPath } suspend fun markAsWatched(mediaId: Int, episodeId: Int) { val episode = getMediaById(mediaId).getEpisodeById(episodeId) episode.watched = true sendCallback(episode.watchedCallback) Log.d(javaClass.name, "Marked episode ${episode.id} as watched") } // TODO don't use jsoup here private suspend fun sendCallback(callbackPath: String) = coroutineScope { launch(Dispatchers.IO) { val headers = mutableMapOf( Pair("Accept", "application/json, text/javascript, */*; q=0.01"), Pair("Accept-Language", "de,en-US;q=0.7,en;q=0.3"), Pair("Accept-Encoding", "gzip, deflate, br"), Pair("X-CSRF-Token", csrfToken), Pair("X-Requested-With", "XMLHttpRequest"), ) try { Jsoup.connect(baseUrl + callbackPath) .ignoreContentType(true) .cookies(sessionCookies) .headers(headers) .execute() } catch (ex: IOException) { Log.e(javaClass.name, "Callback for $callbackPath failed.", ex) } } } /** * load all media from aod into itemMediaList and mediaList * TODO private suspend fun listAnimes() = withContext(Dispatchers.IO) should also work, maybe a bug in android studio? */ private suspend fun listAnimes() = withContext(Dispatchers.IO) { launch(Dispatchers.IO) { val resAnimes = Jsoup.connect(baseUrl + libraryPath).get() //println(resAnimes) guiMediaList.clear() mediaList.clear() val animes = resAnimes.select("div.animebox") guiMediaList.addAll( animes.map { ItemMedia( id = it.select("p.animebox-link").select("a") .attr("href").substringAfterLast("/").toInt(), title = it.select("h3.animebox-title").text(), posterUrl = it.select("p.animebox-image").select("img") .attr("src") ) } ) // TODO legacy resAnimes.select("div.animebox").forEach { val id = it.select("p.animebox-link").select("a").attr("href") .substringAfterLast("/").toInt() val title = it.select("h3.animebox-title").text() val image = it.select("p.animebox-image").select("img").attr("src") val link = it.select("p.animebox-link").select("a").attr("href") val type = when (it.select("p.animebox-link").select("a").text().lowercase(Locale.ROOT)) { "zur serie" -> MediaType.TVSHOW "zum film" -> MediaType.MOVIE else -> MediaType.OTHER } val mediaShortText = it.select("p.animebox-shorttext").text() mediaList.add(Media(id, link, type).apply { info.title = title info.posterUrl = image info.shortDesc = mediaShortText }) } Log.i(javaClass.name, "Total library size is: ${mediaList.size}") } } /** * load new episodes, titles and highlights */ private suspend fun loadHome() = withContext(Dispatchers.IO) { launch(Dispatchers.IO) { val resHome = Jsoup.connect(baseUrl).get() // get highlights from AoD highlightsList.clear() resHome.select("#aod-highlights").select("div.news-item").forEach { val mediaId = it.select("div.news-item-text").select("a.serienlink") .attr("href").substringAfterLast("/").toIntOrNull() val mediaTitle = it.select("div.news-title").select("h2").text() val mediaImage = it.select("img").attr("src") if (mediaId != null) { highlightsList.add(ItemMedia(mediaId, mediaTitle, mediaImage)) } } // get all new episodes from AoD newEpisodesList.clear() resHome.select("h2:contains(Neue Episoden)").next().select("li").forEach { val mediaId = it.select("a.thumbs").attr("href") .substringAfterLast("/").toIntOrNull() val mediaImage = it.select("a.thumbs > img").attr("src") val mediaTitle = "${it.select("a").text()} - ${it.select("span.neweps").text()}" if (mediaId != null) { newEpisodesList.add(ItemMedia(mediaId, mediaTitle, mediaImage)) } } // get new simulcasts from AoD newSimulcastsList.clear() resHome.select("h2:contains(Neue Simulcasts)").next().select("li").forEach { val mediaId = it.select("a.thumbs").attr("href") .substringAfterLast("/").toIntOrNull() val mediaImage = it.select("a.thumbs > img").attr("src") val mediaTitle = it.select("a").text() if (mediaId != null) { newSimulcastsList.add(ItemMedia(mediaId, mediaTitle, mediaImage)) } } // get new titles from AoD newTitlesList.clear() resHome.select("h2:contains(Neue Anime-Titel)").next().select("li").forEach { val mediaId = it.select("a.thumbs").attr("href") .substringAfterLast("/").toIntOrNull() val mediaImage = it.select("a.thumbs > img").attr("src") val mediaTitle = it.select("a").text() if (mediaId != null) { newTitlesList.add(ItemMedia(mediaId, mediaTitle, mediaImage)) } } // get top ten from AoD topTenList.clear() resHome.select("h2:contains(Anime Top 10)").next().select("li").forEach { val mediaId = it.select("a.thumbs").attr("href") .substringAfterLast("/").toIntOrNull() val mediaImage = it.select("a.thumbs > img").attr("src") val mediaTitle = it.select("a").text() if (mediaId != null) { topTenList.add(ItemMedia(mediaId, mediaTitle, mediaImage)) } } // if highlights is empty, add a random new title if (highlightsList.isEmpty()) { if (newTitlesList.isNotEmpty()) { highlightsList.add(newTitlesList[Random.nextInt(0, newTitlesList.size)]) } else { highlightsList.add(ItemMedia(0,"", "")) } } Log.i(javaClass.name, "loaded home") } } /** * TODO rework the media loading process, don't modify media object * TODO catch SocketTimeoutException from loading to show a waring dialog * load streams for the media path, movies have one episode * @param media is used as call ba reference */ private suspend fun loadStreams(media: Media) = coroutineScope { launch(Dispatchers.IO) { if (sessionCookies.isEmpty()) login() if (!loginSuccess) { Log.w(javaClass.name, "Login, was not successful.") return@launch } // get the media page val res = Jsoup.connect(baseUrl + media.link) .cookies(sessionCookies) .get() //println(res) if (csrfToken.isEmpty()) { csrfToken = res.select("meta[name=csrf-token]").attr("content") //Log.i(javaClass.name, "New csrf token is $csrfToken") } val besides = res.select("div.besides").first() val playlists = besides.select("input.streamstarter_html5").map { streamstarter -> parsePlaylistAsync( streamstarter.attr("data-playlist"), streamstarter.attr("data-lang") ) }.awaitAll() playlists.forEach { aod -> aod.list.forEach { ep -> try { if (media.hasEpisode(ep.mediaid)) { media.getEpisodeById(ep.mediaid).streams.add( Stream(ep.sources.first().file, aod.language) ) } else { media.episodes.add(Episode( id = ep.mediaid, streams = mutableListOf(Stream(ep.sources.first().file, aod.language)), posterUrl = ep.image, title = ep.title, description = ep.description, number = getNumberFromTitle(ep.title, media.type) )) } } catch (ex: Exception) { Log.w(javaClass.name, "Could not parse episode information.", ex) } } } Log.i(javaClass.name, "Loaded playlists successfully") // additional info from the media page res.select("table.vertical-table").select("tr").forEach { row -> when (row.select("th").text().lowercase(Locale.ROOT)) { "produktionsjahr" -> media.info.year = row.select("td").text().toInt() "fsk" -> media.info.age = row.select("td").text().toInt() "episodenanzahl" -> { media.info.episodesCount = row.select("td").text() .substringBefore("/") .filter { it.isDigit() } .toInt() } } } // similar titles from media page media.info.similar = res.select("h2:contains(Ähnliche Animes)").next().select("li").mapNotNull { val mediaId = it.select("a.thumbs").attr("href") .substringAfterLast("/").toIntOrNull() val mediaImage = it.select("a.thumbs > img").attr("src") val mediaTitle = it.select("a").text() if (mediaId != null) { ItemMedia(mediaId, mediaTitle, mediaImage) } else { null } } // additional information for tv shows the episode title (description) is loaded from the "api" if (media.type == MediaType.TVSHOW) { res.select("div.three-box-container > div.episodebox").forEach { episodebox -> // make sure the episode has a streaming link if (episodebox.select("input.streamstarter_html5").isNotEmpty()) { val episodeId = episodebox.select("div.flip-front").attr("id").substringAfter("-").toInt() val episodeShortDesc = episodebox.select("p.episodebox-shorttext").text() val episodeWatched = episodebox.select("div.episodebox-icons > div").hasClass("status-icon-orange") val episodeWatchedCallback = episodebox.select("input.streamstarter_html5").eachAttr("data-playlist").first() media.episodes.firstOrNull { it.id == episodeId }?.apply { shortDesc = episodeShortDesc watched = episodeWatched watchedCallback = episodeWatchedCallback } } } } Log.i(javaClass.name, "media loaded successfully") } } private suspend fun loadMediaAsync(aodId: Int): Deferred = coroutineScope { return@coroutineScope async (Dispatchers.IO) { if (sessionCookies.isEmpty()) login() // TODO is this needed? // return none object, if login wasn't successful if (!loginSuccess) { Log.w(javaClass.name, "Login, was not successful.") return@async AoDMediaNone } // get the media page val res = Jsoup.connect("$baseUrl/anime/$aodId") .cookies(sessionCookies) .get() // println(res) if (csrfToken.isEmpty()) { csrfToken = res.select("meta[name=csrf-token]").attr("content") Log.d(javaClass.name, "New csrf token is $csrfToken") } // playlist parsing TODO can this be async to the genral info marsing? val besides = res.select("div.besides").first() val aodPlaylists = besides.select("input.streamstarter_html5").map { streamstarter -> parsePlaylistAsync( streamstarter.attr("data-playlist"), streamstarter.attr("data-lang") ) } /** * generic aod media data */ val title = res.select("h1[itemprop=name]").text() val description = res.select("div[itemprop=description]").text() val posterURL = res.select("img.fullwidth-image").attr("src") val type = when { posterURL.contains("films") -> MediaType.MOVIE posterURL.contains("series") -> MediaType.TVSHOW else -> MediaType.OTHER } var year = 0 var age = 0 res.select("table.vertical-table").select("tr").forEach { row -> when (row.select("th").text().lowercase(Locale.ROOT)) { "produktionsjahr" -> year = row.select("td").text().toInt() "fsk" -> age = row.select("td").text().toInt() } } // similar titles from media page val similar = res.select("h2:contains(Ähnliche Animes)").next().select("li").mapNotNull { val mediaId = it.select("a.thumbs").attr("href") .substringAfterLast("/").toIntOrNull() val mediaImage = it.select("a.thumbs > img").attr("src") val mediaTitle = it.select("a").text() if (mediaId != null) { ItemMedia(mediaId, mediaTitle, mediaImage) } else { null } } /** * additional information for episodes: * description: a short description of the episode * watched: indicates if the episodes has been watched * watched callback: url to set watched in aod */ val episodesInfo: Map = if (type == MediaType.TVSHOW) { res.select("div.three-box-container > div.episodebox").mapNotNull { episodeBox -> // make sure the episode has a streaming link if (episodeBox.select("input.streamstarter_html5").isNotEmpty()) { val mediaId = episodeBox.select("div.flip-front").attr("id").substringAfter("-").toInt() val episodeShortDesc = episodeBox.select("p.episodebox-shorttext").text() val episodeWatched = episodeBox.select("div.episodebox-icons > div").hasClass("status-icon-orange") val episodeWatchedCallback = episodeBox.select("input.streamstarter_html5").eachAttr("data-playlist").first() AoDEpisodeInfo(mediaId, episodeShortDesc, episodeWatched, episodeWatchedCallback) } else { null } }.associateBy { it.aodMediaId } } else { mapOf() } // TODO make AoDPlaylist to teapod playlist val playlist: List = aodPlaylists.awaitAll().flatMap { aodPlaylist -> aodPlaylist.list.mapIndexed { index, episode -> AoDEpisode( mediaId = episode.mediaid, title = episode.title, description = episode.description, shortDesc = episodesInfo[episode.mediaid]?.shortDesc ?: "", imageURL = episode.image, number = index, watched = episodesInfo[episode.mediaid]?.watched ?: false, watchedCallback = episodesInfo[episode.mediaid]?.watchedCallback ?: "", streams = mutableListOf(Stream(episode.sources.first().file, aodPlaylist.language)) ) } }.groupingBy { it.mediaId }.reduce{ _, accumulator, element -> accumulator.copy().also { it.streams.addAll(element.streams) } }.values.toList() println("new playlist object: $playlist") return@async AoDMedia( aodId = aodId, type = type, title = title, shortText = description, posterURL = posterURL, year = year, age = age, similar = similar, playlist = playlist ) } } /** * don't use Gson().fromJson() as we don't have any control over the api and it may change */ private fun parsePlaylistAsync(playlistPath: String, language: String): Deferred { if (playlistPath == "[]") { return CompletableDeferred(AoDPlaylist(listOf(), Locale.ROOT)) } return CoroutineScope(Dispatchers.IO).async(Dispatchers.IO) { val headers = mutableMapOf( Pair("Accept", "application/json, text/javascript, */*; q=0.01"), Pair("Accept-Language", "de,en-US;q=0.7,en;q=0.3"), Pair("Accept-Encoding", "gzip, deflate, br"), Pair("X-CSRF-Token", csrfToken), Pair("X-Requested-With", "XMLHttpRequest"), ) //println("loading streaminfo with cstf: $csrfToken") val res = Jsoup.connect(baseUrl + playlistPath) .ignoreContentType(true) .cookies(sessionCookies) .headers(headers) .timeout(120000) // loading the playlist can take some time .execute() //Gson().fromJson(res.body(), AoDObject::class.java) return@async AoDPlaylist(JsonParser.parseString(res.body()).asJsonObject .get("playlist").asJsonArray.map { Playlist( sources = it.asJsonObject.get("sources").asJsonArray.map { source -> Source(source.asJsonObject.get("file").asString) }, image = it.asJsonObject.get("image").asString, title = it.asJsonObject.get("title").asString, description = it.asJsonObject.get("description").asString, mediaid = it.asJsonObject.get("mediaid").asInt ) }, // TODO improve language handling (via display language etc.) language = when (language) { "ger" -> Locale.GERMAN "jap" -> Locale.JAPANESE else -> Locale.ROOT } ) } } /** * get the episode number from the title * @param title the episode title, containing a number after "Ep." * @param type the media type, if not TVSHOW, return 0 * @return the episode number, on NumberFormatException return 0 */ private fun getNumberFromTitle(title: String, type: MediaType): Int { return if (type == MediaType.TVSHOW) { try { title.substringAfter(", Ep. ").toInt() } catch (nex: NumberFormatException) { 0 } } else { 0 } } }