From 7f3ddad12edc409b3b39fc47e66c5439165fcaa8 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 26 Apr 2024 22:03:59 +0200 Subject: [PATCH 1/4] Videos: Use android test suite client --- src/invidious/videos/parser.cr | 8 +++----- src/invidious/yt_backend/youtube_api.cr | 15 +++++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 75fe4a36..373f7227 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -107,7 +107,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) # decrypted URLs and maybe fix throttling issues (#2194). See the # following issue for an explanation about decrypted URLs: # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - client_config.client_type = YoutubeAPI::ClientType::Android + client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite new_player_response = try_fetch_streaming_data(video_id, client_config) elsif !reason.includes?("your country") # Handled separately # The Android embedded client could help here @@ -142,9 +142,7 @@ end def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") - # CgIIAdgDAQ%3D%3D is a workaround for streaming URLs that returns a 403. - # https://github.com/LuanRT/YouTube.js/pull/624 - response = YoutubeAPI.player(video_id: id, params: "CgIIAdgDAQ%3D%3D", client_config: client_config) + response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config) playability_status = response["playabilityStatus"]["status"] LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") @@ -152,7 +150,7 @@ def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConf if id != response.dig("videoDetails", "videoId") # YouTube may return a different video player response than expected. # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 - raise VideoNotAvailableException.new( + raise InfoException.new( "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)" ) elsif playability_status == "OK" diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 9e0631f6..05ccffac 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -8,16 +8,16 @@ module YoutubeAPI private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history - private ANDROID_APP_VERSION = "19.09.36" - private ANDROID_USER_AGENT = "com.google.android.youtube/19.09.36 (Linux; U; Android 12; US) gzip" + private ANDROID_APP_VERSION = "19.14.42" + private ANDROID_USER_AGENT = "com.google.android.youtube/19.14.42 (Linux; U; Android 12; US) gzip" private ANDROID_SDK_VERSION = 31_i64 private ANDROID_VERSION = "12" # For Apple device names, see https://gist.github.com/adamawolf/3048717 # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases, # then go to the dedicated article of the major version you want. - private IOS_APP_VERSION = "19.09.3" - private IOS_USER_AGENT = "com.google.ios.youtube/19.09.3 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)" + private IOS_APP_VERSION = "19.16.3" + private IOS_USER_AGENT = "com.google.ios.youtube/19.16.3 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)" private IOS_VERSION = "17.4.0.21E219" # Major.Minor.Patch.Build private WINDOWS_VERSION = "10.0" @@ -32,6 +32,7 @@ module YoutubeAPI Android AndroidEmbeddedPlayer AndroidScreenEmbed + AndroidTestSuite IOS IOSEmbedded @@ -114,6 +115,12 @@ module YoutubeAPI os_version: ANDROID_VERSION, platform: "MOBILE", }, + ClientType::AndroidTestSuite => { + name: "ANDROID_TESTSUITE", + name_proto: "30", + version: "1.9", + api_key: DEFAULT_API_KEY, + }, # IOS From d49c76260992f210c09ca2294bdd5e8f55732247 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 26 Apr 2024 22:26:45 +0200 Subject: [PATCH 2/4] YtAPI: Add more client infos for Android test suite --- src/invidious/yt_backend/youtube_api.cr | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 05ccffac..bc4b90ac 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -6,6 +6,7 @@ module YoutubeAPI extend self private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" + private ANDROID_API_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w" # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history private ANDROID_APP_VERSION = "19.14.42" @@ -13,6 +14,9 @@ module YoutubeAPI private ANDROID_SDK_VERSION = 31_i64 private ANDROID_VERSION = "12" + private ANDROID_TS_APP_VERSION = "1.9" + private ANDROID_TS_USER_AGENT = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip" + # For Apple device names, see https://gist.github.com/adamawolf/3048717 # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases, # then go to the dedicated article of the major version you want. @@ -90,7 +94,7 @@ module YoutubeAPI name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", + api_key: ANDROID_API_KEY, android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, os_name: "Android", @@ -116,10 +120,15 @@ module YoutubeAPI platform: "MOBILE", }, ClientType::AndroidTestSuite => { - name: "ANDROID_TESTSUITE", - name_proto: "30", - version: "1.9", - api_key: DEFAULT_API_KEY, + name: "ANDROID_TESTSUITE", + name_proto: "30", + version: ANDROID_TS_APP_VERSION, + api_key: ANDROID_API_KEY, + android_sdk_version: ANDROID_SDK_VERSION, + user_agent: ANDROID_TS_USER_AGENT, + os_name: "Android", + os_version: ANDROID_VERSION, + platform: "MOBILE", }, # IOS From be291e8f0f217c059ba35b414df4446bc00c3f27 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 26 Apr 2024 22:33:08 +0200 Subject: [PATCH 3/4] Videos: Copy captions over between responses --- src/invidious/videos/parser.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 373f7227..ca7fb38d 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -123,8 +123,9 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) # Replace player response and reset reason if !new_player_response.nil? - # Preserve storyboard data before replacement + # Preserve captions & storyboard data before replacement new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? + new_player_response["captions"] = player_response["captions"] if player_response["captions"]? player_response = new_player_response params.delete("reason") From 33f316c864d1bd79cd07d46db9da57a65124f31c Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 26 Apr 2024 23:15:34 +0200 Subject: [PATCH 4/4] Videos: Remove AndroidScreenEmbed client --- src/invidious/videos/parser.cr | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index ca7fb38d..3982c3ff 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -109,10 +109,6 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite new_player_response = try_fetch_streaming_data(video_id, client_config) - elsif !reason.includes?("your country") # Handled separately - # The Android embedded client could help here - client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed - new_player_response = try_fetch_streaming_data(video_id, client_config) end # Last hope