Add support for using HTTP proxies

This commit is contained in:
syeopite 2023-08-25 16:08:02 -07:00
parent eda7444ca4
commit eb8fcc9e88
No known key found for this signature in database
GPG Key ID: A73C186DA3955A1A
6 changed files with 51 additions and 3 deletions

View File

@ -160,6 +160,17 @@ https_only: false
## ##
#force_resolve: #force_resolve:
##
## Configuration for using a HTTP proxy
##
## If unset, then no HTTP proxy will be used.
##
http_proxy:
user:
password:
host:
port:
## ##
## Use Innertube's transcripts API instead of timedtext for closed captions ## Use Innertube's transcripts API instead of timedtext for closed captions

View File

@ -6,11 +6,11 @@ shards:
athena-negotiation: athena-negotiation:
git: https://github.com/athena-framework/negotiation.git git: https://github.com/athena-framework/negotiation.git
version: 0.1.1 version: 0.1.3
backtracer: backtracer:
git: https://github.com/sija/backtracer.cr.git git: https://github.com/sija/backtracer.cr.git
version: 1.2.1 version: 1.2.2
db: db:
git: https://github.com/crystal-lang/crystal-db.git git: https://github.com/crystal-lang/crystal-db.git
@ -20,6 +20,10 @@ shards:
git: https://github.com/crystal-loot/exception_page.git git: https://github.com/crystal-loot/exception_page.git
version: 0.2.2 version: 0.2.2
http_proxy:
git: https://github.com/mamantoha/http_proxy.git
version: 0.10.1
kemal: kemal:
git: https://github.com/kemalcr/kemal.git git: https://github.com/kemalcr/kemal.git
version: 1.1.2 version: 1.1.2
@ -42,7 +46,7 @@ shards:
spectator: spectator:
git: https://github.com/icy-arctic-fox/spectator.git git: https://github.com/icy-arctic-fox/spectator.git
version: 0.10.4 version: 0.10.6
sqlite3: sqlite3:
git: https://github.com/crystal-lang/crystal-sqlite3.git git: https://github.com/crystal-lang/crystal-sqlite3.git

View File

@ -28,6 +28,9 @@ dependencies:
athena-negotiation: athena-negotiation:
github: athena-framework/negotiation github: athena-framework/negotiation
version: ~> 0.1.1 version: ~> 0.1.1
http_proxy:
github: mamantoha/http_proxy
version: ~> 0.10.1
development_dependencies: development_dependencies:
spectator: spectator:

View File

@ -23,6 +23,7 @@ require "kilt"
require "./ext/kemal_content_for.cr" require "./ext/kemal_content_for.cr"
require "./ext/kemal_static_file_handler.cr" require "./ext/kemal_static_file_handler.cr"
require "http_proxy"
require "athena-negotiation" require "athena-negotiation"
require "openssl/hmac" require "openssl/hmac"
require "option_parser" require "option_parser"

View File

@ -54,6 +54,15 @@ struct ConfigPreferences
end end
end end
struct HTTPProxyConfig
include YAML::Serializable
property user : String
property password : String
property host : String
property port : Int32
end
class Config class Config
include YAML::Serializable include YAML::Serializable
@ -126,6 +135,8 @@ class Config
property host_binding : String = "0.0.0.0" property host_binding : String = "0.0.0.0"
# Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property pool_size : Int32 = 100 property pool_size : Int32 = 100
# HTTP Proxy configuration
property http_proxy : HTTPProxyConfig? = nil
# Use Innertube's transcripts API instead of timedtext for closed captions # Use Innertube's transcripts API instead of timedtext for closed captions
property use_innertube_for_captions : Bool = false property use_innertube_for_captions : Bool = false

View File

@ -26,12 +26,14 @@ struct YoutubeConnectionPool
def client(&block) def client(&block)
conn = pool.checkout conn = pool.checkout
conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
begin begin
response = yield conn response = yield conn
rescue ex rescue ex
conn.close conn.close
conn = HTTP::Client.new(url) conn = HTTP::Client.new(url)
conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
conn.family = CONFIG.force_resolve conn.family = CONFIG.force_resolve
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
@ -46,6 +48,7 @@ struct YoutubeConnectionPool
private def build_pool private def build_pool
DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
conn = HTTP::Client.new(url) conn = HTTP::Client.new(url)
conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
conn.family = CONFIG.force_resolve conn.family = CONFIG.force_resolve
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
@ -66,6 +69,8 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false)
client.read_timeout = 10.seconds client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds client.connect_timeout = 10.seconds
client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
return client return client
end end
@ -77,3 +82,16 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &block)
client.close client.close
end end
end end
def make_configured_http_proxy_client
# This method is only called when configuration for an HTTP proxy are set
config_proxy = CONFIG.http_proxy.not_nil!
return HTTP::Proxy::Client.new(
config_proxy.host,
config_proxy.port,
username: config_proxy.user,
password: config_proxy.password,
)
end