2022-07-24 23:30:58 +02:00
enum VideoType
Video
Livestream
Scheduled
end
2019-03-29 22:30:02 +01:00
struct Video
2020-07-26 16:58:50 +02:00
include DB :: Serializable
2022-10-03 21:58:52 +02:00
# Version of the JSON structure
# It prevents us from loading an incompatible version from cache
# (either newer or older, if instances with different versions run
# concurrently, e.g during a version upgrade rollout).
#
# NOTE: don't forget to bump this number if any change is made to
# the `params` structure in videos/parser.cr!!!
#
SCHEMA_VERSION = 2
2020-07-26 16:58:50 +02:00
property id : String
@[ DB :: Field ( converter : Video :: JSONConverter ) ]
property info : Hash ( String , JSON :: Any )
property updated : Time
@[ DB :: Field ( ignore : true ) ]
2023-08-25 01:00:02 +02:00
@captions = [ ] of Invidious :: Videos :: Captions :: Metadata
2020-07-26 16:58:50 +02:00
@[ DB :: Field ( ignore : true ) ]
property adaptive_fmts : Array ( Hash ( String , JSON :: Any ) ) ?
@[ DB :: Field ( ignore : true ) ]
property fmt_stream : Array ( Hash ( String , JSON :: Any ) ) ?
@[ DB :: Field ( ignore : true ) ]
property description : String ?
2020-06-16 00:33:23 +02:00
module JSONConverter
2018-08-04 22:30:44 +02:00
def self . from_rs ( rs )
2020-06-16 00:33:23 +02:00
JSON . parse ( rs . read ( String ) ) . as_h
2018-08-04 22:30:44 +02:00
end
end
2022-08-23 19:03:09 +02:00
# Methods for API v1 JSON
2019-04-11 00:58:42 +02:00
2022-08-23 19:03:09 +02:00
def to_json ( locale : String ?, json : JSON :: Builder )
Invidious :: JSONify :: APIv1 . video ( self , json , locale : locale )
2019-04-11 00:58:42 +02:00
end
2021-10-29 14:53:06 +02:00
# TODO: remove the locale and follow the crystal convention
2021-11-08 23:52:55 +01:00
def to_json ( locale : String ?, _json : Nil )
2022-08-23 19:03:09 +02:00
JSON . build do | json |
Invidious :: JSONify :: APIv1 . video ( self , json , locale : locale )
end
2021-10-29 14:53:06 +02:00
end
def to_json ( json : JSON :: Builder | Nil = nil )
to_json ( nil , json )
2019-06-08 20:31:41 +02:00
end
2022-08-23 19:03:09 +02:00
# Misc methods
2022-07-24 23:30:58 +02:00
def video_type : VideoType
video_type = info [ " videoType " ]? . try & . as_s || " video "
return VideoType . parse? ( video_type ) || VideoType :: Video
2019-03-22 16:32:42 +01:00
end
2022-10-03 21:58:52 +02:00
def schema_version : Int
return info [ " version " ]? . try & . as_i || 1
end
2020-06-16 00:33:23 +02:00
def published : Time
2022-07-24 23:30:58 +02:00
return info [ " published " ]?
2022-01-20 22:22:48 +01:00
. try { | t | Time . parse ( t . as_s , " %Y-%m-%d " , Time :: Location :: UTC ) } || Time . utc
2020-06-16 00:33:23 +02:00
end
2019-03-22 16:32:42 +01:00
2020-06-16 00:33:23 +02:00
def published = ( other : Time )
2022-07-24 23:30:58 +02:00
info [ " published " ] = JSON :: Any . new ( other . to_s ( " %Y-%m-%d " ) )
2020-06-16 00:33:23 +02:00
end
2019-03-22 17:06:58 +01:00
2020-06-16 00:33:23 +02:00
def live_now
2022-07-24 23:30:58 +02:00
return ( self . video_type == VideoType :: Livestream )
2020-06-16 00:33:23 +02:00
end
2019-03-22 18:24:47 +01:00
2020-06-16 00:33:23 +02:00
def premiere_timestamp : Time ?
2022-01-20 22:22:48 +01:00
info
. dig? ( " microformat " , " playerMicroformatRenderer " , " liveBroadcastDetails " , " startTimestamp " )
. try { | t | Time . parse_rfc3339 ( t . as_s ) }
2019-03-22 17:06:58 +01:00
end
2020-06-16 00:33:23 +02:00
def related_videos
info [ " relatedVideos " ]? . try & . as_a . map { | h | h . as_h . transform_values & . as_s } || [ ] of Hash ( String , String )
end
2019-02-26 00:28:35 +01:00
2022-07-24 23:30:58 +02:00
# Methods for parsing streaming data
2019-02-26 00:28:35 +01:00
2020-06-16 00:33:23 +02:00
def fmt_stream
return @fmt_stream . as ( Array ( Hash ( String , JSON :: Any ) ) ) if @fmt_stream
2020-07-26 16:58:50 +02:00
2020-06-16 00:33:23 +02:00
fmt_stream = info [ " streamingData " ]? . try & . [ " formats " ]? . try & . as_a . map & . as_h || [ ] of Hash ( String , JSON :: Any )
fmt_stream . each do | fmt |
if s = ( fmt [ " cipher " ]? || fmt [ " signatureCipher " ]? ) . try { | h | HTTP :: Params . parse ( h . as_s ) }
s . each do | k , v |
fmt [ k ] = JSON :: Any . new ( v )
2019-02-26 00:28:35 +01:00
end
2020-09-27 19:19:44 +02:00
fmt [ " url " ] = JSON :: Any . new ( " #{ fmt [ " url " ] } #{ DECRYPT_FUNCTION . decrypt_signature ( fmt ) } " )
2018-08-05 06:07:38 +02:00
end
2020-06-16 00:33:23 +02:00
fmt [ " url " ] = JSON :: Any . new ( " #{ fmt [ " url " ] } &host= #{ URI . parse ( fmt [ " url " ] . as_s ) . host } " )
fmt [ " url " ] = JSON :: Any . new ( " #{ fmt [ " url " ] } ®ion= #{ self . info [ " region " ] } " ) if self . info [ " region " ]?
2018-08-05 06:07:38 +02:00
end
2022-04-27 00:20:48 +02:00
2020-06-16 00:33:23 +02:00
fmt_stream . sort_by! { | f | f [ " width " ]? . try & . as_i || 0 }
@fmt_stream = fmt_stream
return @fmt_stream . as ( Array ( Hash ( String , JSON :: Any ) ) )
2018-08-05 06:07:38 +02:00
end
2020-06-16 00:33:23 +02:00
def adaptive_fmts
return @adaptive_fmts . as ( Array ( Hash ( String , JSON :: Any ) ) ) if @adaptive_fmts
fmt_stream = info [ " streamingData " ]? . try & . [ " adaptiveFormats " ]? . try & . as_a . map & . as_h || [ ] of Hash ( String , JSON :: Any )
fmt_stream . each do | fmt |
if s = ( fmt [ " cipher " ]? || fmt [ " signatureCipher " ]? ) . try { | h | HTTP :: Params . parse ( h . as_s ) }
s . each do | k , v |
fmt [ k ] = JSON :: Any . new ( v )
2018-09-13 05:31:47 +02:00
end
2020-09-27 19:19:44 +02:00
fmt [ " url " ] = JSON :: Any . new ( " #{ fmt [ " url " ] } #{ DECRYPT_FUNCTION . decrypt_signature ( fmt ) } " )
2018-09-13 05:31:47 +02:00
end
2018-08-05 06:07:38 +02:00
2020-06-16 00:33:23 +02:00
fmt [ " url " ] = JSON :: Any . new ( " #{ fmt [ " url " ] } &host= #{ URI . parse ( fmt [ " url " ] . as_s ) . host } " )
fmt [ " url " ] = JSON :: Any . new ( " #{ fmt [ " url " ] } ®ion= #{ self . info [ " region " ] } " ) if self . info [ " region " ]?
2018-10-02 02:01:44 +02:00
end
2022-04-27 00:20:48 +02:00
2020-06-16 00:33:23 +02:00
fmt_stream . sort_by! { | f | f [ " width " ]? . try & . as_i || 0 }
@adaptive_fmts = fmt_stream
return @adaptive_fmts . as ( Array ( Hash ( String , JSON :: Any ) ) )
2018-08-05 06:07:38 +02:00
end
2020-06-16 00:33:23 +02:00
def video_streams
adaptive_fmts . select & . [ " mimeType " ]? . try & . as_s . starts_with? ( " video " )
2018-08-05 06:07:38 +02:00
end
2020-06-16 00:33:23 +02:00
def audio_streams
adaptive_fmts . select & . [ " mimeType " ]? . try & . as_s . starts_with? ( " audio " )
2018-08-18 18:47:16 +02:00
end
2022-07-24 23:30:58 +02:00
# Misc. methods
2019-04-12 00:00:00 +02:00
def storyboards
2022-01-20 22:22:48 +01:00
storyboards = info . dig? ( " storyboards " , " playerStoryboardSpecRenderer " , " spec " )
. try & . as_s . split ( " | " )
2019-04-12 00:00:00 +02:00
if ! storyboards
2022-01-20 22:22:48 +01:00
if storyboard = info . dig? ( " storyboards " , " playerLiveStoryboardSpecRenderer " , " spec " ) . try & . as_s
2019-04-12 00:00:00 +02:00
return [ {
2019-04-18 23:23:50 +02:00
url : storyboard . split ( " # " ) [ 0 ] ,
width : 106 ,
height : 60 ,
count : - 1 ,
interval : 5000 ,
storyboard_width : 3 ,
storyboard_height : 3 ,
storyboard_count : - 1 ,
} ]
2019-04-12 00:00:00 +02:00
end
end
items = [ ] of NamedTuple (
url : String ,
width : Int32 ,
height : Int32 ,
count : Int32 ,
interval : Int32 ,
storyboard_width : Int32 ,
storyboard_height : Int32 ,
storyboard_count : Int32 )
2020-06-16 00:33:23 +02:00
return items if ! storyboards
2019-04-12 00:00:00 +02:00
2019-05-27 01:55:22 +02:00
url = URI . parse ( storyboards . shift )
params = HTTP :: Params . parse ( url . query || " " )
2019-04-12 00:00:00 +02:00
2022-01-20 17:17:22 +01:00
storyboards . each_with_index do | sb , i |
width , height , count , storyboard_width , storyboard_height , interval , _ , sigh = sb . split ( " # " )
2019-05-27 01:55:22 +02:00
params [ " sigh " ] = sigh
url . query = params . to_s
2019-04-12 00:00:00 +02:00
width = width . to_i
height = height . to_i
count = count . to_i
interval = interval . to_i
storyboard_width = storyboard_width . to_i
storyboard_height = storyboard_height . to_i
2019-10-04 16:23:02 +02:00
storyboard_count = ( count / ( storyboard_width * storyboard_height ) ) . ceil . to_i
2019-04-12 00:00:00 +02:00
items << {
2019-05-27 01:55:22 +02:00
url : url . to_s . sub ( " $L " , i ) . sub ( " $N " , " M$M " ) ,
2019-04-12 00:00:00 +02:00
width : width ,
height : height ,
count : count ,
interval : interval ,
storyboard_width : storyboard_width ,
storyboard_height : storyboard_height ,
2019-10-04 16:23:02 +02:00
storyboard_count : storyboard_count ,
2019-04-12 00:00:00 +02:00
}
end
items
end
2021-08-15 10:38:30 +02:00
def paid
2022-07-24 23:30:58 +02:00
return ( self . reason || " " ) . includes? " requires payment "
2021-08-15 10:38:30 +02:00
end
2018-10-16 18:15:14 +02:00
def premium
2020-06-16 00:33:23 +02:00
keywords . includes? " YouTube Red "
end
2023-08-25 01:00:02 +02:00
def captions : Array ( Invidious :: Videos :: Captions :: Metadata )
2022-05-23 22:37:58 +02:00
if @captions . empty? && @info . has_key? ( " captions " )
2023-08-25 01:00:02 +02:00
@captions = Invidious :: Videos :: Captions :: Metadata . from_yt_json ( info [ " captions " ] )
2019-08-05 03:56:24 +02:00
end
2022-05-23 22:37:58 +02:00
return @captions
2018-10-16 18:15:14 +02:00
end
2020-06-16 00:33:23 +02:00
def hls_manifest_url : String ?
2022-01-20 22:22:48 +01:00
info . dig? ( " streamingData " , " hlsManifestUrl " ) . try & . as_s
2020-06-16 00:33:23 +02:00
end
2023-10-22 17:09:52 +02:00
def dash_manifest_url : String ?
raw_dash_url = info . dig? ( " streamingData " , " dashManifestUrl " ) . try & . as_s
return nil if raw_dash_url . nil?
# Use manifest v5 parameter to reduce file size
# See https://github.com/iv-org/invidious/issues/4186
dash_url = URI . parse ( raw_dash_url )
dash_query = dash_url . query || " "
if dash_query . empty?
dash_url . path = " #{ dash_url . path } /mpd_version/5 "
else
dash_url . query = " #{ dash_query } &mpd_version=5 "
end
return dash_url . to_s
2020-06-16 00:33:23 +02:00
end
2020-06-17 00:51:49 +02:00
def genre_url : String ?
info [ " genreUcid " ]? ? " /channel/ #{ info [ " genreUcid " ] } " : nil
2020-06-16 00:33:23 +02:00
end
2021-08-12 21:26:50 +02:00
def is_vr : Bool ?
2022-07-24 23:30:58 +02:00
return { " EQUIRECTANGULAR " , " MESH " } . includes? self . projection_type
2021-09-10 09:42:39 +02:00
end
def projection_type : String ?
return info . dig? ( " streamingData " , " adaptiveFormats " , 0 , " projectionType " ) . try & . as_s
2021-04-11 15:09:10 +02:00
end
2020-06-16 00:33:23 +02:00
def reason : String ?
info [ " reason " ]? . try & . as_s
end
2022-07-24 23:30:58 +02:00
2023-01-16 13:58:05 +01:00
def music : Array ( VideoMusic )
2023-01-22 00:12:04 +01:00
info [ " music " ] . as_a . map { | music_json |
2023-03-07 20:23:08 +01:00
VideoMusic . new (
music_json [ " song " ] . as_s ,
music_json [ " album " ] . as_s ,
music_json [ " artist " ] . as_s ,
music_json [ " license " ] . as_s
)
2023-01-22 00:12:04 +01:00
}
2023-01-16 13:58:05 +01:00
end
2022-07-24 23:30:58 +02:00
# Macros defining getters/setters for various types of data
private macro getset_string ( name )
# Return {{name.stringify}} from `info`
def {{ name . id . underscore }} : String
return info [ {{ name . stringify }} ]? . try & . as_s || " "
end
# Update {{name.stringify}} into `info`
def {{ name . id . underscore }} = ( value : String )
info [ {{ name . stringify }} ] = JSON :: Any . new ( value )
end
{% if flag? ( :debug_macros ) %} {{ debug }} {% end %}
end
private macro getset_string_array ( name )
# Return {{name.stringify}} from `info`
def {{ name . id . underscore }} : Array ( String )
return info [ {{ name . stringify }} ]? . try & . as_a . map & . as_s || [ ] of String
end
# Update {{name.stringify}} into `info`
def {{ name . id . underscore }} = ( value : Array ( String ) )
info [ {{ name . stringify }} ] = JSON :: Any . new ( value )
end
{% if flag? ( :debug_macros ) %} {{ debug }} {% end %}
end
{% for op , type in { i32 : Int32 , i64 : Int64 } %}
private macro getset_ {{ op }} ( name )
def \ {{ name . id . underscore }} : {{ type }}
2022-11-16 18:18:35 +01:00
return info [ \ {{ name . stringify }} ]? . try & . as_i64 . to_ {{ op }} || 0 _ {{ op }}
2022-07-24 23:30:58 +02:00
end
def \ {{ name . id . underscore }} = ( value : Int )
info [ \ {{ name . stringify }} ] = JSON :: Any . new ( value . to_i64 )
end
\ {% if flag? ( :debug_macros ) %} \ {{ debug }} \ {% end %}
end
{% end %}
private macro getset_bool ( name )
# Return {{name.stringify}} from `info`
def {{ name . id . underscore }} : Bool
return info [ {{ name . stringify }} ]? . try & . as_bool || false
end
# Update {{name.stringify}} into `info`
def {{ name . id . underscore }} = ( value : Bool )
info [ {{ name . stringify }} ] = JSON :: Any . new ( value )
end
{% if flag? ( :debug_macros ) %} {{ debug }} {% end %}
end
# Method definitions, using the macros above
getset_string author
getset_string authorThumbnail
getset_string description
getset_string descriptionHtml
getset_string genre
getset_string genreUcid
getset_string license
getset_string shortDescription
getset_string subCountText
getset_string title
getset_string ucid
getset_string_array allowedRegions
getset_string_array keywords
getset_i32 lengthSeconds
getset_i64 likes
getset_i64 views
getset_bool allowRatings
getset_bool authorVerified
getset_bool isFamilyFriendly
getset_bool isListed
getset_bool isUpcoming
2020-07-26 16:58:50 +02:00
end
2018-10-21 03:37:55 +02:00
2021-12-07 02:55:43 +01:00
def get_video ( id , refresh = true , region = nil , force_refresh = false )
2021-11-26 19:36:31 +01:00
if ( video = Invidious :: Database :: Videos . select ( id ) ) && ! region
2020-06-16 00:33:23 +02:00
# If record was last updated over 10 minutes ago, or video has since premiered,
# refresh (expire param in response lasts for 6 hours)
if ( refresh &&
( Time . utc - video . updated > 10 . minutes ) ||
( video . premiere_timestamp . try & . < Time . utc ) ) ||
2022-10-03 21:58:52 +02:00
force_refresh ||
video . schema_version != Video :: SCHEMA_VERSION # cache control
2020-06-16 00:33:23 +02:00
begin
video = fetch_video ( id , region )
2021-11-26 19:36:31 +01:00
Invidious :: Database :: Videos . update ( video )
2020-06-16 00:33:23 +02:00
rescue ex
2021-11-26 19:36:31 +01:00
Invidious :: Database :: Videos . delete ( id )
2020-06-16 00:33:23 +02:00
raise ex
end
2019-02-06 23:12:11 +01:00
end
else
2020-06-16 00:33:23 +02:00
video = fetch_video ( id , region )
2021-11-26 19:36:31 +01:00
Invidious :: Database :: Videos . insert ( video ) if ! region
2018-08-04 22:30:44 +02:00
end
2020-06-16 00:33:23 +02:00
return video
2022-04-08 22:52:34 +02:00
rescue DB :: Error
# Avoid common `DB::PoolRetryAttemptsExceeded` error and friends
# Note: All DB errors inherit from `DB::Error`
return fetch_video ( id , region )
2019-02-06 23:12:11 +01:00
end
2019-06-29 04:17:56 +02:00
def fetch_video ( id , region )
2021-08-13 22:29:43 +02:00
info = extract_video_info ( video_id : id )
2018-10-07 05:22:22 +02:00
2021-08-13 22:29:43 +02:00
allowed_regions = info
. dig? ( " microformat " , " playerMicroformatRenderer " , " availableCountries " )
. try & . as_a . map & . as_s || [ ] of String
2019-08-13 22:21:00 +02:00
# Check for region-blocks
2020-06-16 00:33:23 +02:00
if info [ " reason " ]? . try & . as_s . includes? ( " your country " )
2019-08-19 16:00:37 +02:00
bypass_regions = PROXY_LIST . keys & allowed_regions
if ! bypass_regions . empty?
region = bypass_regions [ rand ( bypass_regions . size ) ]
2021-08-13 22:29:43 +02:00
region_info = extract_video_info ( video_id : id , proxy_region : region )
2020-06-16 00:33:23 +02:00
region_info [ " region " ] = JSON :: Any . new ( region ) if region
info = region_info if ! region_info [ " reason " ]?
2019-08-19 16:00:37 +02:00
end
2018-08-13 16:17:28 +02:00
end
2022-02-11 06:43:14 +01:00
if reason = info [ " reason " ]?
2022-05-27 15:36:13 +02:00
if reason == " Video unavailable "
raise NotFoundException . new ( reason . as_s || " " )
2023-06-07 15:55:09 +02:00
elsif ! reason . as_s . starts_with? " Premieres "
# dont error when it's a premiere.
# we already parsed most of the data and display the premiere date
2022-05-27 15:36:13 +02:00
raise InfoException . new ( reason . as_s || " " )
end
2022-02-11 06:43:14 +01:00
end
2018-08-04 22:30:44 +02:00
2020-07-26 16:58:50 +02:00
video = Video . new ( {
id : id ,
info : info ,
updated : Time . utc ,
} )
2018-08-04 22:30:44 +02:00
return video
end
2021-12-07 02:55:43 +01:00
def process_continuation ( query , plid , id )
2019-08-06 01:49:13 +02:00
continuation = nil
if plid
if index = query [ " index " ]? . try & . to_i?
continuation = index
else
continuation = id
end
continuation || = 0
end
continuation
end
2020-06-16 00:10:30 +02:00
def build_thumbnails ( id )
2019-03-08 21:42:37 +01:00
return {
2021-04-01 02:23:59 +02:00
{ host : HOST_URL , height : 720 , width : 1280 , name : " maxres " , url : " maxres " } ,
{ host : HOST_URL , height : 720 , width : 1280 , name : " maxresdefault " , url : " maxresdefault " } ,
{ host : HOST_URL , height : 480 , width : 640 , name : " sddefault " , url : " sddefault " } ,
{ host : HOST_URL , height : 360 , width : 480 , name : " high " , url : " hqdefault " } ,
{ host : HOST_URL , height : 180 , width : 320 , name : " medium " , url : " mqdefault " } ,
{ host : HOST_URL , height : 90 , width : 120 , name : " default " , url : " default " } ,
{ host : HOST_URL , height : 90 , width : 120 , name : " start " , url : " 1 " } ,
{ host : HOST_URL , height : 90 , width : 120 , name : " middle " , url : " 2 " } ,
{ host : HOST_URL , height : 90 , width : 120 , name : " end " , url : " 3 " } ,
2019-03-08 21:42:37 +01:00
}
end