mirror/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2025-03-09 12:50:23 -05:00
Code Activity

Cleanup code

Browse source
This commit is contained in:
Michaël De Boey 2025-03-04 23:53:02 +01:00
parent f66affa65b
commit 5386daa073
No known key found for this signature in database
1 changed files with 150 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

262
yt_dlp/extractor/vrt.py
View file

@ -90,24 +90,36 @@ def _extract_formats_and_subtitles(self, data, video_id):
def _call_api(self, video_id, client='null', id_token=None, version='v2'): def _call_api(self, video_id, client='null', id_token=None, version='v2'):
player_info = {'exp': (round(time.time(), 3) + 900), **self._PLAYER_INFO} player_info = {'exp': (round(time.time(), 3) + 900), **self._PLAYER_INFO}
player_token = self._download_json( vrt_player_token = self._download_json(
'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v2/tokens', f'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/{version}/tokens',
video_id, 'Downloading player token', headers={ video_id,
'Downloading player token',
'Failed to download player token',
headers={
**self.geo_verification_headers(), **self.geo_verification_headers(),
'Content-Type': 'application/json', 'Content-Type': 'application/json',
}, data=json.dumps({ },
data=json.dumps({
'identityToken': id_token or '', 'identityToken': id_token or '',
'playerInfo': jwt_encode_hs256(player_info, self._JWT_SIGNING_KEY, headers={ 'playerInfo': jwt_encode_hs256(
'kid': self._JWT_KEY_ID, player_info,
}).decode(), self._JWT_SIGNING_KEY,
}, separators=(',', ':')).encode())['vrtPlayerToken'] headers={'kid': self._JWT_KEY_ID},
).decode(),
}, separators=(',', ':')).encode(),
)['vrtPlayerToken']
return self._download_json( return self._download_json(
f'https://media-services-public.vrt.be/media-aggregator/{version}/media-items/{video_id}', f'https://media-services-public.vrt.be/media-aggregator/{version}/media-items/{video_id}',
video_id, 'Downloading API JSON', query={ video_id,
'vrtPlayerToken': player_token, 'Downloading API JSON',
'Failed to download API JSON',
query={
'client': client, 'client': client,
}, expected_status=400) 'vrtPlayerToken': vrt_player_token,
},
expected_status=400,
)
class VRTIE(VRTBaseIE): class VRTIE(VRTBaseIE):
@ -177,48 +189,52 @@ def _real_extract(self, url):
} }
def jwt_is_expired(token):
return jwt_decode_hs256(token)['exp'] - time.time() < 300
class VrtNUIE(VRTBaseIE): class VrtNUIE(VRTBaseIE):
IE_DESC = 'VRT MAX' IE_DESC = 'VRT MAX'
_VALID_URL = r'https?://(?:www\.)?vrt\.be/(?:vrtnu|vrtmax)/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)' _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?:vrtnu|vrtmax)/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
_TESTS = [{ _TESTS = [{
'url': 'https://www.vrt.be/vrtmax/a-z/ket---doc/trailer/ket---doc-trailer-s6/', 'url': 'https://www.vrt.be/vrtmax/a-z/ket---doc/trailer/ket---doc-trailer-s6/',
'info_dict': { 'info_dict': {
'id': 'pbs-pub-c8a78645-5d3e-468a-89ec-6f3ed5534bd5$vid-242ddfe9-18f5-4e16-ab45-09b122a19251',
'ext': 'mp4',
'display_id': 'ket---doc-trailer-s6',
'title': 'Reeks 6 volledig vanaf 3 maart',
'description': 'Neem een kijkje in de bijzondere wereld van deze Ketnetters.',
'channel': 'VRT', 'channel': 'VRT',
'description': 'Neem een kijkje in de bijzondere wereld van deze Ketnetters.',
'display_id': 'ket---doc-trailer-s6',
'duration': 30.0, 'duration': 30.0,
'thumbnail': 'https://images.vrt.be/orig/2025/02/21/63f07122-5bbd-4ca1-b42e-8565c6cd95df.jpg',
'series': 'Ket & Doc',
'season': 'Trailer',
'season_id': '1739450401467',
'episode': 'Reeks 6 volledig vanaf 3 maart', 'episode': 'Reeks 6 volledig vanaf 3 maart',
'episode_id': '1739450401467', 'episode_id': '1739450401467',
'ext': 'mp4',
'id': 'pbs-pub-c8a78645-5d3e-468a-89ec-6f3ed5534bd5$vid-242ddfe9-18f5-4e16-ab45-09b122a19251',
'season': 'Trailer',
'season_id': '1739450401467',
'series': 'Ket & Doc',
'thumbnail': 'https://images.vrt.be/orig/2025/02/21/63f07122-5bbd-4ca1-b42e-8565c6cd95df.jpg',
'timestamp': 1740373200, 'timestamp': 1740373200,
'title': 'Reeks 6 volledig vanaf 3 maart',
'upload_date': '20250224', 'upload_date': '20250224',
'_old_archive_ids': ['canvas pbs-pub-c8a78645-5d3e-468a-89ec-6f3ed5534bd5$vid-242ddfe9-18f5-4e16-ab45-09b122a19251'], '_old_archive_ids': ['canvas pbs-pub-c8a78645-5d3e-468a-89ec-6f3ed5534bd5$vid-242ddfe9-18f5-4e16-ab45-09b122a19251'],
}, },
}, { }, {
'url': 'https://www.vrt.be/vrtnu/a-z/taboe/3/taboe-s3a4/', 'url': 'https://www.vrt.be/vrtnu/a-z/taboe/3/taboe-s3a4/',
'info_dict': { 'info_dict': {
'id': 'pbs-pub-f50faa3a-1778-46b6-9117-4ba85f197703$vid-547507fe-1c8b-4394-b361-21e627cbd0fd',
'ext': 'mp4',
'display_id': 'taboe-s3a4',
'title': 'Mensen met het syndroom van Gilles de la Tourette',
'description': 'md5:bf61345a95eca9393a95de4a7a54b5c6',
'channel': 'VRT', 'channel': 'VRT',
'description': 'md5:bf61345a95eca9393a95de4a7a54b5c6',
'display_id': 'taboe-s3a4',
'duration': 2882.02, 'duration': 2882.02,
'thumbnail': 'https://images.vrt.be/orig/2025/02/19/8198496c-d1ae-4bca-9a48-761cf3ea3ff2.jpg', 'ext': 'mp4',
'series': 'Taboe',
'season': '3',
'season_number': 3,
'season_id': '1739055911734',
'episode': 'Mensen met het syndroom van Gilles de la Tourette', 'episode': 'Mensen met het syndroom van Gilles de la Tourette',
'episode_number': 4,
'episode_id': '1739055911734', 'episode_id': '1739055911734',
'episode_number': 4,
'id': 'pbs-pub-f50faa3a-1778-46b6-9117-4ba85f197703$vid-547507fe-1c8b-4394-b361-21e627cbd0fd',
'season': '3',
'season_id': '1739055911734',
'season_number': 3,
'series': 'Taboe',
'thumbnail': 'https://images.vrt.be/orig/2025/02/19/8198496c-d1ae-4bca-9a48-761cf3ea3ff2.jpg',
'timestamp': 1740286800, 'timestamp': 1740286800,
'title': 'Mensen met het syndroom van Gilles de la Tourette',
'upload_date': '20250223', 'upload_date': '20250223',
'_old_archive_ids': ['canvas pbs-pub-f50faa3a-1778-46b6-9117-4ba85f197703$vid-547507fe-1c8b-4394-b361-21e627cbd0fd'], '_old_archive_ids': ['canvas pbs-pub-f50faa3a-1778-46b6-9117-4ba85f197703$vid-547507fe-1c8b-4394-b361-21e627cbd0fd'],
}, },
@ -226,71 +242,62 @@ class VrtNUIE(VRTBaseIE):
_NETRC_MACHINE = 'vrtnu' _NETRC_MACHINE = 'vrtnu'
_VIDEO_PAGE_QUERY = ''' _VIDEO_PAGE_QUERY = '''
query VideoPage($pageId: ID!) { query VideoPage($pageId: ID!) {
page(id: $pageId) { page(id: $pageId) {
... on EpisodePage { ... on EpisodePage {
ldjson episode {
episode { ageRaw
id description
title durationRaw
onTimeRaw episodeNumberRaw
ageRaw id
description name
durationRaw onTimeRaw
name program {
episodeNumberRaw title
season { }
id season {
titleRaw id
titleRaw
}
title
}
ldjson
player {
image {
templateUrl
}
modes {
streamId
}
}
}
} }
program {
title
}
}
player {
image {
templateUrl
}
modes {
streamId
}
}
} }
} '''
}
'''
def _get_video_token_cookie(self):
return try_call(lambda: self._get_cookies('https://www.vrt.be')['vrtnu-site_profile_vt'].value)
def _get_refresh_token_cookie(self):
return try_call(lambda: self._get_cookies('https://www.vrt.be/vrtmax/sso')['vrtnu-site_profile_rt'].value)
def _is_jwt_expired(self, token):
return jwt_decode_hs256(token)['exp'] - time.time() < 300
def _fetch_refresh_token(self): def _fetch_refresh_token(self):
refresh_token = self._get_refresh_token_cookie() refresh_token = self._get_refresh_token_from_cookie()
if refresh_token and not self._is_jwt_expired(refresh_token): if refresh_token and not jwt_is_expired(refresh_token):
return refresh_token return refresh_token
if not self._get_login_info()[0]: if not self._get_login_info()[0]:
return return
refresh_token = self.cache.load(self._NETRC_MACHINE, 'refresh_token', default=None) refresh_token = self.cache.load(self._NETRC_MACHINE, 'refresh_token', default=None)
if refresh_token and not self._is_jwt_expired(refresh_token): if refresh_token and not jwt_is_expired(refresh_token):
self.write_debug('Restored refresh token from cache') self.write_debug('Restored refresh token from cache')
self._set_cookie('.www.vrt.be', 'vrtnu-site_profile_rt', refresh_token, path='/vrtmax/sso') self._set_cookie('.www.vrt.be', 'vrtnu-site_profile_rt', refresh_token, path='/vrtmax/sso')
return refresh_token return refresh_token
def _fetch_video_token(self): def _fetch_video_token(self):
video_token = self._get_video_token_cookie() video_token = self._get_video_token_from_cookie()
if video_token and not self._is_jwt_expired(video_token): if video_token and not jwt_is_expired(video_token):
return video_token return video_token
if self._get_login_info()[0]: if self._get_login_info()[0]:
video_token = self.cache.load(self._NETRC_MACHINE, 'video_token', default=None) video_token = self.cache.load(self._NETRC_MACHINE, 'video_token', default=None)
if video_token and not self._is_jwt_expired(video_token): if video_token and not jwt_is_expired(video_token):
self.write_debug('Restored video token from cache') self.write_debug('Restored video token from cache')
self._set_cookie('.www.vrt.be', 'vrtnu-site_profile_vt', video_token) self._set_cookie('.www.vrt.be', 'vrtnu-site_profile_vt', video_token)
return video_token return video_token
@ -300,10 +307,13 @@ def _fetch_video_token(self):
return None return None
self._download_webpage( self._download_webpage(
'https://www.vrt.be/vrtmax/sso/login', None, 'https://www.vrt.be/vrtmax/sso/login',
'Refreshing video token', query={'scope': 'openid,mid'}) None,
'Refreshing video token',
query={'scope': 'openid,mid'},
)
video_token = self._get_video_token_cookie() video_token = self._get_video_token_from_cookie()
if not video_token: if not video_token:
self.cache.store(self._NETRC_MACHINE, 'refresh_token', None) self.cache.store(self._NETRC_MACHINE, 'refresh_token', None)
self.report_warning('Refreshing of video token failed') self.report_warning('Refreshing of video token failed')
@ -312,53 +322,77 @@ def _fetch_video_token(self):
self.cache.store(self._NETRC_MACHINE, 'video_token', video_token) self.cache.store(self._NETRC_MACHINE, 'video_token', video_token)
return video_token return video_token
def _get_video_token_from_cookie(self):
return try_call(lambda: self._get_cookies('https://www.vrt.be')['vrtnu-site_profile_vt'].value)
def _get_refresh_token_from_cookie(self):
return try_call(lambda: self._get_cookies('https://www.vrt.be/vrtmax/sso')['vrtnu-site_profile_rt'].value)
def _perform_login(self, username, password): def _perform_login(self, username, password):
if self._fetch_refresh_token(): if self._fetch_refresh_token():
self.write_debug('Refresh token already present, skipping login') self.write_debug('Refresh token already present, skipping login')
return return
self._request_webpage( self._request_webpage(
'https://www.vrt.be/vrtmax/sso/login', None, 'https://www.vrt.be/vrtmax/sso/login',
note='Getting session cookies', errnote='Failed to get session cookies') None,
note='Getting session cookies',
errnote='Failed to get session cookies',
)
login_data = self._download_json( login_data = self._download_json(
'https://login.vrt.be/perform_login', None, data=json.dumps({ 'https://login.vrt.be/perform_login',
None,
data=json.dumps({
'clientId': 'vrtnu-site',
'loginID': username, 'loginID': username,
'password': password, 'password': password,
'clientId': 'vrtnu-site', }).encode(),
}).encode(), headers={ headers={
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Oidcxsrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value, 'Oidcxsrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
}, note='Logging in', errnote='Login failed', expected_status=403) },
note='Logging in',
errnote='Login failed',
expected_status=403,
)
if login_data.get('errorCode'): if login_data.get('errorCode'):
raise ExtractorError( raise ExtractorError(f'Login failed: {login_data.get("errorMessage")}', expected=True)
f'Login failed: {login_data.get("errorMessage")}', expected=True)
self._download_webpage( self._download_webpage(
login_data['redirectUrl'], None, note='Getting access token', errnote='Failed to get access token') login_data['redirectUrl'],
None,
note='Getting access token',
errnote='Failed to get access token',
)
self.cache.store(self._NETRC_MACHINE, 'video_token', self._get_video_token_cookie()) self.cache.store(self._NETRC_MACHINE, 'video_token', self._get_video_token_from_cookie())
self.cache.store(self._NETRC_MACHINE, 'refresh_token', self._get_refresh_token_cookie()) self.cache.store(self._NETRC_MACHINE, 'refresh_token', self._get_refresh_token_from_cookie())
def _real_extract(self, url): def _real_extract(self, url):
display_id = self._match_id(url) display_id = self._match_id(url)
metadata = self._download_json( metadata = self._download_json(
'https://www.vrt.be/vrtnu-api/graphql/public/v1', 'https://www.vrt.be/vrtnu-api/graphql/public/v1',
display_id, 'Downloading asset JSON', 'Unable to download asset JSON', display_id,
'Downloading asset JSON',
'Unable to download asset JSON',
data=json.dumps({ data=json.dumps({
'operationName': 'VideoPage', 'operationName': 'VideoPage',
'query': self._VIDEO_PAGE_QUERY, 'query': self._VIDEO_PAGE_QUERY,
'variables': {'pageId': urllib.parse.urlparse(url).path}, 'variables': {
}).encode(), headers={ 'pageId': urllib.parse.urlparse(url).path,
},
}).encode(),
headers={
'content-type': 'application/json', 'content-type': 'application/json',
'x-vrt-client-name': 'WEB', 'x-vrt-client-name': 'WEB',
'x-vrt-client-version': '1.5.9', 'x-vrt-client-version': '1.5.9',
'x-vrt-zone': 'default', 'x-vrt-zone': 'default',
})['data']['page'] },
)['data']['page']
video_id = metadata['player']['modes'][0]['streamId'] video_id = metadata['player']['modes'][0]['streamId']
video_token = self._fetch_video_token() video_token = self._fetch_video_token()
try: try:
@ -380,28 +414,32 @@ def _real_extract(self, url):
raise ExtractorError(f'Unable to extract formats: {code}') raise ExtractorError(f'Unable to extract formats: {code}')
return { return {
**self._json_ld(traverse_obj(metadata, ('ldjson', ..., {json.loads})), video_id, fatal=False), **self._json_ld(
traverse_obj(metadata, ('ldjson', ..., {json.loads})),
video_id,
fatal=False,
),
**traverse_obj(metadata, ('episode', { **traverse_obj(metadata, ('episode', {
'title': ('title', {str}),
'description': ('description', {str}),
'timestamp': ('onTimeRaw', {parse_iso8601}),
'duration': ('durationRaw', {parse_duration}),
'series': ('program', 'title', {str}),
'season': ('season', 'titleRaw', {str}),
'season_number': ('season', 'titleRaw', {int_or_none}),
'season_id': ('id', {str_or_none}),
'episode': ('title', {str}),
'episode_number': ('episodeNumberRaw', {int_or_none}),
'episode_id': ('id', {str_or_none}),
'age_limit': ('ageRaw', {parse_age_limit}), 'age_limit': ('ageRaw', {parse_age_limit}),
'description': ('description', {str}),
'duration': ('durationRaw', {parse_duration}),
'episode': ('title', {str}),
'episode_id': ('id', {str_or_none}),
'episode_number': ('episodeNumberRaw', {int_or_none}),
'season': ('season', 'titleRaw', {str}),
'season_id': ('id', {str_or_none}),
'season_number': ('season', 'titleRaw', {int_or_none}),
'series': ('program', 'title', {str}),
'timestamp': ('onTimeRaw', {parse_iso8601}),
'title': ('title', {str}),
})), })),
'id': video_id,
'display_id': display_id,
'channel': 'VRT', 'channel': 'VRT',
'formats': formats, 'display_id': display_id,
'duration': float_or_none(streaming_info.get('duration'), 1000), 'duration': float_or_none(streaming_info.get('duration'), 1000),
'thumbnail': url_or_none(streaming_info.get('posterImageUrl')), 'formats': formats,
'id': video_id,
'subtitles': subtitles, 'subtitles': subtitles,
'thumbnail': url_or_none(streaming_info.get('posterImageUrl')),
'_old_archive_ids': [make_archive_id('Canvas', video_id)], '_old_archive_ids': [make_archive_id('Canvas', video_id)],
} }