mirror/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2025-03-09 12:50:23 -05:00
Code Activity

[VRT] Update VRT MAX extractor

Browse source
This commit is contained in:
Michaël De Boey 2025-02-26 04:17:41 +01:00
parent 0bb3978862
commit cfd3f5ba80
No known key found for this signature in database
3 changed files with 240 additions and 153 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
yt_dlp/extractor/_extractors.py
View file

@ -2397,10 +2397,10 @@
)
from .vrt import (
VRTIE,
VRTMAXIE,
DagelijkseKostIE,
KetnetIE,
Radio1BeIE,
VrtNUIE,
)
from .vtm import VTMIE
from .vtv import (

19
yt_dlp/extractor/gigya.py
View file

@ -1,19 +0,0 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
urlencode_postdata,
)
class GigyaBaseIE(InfoExtractor):
def _gigya_login(self, auth_data):
auth_info = self._download_json(
'https://accounts.eu1.gigya.com/accounts.login', None,
note='Logging in', errnote='Unable to log in',
data=urlencode_postdata(auth_data))
error_message = auth_info.get('errorDetails') or auth_info.get('errorMessage')
if error_message:
raise ExtractorError(
f'Unable to login: {error_message}', expected=True)
return auth_info

372
yt_dlp/extractor/vrt.py
View file

@ -1,18 +1,16 @@
import json
import time
import urllib.parse
import urllib.request
from .gigya import GigyaBaseIE
from ..networking.exceptions import HTTPError
from .common import InfoExtractor
from ..utils import (
ExtractorError,
clean_html,
extract_attributes,
float_or_none,
get_element_by_class,
get_element_html_by_class,
int_or_none,
join_nonempty,
jwt_encode_hs256,
make_archive_id,
merge_dicts,
@ -21,12 +19,12 @@
str_or_none,
strip_or_none,
traverse_obj,
unified_strdate,
url_or_none,
urlencode_postdata,
)
class VRTBaseIE(GigyaBaseIE):
class VRTBaseIE(InfoExtractor):
_GEO_BYPASS = False
_PLAYER_INFO = {
'platform': 'desktop',
@ -109,6 +107,118 @@ def _call_api(self, video_id, client='null', id_token=None, version='v2'):
}, expected_status=400)
class VRTMAXBaseIE(InfoExtractor):
_GEO_BYPASS = False
def _call_api(self, video_id, client='null', id_token=None, version='v2'):
vrt_player_token = self._download_json(
f'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/{version}/tokens',
None,
'Downloading player token',
'Failed to download player token',
headers={
'Content-Type': 'application/json',
},
data=json.dumps({
'identityToken': id_token or self._get_identity_token_from_cookie(),
}).encode(),
)['vrtPlayerToken']
return self._download_json(
f'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/{version}/videos/{video_id}',
video_id,
'Downloading API JSON',
'Failed to download API JSON',
query={
'client': client,
'vrtPlayerToken': vrt_player_token,
},
)
def _extract_formats_and_subtitles(self, data, video_id):
# probably needs an extra check against `drmExpired`
if traverse_obj(data, 'drm'):
self.report_drm(video_id)
formats, subtitles = [], {}
for target in traverse_obj(
data, ('targetUrls', lambda _, v: url_or_none(v['url']) and v['type']),
):
format_type = target['type'].upper()
format_url = target['url']
if format_type in ('HLS', 'HLS_AES'):
fmts, subs = self._extract_m3u8_formats_and_subtitles(
format_url, video_id, 'mp4', m3u8_id=format_type, fatal=False,
)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
elif format_type == 'HDS':
formats.extend(
self._extract_f4m_formats(
format_url, video_id, f4m_id=format_type, fatal=False,
),
)
elif format_type == 'MPEG_DASH':
fmts, subs = self._extract_mpd_formats_and_subtitles(
format_url, video_id, mpd_id=format_type, fatal=False,
)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
elif format_type == 'HSS':
fmts, subs = self._extract_ism_formats_and_subtitles(
format_url, video_id, ism_id='mss', fatal=False,
)
formats.extend(fmts)
self._merge_subtitles(subs, target=subtitles)
else:
formats.append({
'format_id': format_type,
'url': format_url,
})
for sub in traverse_obj(
data, ('subtitleUrls', lambda _, v: v['url'] and v['type'] == 'CLOSED'),
):
subtitles.setdefault('nl', []).append({'url': sub['url']})
return formats, subtitles
def _get_authorization_token_from_cookie(self):
return self._get_token_from_cookie('vrtnu-site_profile_at')
def _get_identity_token_from_cookie(self):
return self._get_token_from_cookie('vrtnu-site_profile_vt')
def _get_token_from_cookie(self, cookie_name):
return self._get_cookies('https://www.vrt.be').get(cookie_name).value
def _perform_login(self, username, password):
self._request_webpage(
'https://www.vrt.be/vrtnu/sso/login',
None,
note='Getting session cookies',
errnote='Failed to get session cookies',
)
self._download_json(
'https://login.vrt.be/perform_login',
None,
data=json.dumps({
'loginID': username,
'password': password,
'clientId': 'vrtnu-site',
}).encode(),
headers={
'Content-Type': 'application/json',
'Oidcxsrf': self._get_cookies('https://login.vrt.be')
.get('OIDCXSRF')
.value,
},
note='Logging in',
errnote='Login failed',
)
class VRTIE(VRTBaseIE):
IE_DESC = 'VRT NWS, Flanders News, Flandern Info and Sporza'
_VALID_URL = r'https?://(?:www\.)?(?P<site>vrt\.be/vrtnws|sporza\.be)/[a-z]{2}/\d{4}/\d{2}/\d{2}/(?P<id>[^/?&#]+)'
@ -176,157 +286,153 @@ def _real_extract(self, url):
}
class VrtNUIE(VRTBaseIE):
class VRTMAXIE(VRTMAXBaseIE):
IE_DESC = 'VRT MAX'
_VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
_NETRC_MACHINE = 'vrtmax'
_TESTS = [{
# CONTENT_IS_AGE_RESTRICTED
'url': 'https://www.vrt.be/vrtnu/a-z/de-ideale-wereld/2023-vj/de-ideale-wereld-d20230116/',
'url': 'https://www.vrt.be/vrtmax/a-z/pano/trailer/pano-trailer-najaar-2023/',
'info_dict': {
'id': 'pbs-pub-855b00a8-6ce2-4032-ac4f-1fcf3ae78524$vid-d2243aa1-ec46-4e34-a55b-92568459906f',
'channel': 'vrtnws',
'description': 'md5:2e716da5a62687ecda1f40abfd742f81',
'duration': 37.16,
'episode_id': '3226122918145',
'ext': 'mp4',
'title': 'Tom Waes',
'description': 'Satirisch actualiteitenmagazine met Ella Leyers. Tom Waes is te gast.',
'timestamp': 1673905125,
'release_timestamp': 1673905125,
'series': 'De ideale wereld',
'season_id': '1672830988794',
'episode': 'Aflevering 1',
'episode_number': 1,
'episode_id': '1672830988861',
'display_id': 'de-ideale-wereld-d20230116',
'channel': 'VRT',
'duration': 1939.0,
'thumbnail': 'https://images.vrt.be/orig/2023/01/10/1bb39cb3-9115-11ed-b07d-02b7b76bf47f.jpg',
'release_date': '20230116',
'upload_date': '20230116',
'age_limit': 12,
'id': 'pbs-pub-5260ad6d-372c-46d3-a542-0e781fd5831a$vid-75fdb750-82f5-4157-8ea9-4485f303f20b',
'release_date': '20231106',
'release_timestamp': 1699246800,
'season': 'Trailer',
'season_id': '/vrtnu/a-z/pano/trailer/#tvseason',
'season_number': 2023,
'series': 'Pano',
'thumbnail': 'https://images.vrt.be/orig/2023/11/03/f570eb9b-7a4e-11ee-91d7-02b7b76bf47f.jpg',
'timestamp': 1699246800,
'title': 'Pano - Nieuwe afleveringen vanaf 15 november - Trailer | VRT MAX',
'upload_date': '20231106',
},
}, {
'url': 'https://www.vrt.be/vrtnu/a-z/buurman--wat-doet-u-nu-/6/buurman--wat-doet-u-nu--s6-trailer/',
'url': 'https://www.vrt.be/vrtnu/a-z/factcheckers/trailer/factcheckers-trailer-s4/',
'info_dict': {
'id': 'pbs-pub-ad4050eb-d9e5-48c2-9ec8-b6c355032361$vid-0465537a-34a8-4617-8352-4d8d983b4eee',
'ext': 'mp4',
'title': 'Trailer seizoen 6 \'Buurman, wat doet u nu?\'',
'description': 'md5:197424726c61384b4e5c519f16c0cf02',
'timestamp': 1652940000,
'release_timestamp': 1652940000,
'series': 'Buurman, wat doet u nu?',
'season': 'Seizoen 6',
'season_number': 6,
'season_id': '1652344200907',
'episode': 'Aflevering 0',
'channel': 'een',
'description': 'md5:e7924e23d6879fe0af1ebe240d1c92ca',
'duration': 33.08,
'episode': '0',
'episode_id': '3179360900145',
'episode_number': 0,
'episode_id': '1652951873524',
'display_id': 'buurman--wat-doet-u-nu--s6-trailer',
'channel': 'VRT',
'duration': 33.13,
'thumbnail': 'https://images.vrt.be/orig/2022/05/23/3c234d21-da83-11ec-b07d-02b7b76bf47f.jpg',
'release_date': '20220519',
'upload_date': '20220519',
'ext': 'mp4',
'id': 'pbs-pub-aa9397e9-ec2b-45f9-9148-7ce71b690b45$vid-04c67438-4866-4f5c-8978-51d173c0074b',
'release_timestamp': 1699160400,
'release_date': '20231105',
'season': 'Trailer',
'season_id': '/vrtnu/a-z/factcheckers/trailer/#tvseason',
'season_number': 2023,
'series': 'Factcheckers',
'timestamp': 1699160400,
'title': 'Factcheckers - Nieuwe afleveringen vanaf 15 november - Trailer | VRT MAX',
'thumbnail': 'https://images.vrt.be/orig/2023/11/07/37d244f0-7d8a-11ee-91d7-02b7b76bf47f.jpg',
'upload_date': '20231105',
},
'params': {'skip_download': 'm3u8'},
}]
_NETRC_MACHINE = 'vrtnu'
_authenticated = False
_VALID_URL = r'https?://(?:www\.)?vrt\.be/(vrtmax|vrtnu)/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
def _perform_login(self, username, password):
auth_info = self._gigya_login({
'APIKey': '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy',
'targetEnv': 'jssdk',
'loginID': username,
'password': password,
'authMode': 'cookie',
})
if auth_info.get('errorDetails'):
raise ExtractorError(f'Unable to login. VrtNU said: {auth_info["errorDetails"]}', expected=True)
# Sometimes authentication fails for no good reason, retry
for retry in self.RetryManager():
if retry.attempt > 1:
self._sleep(1, None)
try:
self._request_webpage(
'https://token.vrt.be/vrtnuinitlogin', None, note='Requesting XSRF Token',
errnote='Could not get XSRF Token', query={
'provider': 'site',
'destination': 'https://www.vrt.be/vrtnu/',
})
self._request_webpage(
'https://login.vrt.be/perform_login', None,
note='Performing login', errnote='Login failed',
query={'client_id': 'vrtnu-site'}, data=urlencode_postdata({
'UID': auth_info['UID'],
'UIDSignature': auth_info['UIDSignature'],
'signatureTimestamp': auth_info['signatureTimestamp'],
'_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
}))
except ExtractorError as e:
if isinstance(e.cause, HTTPError) and e.cause.status == 401:
retry.error = e
continue
raise
self._authenticated = True
_VIDEO_PAGE_QUERY = '''
query VideoPage($pageId: ID!) {
page(id: $pageId) {
... on EpisodePage {
id
title
seo {
... on SeoProperties {
__typename
description
title
}
__typename
}
ldjson
episode {
ageRaw
episodeNumberRaw
program {
title
__typename
}
name
onTimeRaw
watchAction {
streamId
__typename
}
__typename
}
__typename
}
__typename
}
}
'''
def _real_extract(self, url):
display_id = self._match_id(url)
parsed_url = urllib.parse.urlparse(url)
details = self._download_json(
f'{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rstrip("/")}.model.json',
display_id, 'Downloading asset JSON', 'Unable to download asset JSON')['details']
watch_info = traverse_obj(details, (
'actions', lambda _, v: v['type'] == 'watch-episode', {dict}), get_all=False) or {}
video_id = join_nonempty(
'episodePublicationId', 'episodeVideoId', delim='$', from_dict=watch_info)
if '$' not in video_id:
raise ExtractorError('Unable to extract video ID')
self._request_webpage(
'https://www.vrt.be/vrtnu/sso/login',
None,
note='Getting tokens',
errnote='Failed to get tokens',
)
vrtnutoken = self._download_json(
'https://token.vrt.be/refreshtoken', video_id, note='Retrieving vrtnutoken',
errnote='Token refresh failed')['vrtnutoken'] if self._authenticated else None
metadata = self._download_json(
'https://www.vrt.be/vrtnu-api/graphql/v1',
display_id,
'Downloading asset JSON',
'Unable to download asset JSON',
headers={
'Authorization': f'Bearer {self._get_authorization_token_from_cookie()}',
'Content-Type': 'application/json',
'x-vrt-client-name': 'WEB',
},
data=json.dumps({
'operationName': 'VideoPage',
'query': self._VIDEO_PAGE_QUERY,
'variables': {
'pageId': f'{parsed_url.path.rstrip("/")}.model.json',
},
}).encode(),
)['data']['page']
video_info = self._call_api(video_id, 'vrtnu-web@PROD', vrtnutoken)
video_id = metadata['episode']['watchAction']['streamId']
ld_json = self._parse_json(traverse_obj(metadata, ('ldjson', 1)) or '', video_id, fatal=False) or {}
if 'title' not in video_info:
code = video_info.get('code')
if code in ('AUTHENTICATION_REQUIRED', 'CONTENT_IS_AGE_RESTRICTED'):
self.raise_login_required(code, method='password')
elif code in ('INVALID_LOCATION', 'CONTENT_AVAILABLE_ONLY_IN_BE'):
self.raise_geo_restricted(countries=['BE'])
elif code == 'CONTENT_AVAILABLE_ONLY_FOR_BE_RESIDENTS_AND_EXPATS':
if not self._authenticated:
self.raise_login_required(code, method='password')
self.raise_geo_restricted(countries=['BE'])
raise ExtractorError(code, expected=True)
formats, subtitles = self._extract_formats_and_subtitles(video_info, video_id)
streaming_info = self._call_api(video_id, client='vrtnu-web@PROD')
formats, subtitles = self._extract_formats_and_subtitles(streaming_info, video_id)
return {
**traverse_obj(details, {
'title': 'title',
'description': ('description', {clean_html}),
'timestamp': ('data', 'episode', 'onTime', 'raw', {parse_iso8601}),
'release_timestamp': ('data', 'episode', 'onTime', 'raw', {parse_iso8601}),
'series': ('data', 'program', 'title'),
'season': ('data', 'season', 'title', 'value'),
'season_number': ('data', 'season', 'title', 'raw', {int_or_none}),
'season_id': ('data', 'season', 'id', {str_or_none}),
'episode': ('data', 'episode', 'number', 'value', {str_or_none}),
'episode_number': ('data', 'episode', 'number', 'raw', {int_or_none}),
'episode_id': ('data', 'episode', 'id', {str_or_none}),
'age_limit': ('data', 'episode', 'age', 'raw', {parse_age_limit}),
**traverse_obj(ld_json, {
'episode': ('episodeNumber', {int_or_none}),
'episode_id': ('@id', {str_or_none}),
'episode_number': ('episodeNumber', {int_or_none}),
'season': ('partOfSeason', 'name'),
'season_id': ('partOfSeason', '@id', {str_or_none}),
'series': ('partOfSeries', 'name'),
}),
'id': video_id,
'display_id': display_id,
'channel': 'VRT',
**traverse_obj(metadata, {
'age_limit': ('episode', 'ageRaw', {parse_age_limit}),
'channel': ('episode', 'brand'),
'description': ('seo', 'description', {str_or_none}),
'display_id': ('episode', 'name', {parse_age_limit}),
'release_date': ('episode', 'onTimeRaw', {unified_strdate}),
'release_timestamp': ('episode', 'onTimeRaw', {parse_iso8601}),
'season_number': ('episode', 'onTimeRaw', {lambda x: x[:4]}, {int_or_none}),
'timestamp': ('episode', 'onTimeRaw', {parse_iso8601}),
'title': ('seo', 'title', {str_or_none}),
'upload_date': ('episode', 'onTimeRaw', {unified_strdate}),
}),
'duration': float_or_none(streaming_info.get('duration'), 1000),
'formats': formats,
'duration': float_or_none(video_info.get('duration'), 1000),
'thumbnail': url_or_none(video_info.get('posterImageUrl')),
'id': video_id,
'subtitles': subtitles,
'_old_archive_ids': [make_archive_id('Canvas', video_id)],
'thumbnail': url_or_none(streaming_info.get('posterImageUrl')),
}