From cfd3f5ba8047520ba8966b5f4ebcb94e1b01725f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20De=20Boey?= Date: Wed, 26 Feb 2025 04:17:41 +0100 Subject: [PATCH] [VRT] Update VRT MAX extractor --- yt_dlp/extractor/_extractors.py | 2 +- yt_dlp/extractor/gigya.py | 19 -- yt_dlp/extractor/vrt.py | 372 ++++++++++++++++++++------------ 3 files changed, 240 insertions(+), 153 deletions(-) delete mode 100644 yt_dlp/extractor/gigya.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 403e1f1f6..b2fb51986 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -2397,10 +2397,10 @@ ) from .vrt import ( VRTIE, + VRTMAXIE, DagelijkseKostIE, KetnetIE, Radio1BeIE, - VrtNUIE, ) from .vtm import VTMIE from .vtv import ( diff --git a/yt_dlp/extractor/gigya.py b/yt_dlp/extractor/gigya.py deleted file mode 100644 index cc18ee67c..000000000 --- a/yt_dlp/extractor/gigya.py +++ /dev/null @@ -1,19 +0,0 @@ -from .common import InfoExtractor -from ..utils import ( - ExtractorError, - urlencode_postdata, -) - - -class GigyaBaseIE(InfoExtractor): - def _gigya_login(self, auth_data): - auth_info = self._download_json( - 'https://accounts.eu1.gigya.com/accounts.login', None, - note='Logging in', errnote='Unable to log in', - data=urlencode_postdata(auth_data)) - - error_message = auth_info.get('errorDetails') or auth_info.get('errorMessage') - if error_message: - raise ExtractorError( - f'Unable to login: {error_message}', expected=True) - return auth_info diff --git a/yt_dlp/extractor/vrt.py b/yt_dlp/extractor/vrt.py index 9345ca962..7923ff678 100644 --- a/yt_dlp/extractor/vrt.py +++ b/yt_dlp/extractor/vrt.py @@ -1,18 +1,16 @@ import json import time import urllib.parse +import urllib.request -from .gigya import GigyaBaseIE -from ..networking.exceptions import HTTPError +from .common import InfoExtractor from ..utils import ( - ExtractorError, clean_html, extract_attributes, float_or_none, get_element_by_class, get_element_html_by_class, int_or_none, - join_nonempty, jwt_encode_hs256, make_archive_id, merge_dicts, @@ -21,12 +19,12 @@ str_or_none, strip_or_none, traverse_obj, + unified_strdate, url_or_none, - urlencode_postdata, ) -class VRTBaseIE(GigyaBaseIE): +class VRTBaseIE(InfoExtractor): _GEO_BYPASS = False _PLAYER_INFO = { 'platform': 'desktop', @@ -109,6 +107,118 @@ def _call_api(self, video_id, client='null', id_token=None, version='v2'): }, expected_status=400) +class VRTMAXBaseIE(InfoExtractor): + _GEO_BYPASS = False + + def _call_api(self, video_id, client='null', id_token=None, version='v2'): + vrt_player_token = self._download_json( + f'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/{version}/tokens', + None, + 'Downloading player token', + 'Failed to download player token', + headers={ + 'Content-Type': 'application/json', + }, + data=json.dumps({ + 'identityToken': id_token or self._get_identity_token_from_cookie(), + }).encode(), + )['vrtPlayerToken'] + + return self._download_json( + f'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/{version}/videos/{video_id}', + video_id, + 'Downloading API JSON', + 'Failed to download API JSON', + query={ + 'client': client, + 'vrtPlayerToken': vrt_player_token, + }, + ) + + def _extract_formats_and_subtitles(self, data, video_id): + # probably needs an extra check against `drmExpired` + if traverse_obj(data, 'drm'): + self.report_drm(video_id) + + formats, subtitles = [], {} + for target in traverse_obj( + data, ('targetUrls', lambda _, v: url_or_none(v['url']) and v['type']), + ): + format_type = target['type'].upper() + format_url = target['url'] + if format_type in ('HLS', 'HLS_AES'): + fmts, subs = self._extract_m3u8_formats_and_subtitles( + format_url, video_id, 'mp4', m3u8_id=format_type, fatal=False, + ) + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) + elif format_type == 'HDS': + formats.extend( + self._extract_f4m_formats( + format_url, video_id, f4m_id=format_type, fatal=False, + ), + ) + elif format_type == 'MPEG_DASH': + fmts, subs = self._extract_mpd_formats_and_subtitles( + format_url, video_id, mpd_id=format_type, fatal=False, + ) + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) + elif format_type == 'HSS': + fmts, subs = self._extract_ism_formats_and_subtitles( + format_url, video_id, ism_id='mss', fatal=False, + ) + formats.extend(fmts) + self._merge_subtitles(subs, target=subtitles) + else: + formats.append({ + 'format_id': format_type, + 'url': format_url, + }) + + for sub in traverse_obj( + data, ('subtitleUrls', lambda _, v: v['url'] and v['type'] == 'CLOSED'), + ): + subtitles.setdefault('nl', []).append({'url': sub['url']}) + + return formats, subtitles + + def _get_authorization_token_from_cookie(self): + return self._get_token_from_cookie('vrtnu-site_profile_at') + + def _get_identity_token_from_cookie(self): + return self._get_token_from_cookie('vrtnu-site_profile_vt') + + def _get_token_from_cookie(self, cookie_name): + return self._get_cookies('https://www.vrt.be').get(cookie_name).value + + def _perform_login(self, username, password): + self._request_webpage( + 'https://www.vrt.be/vrtnu/sso/login', + None, + note='Getting session cookies', + errnote='Failed to get session cookies', + ) + + self._download_json( + 'https://login.vrt.be/perform_login', + None, + data=json.dumps({ + 'loginID': username, + 'password': password, + 'clientId': 'vrtnu-site', + }).encode(), + headers={ + 'Content-Type': 'application/json', + 'Oidcxsrf': self._get_cookies('https://login.vrt.be') + .get('OIDCXSRF') + .value, + }, + note='Logging in', + errnote='Login failed', + ) + + class VRTIE(VRTBaseIE): IE_DESC = 'VRT NWS, Flanders News, Flandern Info and Sporza' _VALID_URL = r'https?://(?:www\.)?(?Pvrt\.be/vrtnws|sporza\.be)/[a-z]{2}/\d{4}/\d{2}/\d{2}/(?P[^/?&#]+)' @@ -176,157 +286,153 @@ def _real_extract(self, url): } -class VrtNUIE(VRTBaseIE): +class VRTMAXIE(VRTMAXBaseIE): IE_DESC = 'VRT MAX' - _VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P[^/?#&]+)' + _NETRC_MACHINE = 'vrtmax' _TESTS = [{ - # CONTENT_IS_AGE_RESTRICTED - 'url': 'https://www.vrt.be/vrtnu/a-z/de-ideale-wereld/2023-vj/de-ideale-wereld-d20230116/', + 'url': 'https://www.vrt.be/vrtmax/a-z/pano/trailer/pano-trailer-najaar-2023/', 'info_dict': { - 'id': 'pbs-pub-855b00a8-6ce2-4032-ac4f-1fcf3ae78524$vid-d2243aa1-ec46-4e34-a55b-92568459906f', + 'channel': 'vrtnws', + 'description': 'md5:2e716da5a62687ecda1f40abfd742f81', + 'duration': 37.16, + 'episode_id': '3226122918145', 'ext': 'mp4', - 'title': 'Tom Waes', - 'description': 'Satirisch actualiteitenmagazine met Ella Leyers. Tom Waes is te gast.', - 'timestamp': 1673905125, - 'release_timestamp': 1673905125, - 'series': 'De ideale wereld', - 'season_id': '1672830988794', - 'episode': 'Aflevering 1', - 'episode_number': 1, - 'episode_id': '1672830988861', - 'display_id': 'de-ideale-wereld-d20230116', - 'channel': 'VRT', - 'duration': 1939.0, - 'thumbnail': 'https://images.vrt.be/orig/2023/01/10/1bb39cb3-9115-11ed-b07d-02b7b76bf47f.jpg', - 'release_date': '20230116', - 'upload_date': '20230116', - 'age_limit': 12, + 'id': 'pbs-pub-5260ad6d-372c-46d3-a542-0e781fd5831a$vid-75fdb750-82f5-4157-8ea9-4485f303f20b', + 'release_date': '20231106', + 'release_timestamp': 1699246800, + 'season': 'Trailer', + 'season_id': '/vrtnu/a-z/pano/trailer/#tvseason', + 'season_number': 2023, + 'series': 'Pano', + 'thumbnail': 'https://images.vrt.be/orig/2023/11/03/f570eb9b-7a4e-11ee-91d7-02b7b76bf47f.jpg', + 'timestamp': 1699246800, + 'title': 'Pano - Nieuwe afleveringen vanaf 15 november - Trailer | VRT MAX', + 'upload_date': '20231106', }, }, { - 'url': 'https://www.vrt.be/vrtnu/a-z/buurman--wat-doet-u-nu-/6/buurman--wat-doet-u-nu--s6-trailer/', + 'url': 'https://www.vrt.be/vrtnu/a-z/factcheckers/trailer/factcheckers-trailer-s4/', 'info_dict': { - 'id': 'pbs-pub-ad4050eb-d9e5-48c2-9ec8-b6c355032361$vid-0465537a-34a8-4617-8352-4d8d983b4eee', - 'ext': 'mp4', - 'title': 'Trailer seizoen 6 \'Buurman, wat doet u nu?\'', - 'description': 'md5:197424726c61384b4e5c519f16c0cf02', - 'timestamp': 1652940000, - 'release_timestamp': 1652940000, - 'series': 'Buurman, wat doet u nu?', - 'season': 'Seizoen 6', - 'season_number': 6, - 'season_id': '1652344200907', - 'episode': 'Aflevering 0', + 'channel': 'een', + 'description': 'md5:e7924e23d6879fe0af1ebe240d1c92ca', + 'duration': 33.08, + 'episode': '0', + 'episode_id': '3179360900145', 'episode_number': 0, - 'episode_id': '1652951873524', - 'display_id': 'buurman--wat-doet-u-nu--s6-trailer', - 'channel': 'VRT', - 'duration': 33.13, - 'thumbnail': 'https://images.vrt.be/orig/2022/05/23/3c234d21-da83-11ec-b07d-02b7b76bf47f.jpg', - 'release_date': '20220519', - 'upload_date': '20220519', + 'ext': 'mp4', + 'id': 'pbs-pub-aa9397e9-ec2b-45f9-9148-7ce71b690b45$vid-04c67438-4866-4f5c-8978-51d173c0074b', + 'release_timestamp': 1699160400, + 'release_date': '20231105', + 'season': 'Trailer', + 'season_id': '/vrtnu/a-z/factcheckers/trailer/#tvseason', + 'season_number': 2023, + 'series': 'Factcheckers', + 'timestamp': 1699160400, + 'title': 'Factcheckers - Nieuwe afleveringen vanaf 15 november - Trailer | VRT MAX', + 'thumbnail': 'https://images.vrt.be/orig/2023/11/07/37d244f0-7d8a-11ee-91d7-02b7b76bf47f.jpg', + 'upload_date': '20231105', }, - 'params': {'skip_download': 'm3u8'}, }] - _NETRC_MACHINE = 'vrtnu' - _authenticated = False + _VALID_URL = r'https?://(?:www\.)?vrt\.be/(vrtmax|vrtnu)/a-z/(?:[^/]+/){2}(?P[^/?#&]+)' - def _perform_login(self, username, password): - auth_info = self._gigya_login({ - 'APIKey': '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy', - 'targetEnv': 'jssdk', - 'loginID': username, - 'password': password, - 'authMode': 'cookie', - }) - - if auth_info.get('errorDetails'): - raise ExtractorError(f'Unable to login. VrtNU said: {auth_info["errorDetails"]}', expected=True) - - # Sometimes authentication fails for no good reason, retry - for retry in self.RetryManager(): - if retry.attempt > 1: - self._sleep(1, None) - try: - self._request_webpage( - 'https://token.vrt.be/vrtnuinitlogin', None, note='Requesting XSRF Token', - errnote='Could not get XSRF Token', query={ - 'provider': 'site', - 'destination': 'https://www.vrt.be/vrtnu/', - }) - self._request_webpage( - 'https://login.vrt.be/perform_login', None, - note='Performing login', errnote='Login failed', - query={'client_id': 'vrtnu-site'}, data=urlencode_postdata({ - 'UID': auth_info['UID'], - 'UIDSignature': auth_info['UIDSignature'], - 'signatureTimestamp': auth_info['signatureTimestamp'], - '_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value, - })) - except ExtractorError as e: - if isinstance(e.cause, HTTPError) and e.cause.status == 401: - retry.error = e - continue - raise - - self._authenticated = True + _VIDEO_PAGE_QUERY = ''' + query VideoPage($pageId: ID!) { + page(id: $pageId) { + ... on EpisodePage { + id + title + seo { + ... on SeoProperties { + __typename + description + title + } + __typename + } + ldjson + episode { + ageRaw + episodeNumberRaw + program { + title + __typename + } + name + onTimeRaw + watchAction { + streamId + __typename + } + __typename + } + __typename + } + __typename + } + } + ''' def _real_extract(self, url): display_id = self._match_id(url) parsed_url = urllib.parse.urlparse(url) - details = self._download_json( - f'{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path.rstrip("/")}.model.json', - display_id, 'Downloading asset JSON', 'Unable to download asset JSON')['details'] - watch_info = traverse_obj(details, ( - 'actions', lambda _, v: v['type'] == 'watch-episode', {dict}), get_all=False) or {} - video_id = join_nonempty( - 'episodePublicationId', 'episodeVideoId', delim='$', from_dict=watch_info) - if '$' not in video_id: - raise ExtractorError('Unable to extract video ID') + self._request_webpage( + 'https://www.vrt.be/vrtnu/sso/login', + None, + note='Getting tokens', + errnote='Failed to get tokens', + ) - vrtnutoken = self._download_json( - 'https://token.vrt.be/refreshtoken', video_id, note='Retrieving vrtnutoken', - errnote='Token refresh failed')['vrtnutoken'] if self._authenticated else None + metadata = self._download_json( + 'https://www.vrt.be/vrtnu-api/graphql/v1', + display_id, + 'Downloading asset JSON', + 'Unable to download asset JSON', + headers={ + 'Authorization': f'Bearer {self._get_authorization_token_from_cookie()}', + 'Content-Type': 'application/json', + 'x-vrt-client-name': 'WEB', + }, + data=json.dumps({ + 'operationName': 'VideoPage', + 'query': self._VIDEO_PAGE_QUERY, + 'variables': { + 'pageId': f'{parsed_url.path.rstrip("/")}.model.json', + }, + }).encode(), + )['data']['page'] - video_info = self._call_api(video_id, 'vrtnu-web@PROD', vrtnutoken) + video_id = metadata['episode']['watchAction']['streamId'] + ld_json = self._parse_json(traverse_obj(metadata, ('ldjson', 1)) or '', video_id, fatal=False) or {} - if 'title' not in video_info: - code = video_info.get('code') - if code in ('AUTHENTICATION_REQUIRED', 'CONTENT_IS_AGE_RESTRICTED'): - self.raise_login_required(code, method='password') - elif code in ('INVALID_LOCATION', 'CONTENT_AVAILABLE_ONLY_IN_BE'): - self.raise_geo_restricted(countries=['BE']) - elif code == 'CONTENT_AVAILABLE_ONLY_FOR_BE_RESIDENTS_AND_EXPATS': - if not self._authenticated: - self.raise_login_required(code, method='password') - self.raise_geo_restricted(countries=['BE']) - raise ExtractorError(code, expected=True) - - formats, subtitles = self._extract_formats_and_subtitles(video_info, video_id) + streaming_info = self._call_api(video_id, client='vrtnu-web@PROD') + formats, subtitles = self._extract_formats_and_subtitles(streaming_info, video_id) return { - **traverse_obj(details, { - 'title': 'title', - 'description': ('description', {clean_html}), - 'timestamp': ('data', 'episode', 'onTime', 'raw', {parse_iso8601}), - 'release_timestamp': ('data', 'episode', 'onTime', 'raw', {parse_iso8601}), - 'series': ('data', 'program', 'title'), - 'season': ('data', 'season', 'title', 'value'), - 'season_number': ('data', 'season', 'title', 'raw', {int_or_none}), - 'season_id': ('data', 'season', 'id', {str_or_none}), - 'episode': ('data', 'episode', 'number', 'value', {str_or_none}), - 'episode_number': ('data', 'episode', 'number', 'raw', {int_or_none}), - 'episode_id': ('data', 'episode', 'id', {str_or_none}), - 'age_limit': ('data', 'episode', 'age', 'raw', {parse_age_limit}), + **traverse_obj(ld_json, { + 'episode': ('episodeNumber', {int_or_none}), + 'episode_id': ('@id', {str_or_none}), + 'episode_number': ('episodeNumber', {int_or_none}), + 'season': ('partOfSeason', 'name'), + 'season_id': ('partOfSeason', '@id', {str_or_none}), + 'series': ('partOfSeries', 'name'), }), - 'id': video_id, - 'display_id': display_id, - 'channel': 'VRT', + **traverse_obj(metadata, { + 'age_limit': ('episode', 'ageRaw', {parse_age_limit}), + 'channel': ('episode', 'brand'), + 'description': ('seo', 'description', {str_or_none}), + 'display_id': ('episode', 'name', {parse_age_limit}), + 'release_date': ('episode', 'onTimeRaw', {unified_strdate}), + 'release_timestamp': ('episode', 'onTimeRaw', {parse_iso8601}), + 'season_number': ('episode', 'onTimeRaw', {lambda x: x[:4]}, {int_or_none}), + 'timestamp': ('episode', 'onTimeRaw', {parse_iso8601}), + 'title': ('seo', 'title', {str_or_none}), + 'upload_date': ('episode', 'onTimeRaw', {unified_strdate}), + }), + 'duration': float_or_none(streaming_info.get('duration'), 1000), 'formats': formats, - 'duration': float_or_none(video_info.get('duration'), 1000), - 'thumbnail': url_or_none(video_info.get('posterImageUrl')), + 'id': video_id, 'subtitles': subtitles, - '_old_archive_ids': [make_archive_id('Canvas', video_id)], + 'thumbnail': url_or_none(streaming_info.get('posterImageUrl')), }