From 076ca745aab9154d65068fcd106be0fea68d41b4 Mon Sep 17 00:00:00 2001 From: c-basalt <117849907+c-basalt@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:50:05 -0500 Subject: [PATCH] matrix download test --- test/test_download.py | 21 +++++++++++++++++++++ yt_dlp/extractor/iqiyi.py | 22 ++++++++++++++++++++++ yt_dlp/jsinterp/common.py | 37 +++++++++++++++++-------------------- 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index 3f36869d9..5c6d4f99d 100755 --- a/test/test_download.py +++ b/test/test_download.py @@ -25,6 +25,7 @@ import yt_dlp.YoutubeDL # isort: split from yt_dlp.extractor import get_info_extractor +from yt_dlp.jsinterp.common import filter_jsi_feature, filter_jsi_include from yt_dlp.networking.exceptions import HTTPError, TransportError from yt_dlp.utils import ( DownloadError, @@ -82,6 +83,26 @@ def __str__(self): # Dynamically generate tests def generator(test_case, tname): + def generate_sub_case(jsi_key): + sub_case = {k: v for k, v in test_case.items() if not k.startswith('jsi_matrix')} + sub_case['params'] = {**test_case.get('params', {}), 'jsi_preference': [jsi_key]} + return generator(sub_case, f'{tname}_{jsi_key}') + + # setting `jsi_matrix` to True, `jsi_matrix_features` to list, or + # setting `jsi_matrix_only_include` or `jsi_matrix_exclude` to non-empty + # to trigger matrix behavior + if isinstance(test_case.get('jsi_matrix_features'), list) or any(test_case.get(key) for key in [ + 'jsi_matrix', 'jsi_matrix_only_include', 'jsi_matrix_exclude', + ]): + jsi_keys = filter_jsi_feature(test_case.get('jsi_matrix_features', []), filter_jsi_include( + test_case.get('jsi_matrix_only_include', None), test_case.get('jsi_matrix_exclude', None))) + + def run_sub_cases(self): + for i, jsi_key in enumerate(jsi_keys): + print(f'Running case {tname} using JSI: {jsi_key} ({i + 1}/{len(jsi_keys)})') + generate_sub_case(jsi_key)(self) + return run_sub_cases + def test_template(self): if self.COMPLETED_TESTS.get(tname): return diff --git a/yt_dlp/extractor/iqiyi.py b/yt_dlp/extractor/iqiyi.py index 6e1e18833..813984769 100644 --- a/yt_dlp/extractor/iqiyi.py +++ b/yt_dlp/extractor/iqiyi.py @@ -398,6 +398,27 @@ class IqIE(InfoExtractor): IE_DESC = 'International version of iQiyi' _VALID_URL = r'https?://(?:www\.)?iq\.com/play/(?:[\w%-]*-)?(?P\w+)' _TESTS = [{ + 'url': 'https://www.iq.com/play/sangmin-dinneaw-episode-1-xmk7546rfw', + 'md5': '63fcb4b7d4863472fe0a9be75d9e9d60', + 'info_dict': { + 'ext': 'mp4', + 'id': 'xmk7546rfw', + 'title': '尚岷与丁尼奥 第1集', + 'description': 'md5:e8fe4a8da25f4b8c86bc5506b1c3faaa', + 'duration': 3092, + 'timestamp': 1735520401, + 'upload_date': '20241230', + 'episode_number': 1, + 'episode': 'Episode 1', + 'series': 'Sangmin Dinneaw', + 'age_limit': 18, + 'average_rating': float, + 'categories': [], + 'cast': ['Sangmin Choi', 'Ratana Aiamsaart'], + }, + 'expected_warnings': ['format is restricted'], + 'jsi_matrix_features': ['dom'], + }, { 'url': 'https://www.iq.com/play/one-piece-episode-1000-1ma1i6ferf4', 'md5': '2d7caf6eeca8a32b407094b33b757d39', 'info_dict': { @@ -418,6 +439,7 @@ class IqIE(InfoExtractor): 'format': '500', }, 'expected_warnings': ['format is restricted'], + 'skip': 'geo-restricted', }, { # VIP-restricted video 'url': 'https://www.iq.com/play/mermaid-in-the-fog-2021-gbdpx13bs4', diff --git a/yt_dlp/jsinterp/common.py b/yt_dlp/jsinterp/common.py index 552f12ee5..f95b9ab63 100644 --- a/yt_dlp/jsinterp/common.py +++ b/yt_dlp/jsinterp/common.py @@ -31,6 +31,17 @@ def get_jsi_keys(jsi_or_keys: typing.Iterable[str | type[JSI] | JSI]) -> list[st return [jok if isinstance(jok, str) else jok.JSI_KEY for jok in jsi_or_keys] +def filter_jsi_include(only_include: typing.Iterable[str] | None, exclude: typing.Iterable[str] | None): + keys = get_jsi_keys(only_include) if only_include else _JSI_HANDLERS.keys() + return [key for key in keys if key not in (exclude or [])] + + +def filter_jsi_feature(features: typing.Iterable[str], keys=None): + keys = keys if keys is not None else _JSI_HANDLERS.keys() + return [key for key in keys if key in _JSI_HANDLERS + and _JSI_HANDLERS[key]._SUPPORTED_FEATURES.issuperset(features)] + + def order_to_pref(jsi_order: typing.Iterable[str | type[JSI] | JSI], multiplier: int) -> JSIPreference: jsi_order = reversed(get_jsi_keys(jsi_order)) pref_score = {jsi_cls: (i + 1) * multiplier for i, jsi_cls in enumerate(jsi_order)} @@ -112,10 +123,9 @@ def __init__( self.report_warning(f'`{invalid_key}` is not a valid JSI, ignoring preference setting') user_prefs.remove(invalid_key) - jsi_keys = [key for key in get_jsi_keys(only_include or _JSI_HANDLERS) if key not in get_jsi_keys(exclude)] + jsi_keys = filter_jsi_include(only_include, exclude) self.write_debug(f'Allowed JSI keys: {jsi_keys}') - handler_classes = [_JSI_HANDLERS[key] for key in jsi_keys - if _JSI_HANDLERS[key]._SUPPORTED_FEATURES.issuperset(self._features)] + handler_classes = [_JSI_HANDLERS[key] for key in filter_jsi_feature(self._features, jsi_keys)] self.write_debug(f'Select JSI for features={self._features}: {get_jsi_keys(handler_classes)}, ' f'included: {get_jsi_keys(only_include) or "all"}, excluded: {get_jsi_keys(exclude)}') if not handler_classes: @@ -159,38 +169,25 @@ def _dispatch_request(self, method_name: str, *args, **kwargs): unavailable: list[str] = [] exceptions: list[tuple[JSI, Exception]] = [] - test_results: list[tuple[JSI, typing.Any]] = [] for handler in handlers: if not handler.is_available(): if self._is_test: - raise Exception(f'{handler.JSI_NAME} is not available for testing, ' - f'add "{handler.JSI_KEY}" in `exclude` if it should not be used') + raise ExtractorError(f'{handler.JSI_NAME} is not available for testing, ' + f'add "{handler.JSI_KEY}" in `exclude` if it should not be used') self.write_debug(f'{handler.JSI_KEY} is not available') unavailable.append(handler.JSI_NAME) continue try: self.write_debug(f'Dispatching `{method_name}` task to {handler.JSI_NAME}') - result = getattr(handler, method_name)(*args, **kwargs) - if self._is_test: - test_results.append((handler, result)) - else: - return result - except Exception as e: + return getattr(handler, method_name)(*args, **kwargs) + except ExtractorError as e: if handler.JSI_KEY not in self._fallback_jsi: raise else: exceptions.append((handler, e)) self.write_debug(f'{handler.JSI_NAME} encountered error, fallback to next handler: {e}') - if self._is_test and test_results: - ref_handler, ref_result = test_results[0] - for handler, result in test_results[1:]: - if result != ref_result: - self.report_warning( - f'Different JSI results produced from {ref_handler.JSI_NAME} and {handler.JSI_NAME}') - return ref_result - if not exceptions: msg = f'No available JSI installed, please install one of: {", ".join(unavailable)}' else: