playlist and misc

yt_dlp/extractor/rplay.py

@@ -17,6 +17,7 @@
     encode_data_uri,
     float_or_none,
     parse_iso8601,
+    parse_qs,
     traverse_obj,
     url_or_none,
 )
@@ -41,6 +42,20 @@ def login_type(self):
     def jwt_token(self):
         return self._jwt_token
 
+    @property
+    def requestor_query(self):
+        return {
+            'requestorOid': self.user_id,
+            'loginType': self.login_type,
+        } if self.user_id else {}
+
+    @property
+    def jwt_header(self):
+        return {
+            'Referer': 'https://rplay.live/',
+            'Authorization': self.jwt_token or 'null'
+        }
+
     def _jwt_encode_hs256(self, payload: dict, key: str):
         # ..utils.jwt_encode_hs256() uses slightly different details that would fails
         # and we need to re-implement it with minor changes
@@ -66,7 +81,7 @@ def _perform_login(self, username, password):
     def _login_by_token(self, jwt_token):
         user_info = self._download_json(
             'https://api.rplay.live/account/login', 'login', note='performing login', errnote='Failed to login',
-            data=f'{{"token":"{jwt_token}","lang":"en","loginType":null,"checkAdmin":null}}'.encode(),
+            data=f'{{"token":"{jwt_token}","loginType":null,"checkAdmin":null}}'.encode(),
             headers={'Content-Type': 'application/json', 'Authorization': 'null'}, fatal=False)
 
         if user_info:
@@ -80,57 +95,51 @@ def _get_butter_files(self):
         cache = self.cache.load('rplay', 'butter-code') or {}
         if cache.get('date', 0) > time.time() - 86400:
             return cache['js'], cache['wasm']
-        butter_js = self._download_webpage('https://pb.rplay.live/kr/public/smooth_like_butter.js', 'butter',
+        butter_js = self._download_webpage(
-                                           'getting butter-sign js')
+            'https://pb.rplay.live/kr/public/smooth_like_butter.js', 'butter', 'getting butter-sign js')
-        urlh = self._request_webpage('https://pb.rplay.live/kr/public/smooth_like_butter_bg.wasm', 'butter',
+        urlh = self._request_webpage(
-                                     'getting butter-sign wasm')
+            'https://pb.rplay.live/kr/public/smooth_like_butter_bg.wasm', 'butter', 'getting butter-sign wasm')
         butter_wasm_array = list(urlh.read())
         self.cache.store('rplay', 'butter-code', {'js': butter_js, 'wasm': butter_wasm_array, 'date': time.time()})
         return butter_js, butter_wasm_array
 
-    def _playwright_eval(self, jscode, goto=None, wait_until='commit', stop_loading=True):
+    def _playwright_eval(self, jscode, location='about:blank', body=''):
         async def __aeval():
             async with async_playwright() as p:
                 browser = await p.chromium.launch(chromium_sandbox=True)
                 page = await browser.new_page()
-                if goto:
+                # use page.route to skip network request while allowing changing window.location
+                await page.route('**', lambda route: route.fulfill(status=200, body=body))
+                # mock navigator to mimic regular browser
+                await page.add_init_script('''const proxy = new Proxy(window.navigator, {get(target, prop, receiver) {
+                        if (prop === "webdriver") return false;
+                        if (prop === "appVersion" || prop === "userAgent") return target[prop].replace(/Headless/g, '');
+                        return target[prop];
+                    }});
+                    Object.defineProperty(window, "navigator", {get: ()=> proxy});''')
+
+                def _page_eval_js(exp, timeout=10):
+                    return asyncio.wait_for(page.evaluate(exp), timeout=timeout)
                 try:
+                    await page.goto(location)  # always navigate once to trigger init script
                     start = time.time()
-                        await page.goto(goto, wait_until=wait_until)
+                    value = await _page_eval_js(jscode)
-                        self.write_debug(f'{wait_until} loaded in {time.time() - start:.3f}s')
-                        if stop_loading:
-                            await page.evaluate('window.stop();')
-                    except Exception as e:
-                        self.report_warning(f'Failed to navigate to {goto}: {e}')
-                        await browser.close()
-                        return
-                try:
-                    start = time.time()
-                    value = await asyncio.wait_for(page.evaluate(jscode), timeout=10)
                     self.write_debug(f'JS execution finished in {time.time() - start:.3f}s')
+                    return value
                 except asyncio.TimeoutError:
                     self.report_warning('PlayWright JS evaluation timed out')
-                    value = None
                 finally:
                     await browser.close()
-            return value
 
         try:
             return asyncio.run(__aeval())
         except asyncio.InvalidStateError:
-            pass
+            self.report_warning('PlayWright failed to evaluate JS')
 
     def _calc_butter_token(self):
         butter_js, butter_wasm_array = self._get_butter_files()
-        butter_js = butter_js.replace('export{initSync};export default __wbg_init;', '')
+        butter_js = re.sub(r'export(?:\s+default)?([\s{])', r'\1', butter_js)
-        butter_js = butter_js.replace('export class', 'class')
+        butter_js = butter_js.replace('import.meta', '{}')
-        butter_js = butter_js.replace('new URL("smooth_like_butter_bg.wasm",import.meta.url)', '""')
-
-        butter_js += ''';const proxy = new Proxy(window.navigator, {get(target, prop, receiver) {
-            if (prop == "webdriver") return false;
-            return target[prop];
-        }});
-        Object.defineProperty(window, "navigator", {get: ()=> proxy});'''
 
         butter_js += '''__new_init = async () => {
             const t = __wbg_get_imports();
@@ -141,8 +150,8 @@ def _calc_butter_token(self):
 
         butter_js += '__new_init().then(() => (new ButterFactory()).generate_butter())'
 
-        # The generator checks `navigator` and `location` to generate correct token
+        # The script checks `navigator.webdriver` and `location.origin` to generate correct token
-        return self._playwright_eval(butter_js, goto='https://rplay.live/')
+        return self._playwright_eval(butter_js, location='https://rplay.live')
 
     def get_butter_token(self):
         cache = self.cache.load('rplay', 'butter-token') or {}
@@ -173,6 +182,7 @@ class RPlayVideoIE(RPlayBaseIE):
             'uploader_id': '667adc9e9aa7f739a2158ff3',
             'tags': ['杏都める', 'めいどるーちぇ', '無料', '耳舐め', 'ASMR'],
         },
+        'params': {'cachedir': False},
     }, {
         'url': 'https://rplay.live/play/660bee4fd3c1d09d69db6870/',
         'info_dict': {
@@ -196,17 +206,26 @@ class RPlayVideoIE(RPlayBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        headers = {'Origin': 'https://rplay.live', 'Referer': 'https://rplay.live/'}
+        playlist_id = traverse_obj(parse_qs(url), ('playlist', ..., any))
+        if playlist_id and self._yes_playlist(playlist_id, video_id):
+            playlist_info = self._download_json(
+                'https://api.rplay.live/content/playlist', playlist_id,
+                query={'playlistOid': playlist_id, **self.requestor_query},
+                headers=self.jwt_header, fatal=False)
+            if playlist_info:
+                entries = traverse_obj(playlist_info, ('contentData', ..., '_id', {
+                    lambda x: self.url_result(f'https://rplay.live/play/{x}/', ie=RPlayVideoIE, video_id=x)}))
+                return self.playlist_result(entries, playlist_id, playlist_info.get('name'))
+            else:
+                self.report_warning('Failed to get playlist, downloading video only')
+
         video_info = self._download_json('https://api.rplay.live/content', video_id, query={
             'contentOid': video_id,
             'status': 'published',
             'withComments': True,
             'requestCanView': True,
-            **({
+            **self.requestor_query,
-                'requestorOid': self.user_id,
+        }, headers=self.jwt_header)
-                'loginType': self.login_type,
-            } if self.user_id else {}),
-        }, headers={**headers, 'Authorization': self.jwt_token or 'null'})
         if video_info.get('drm'):
             raise ExtractorError('This video is DRM-protected')
 
@@ -222,37 +241,36 @@ def _real_extract(self, url):
             'age_limit': (('hideContent', 'isAdultContent'), {lambda x: 18 if x else None}, any),
         })
 
-        m3u8_url = traverse_obj(video_info, ('canView', 'url'))
+        m3u8_url = traverse_obj(video_info, ('canView', 'url', {url_or_none}))
         if not m3u8_url:
             msg = 'You do not have access to this video'
             if traverse_obj(video_info, ('viewableTiers', 'free')):
-                msg += '. This video requires a free subscription'
+                msg = 'This video requires a free subscription to access'
             if not self.user_id:
                 msg += f'. {self._login_hint(method="password")}'
-            raise ExtractorError(msg)
+            raise ExtractorError(msg, expected=True)
 
-        thumbnail_key = traverse_obj(video_info, ('streamables', lambda _, v: v['type'].startswith('image/'), 's3key', any))
+        thumbnail_key = traverse_obj(video_info, (
+            'streamables', lambda _, v: v['type'].startswith('image/'), 's3key', any))
         if thumbnail_key:
             metainfo['thumbnail'] = url_or_none(self._download_webpage(
                 'https://api.rplay.live/upload/privateasset', video_id, 'getting cover url', query={
                     'key': thumbnail_key,
                     'contentOid': video_id,
                     'creatorOid': metainfo.get('uploader_id'),
-                    **({
+                    **self.requestor_query,
-                        'requestorOid': self.user_id,
-                        'loginType': self.login_type,
-                    } if self.user_id else {}),
                 }, fatal=False))
 
-        formats = self._extract_m3u8_formats(m3u8_url, video_id, headers={**headers, 'Butter': self.get_butter_token()})
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, headers={
+            'Referer': 'https://rplay.live/', 'Butter': self.get_butter_token()})
         for fmt in formats:
-            m3u8_doc = self._download_webpage(fmt['url'], video_id, 'getting m3u8 contents',
+            m3u8_doc = self._download_webpage(fmt['url'], video_id, 'getting m3u8 contents', headers={
-                                              headers={**headers, 'Butter': self.get_butter_token()})
+                'Referer': 'https://rplay.live/', 'Butter': self.get_butter_token()})
             fmt['url'] = encode_data_uri(m3u8_doc.encode(), 'application/x-mpegurl')
             match = re.search(r'^#EXT-X-KEY.*?URI="([^"]+)"', m3u8_doc, flags=re.M)
             if match:
                 urlh = self._request_webpage(match[1], video_id, 'getting hls key', headers={
-                    **headers,
+                    'Referer': 'https://rplay.live/',
                     'rplay-private-content-requestor': self.user_id or 'not-logged-in',
                     'age': random.randint(1, 4999),
                 })
@@ -262,11 +280,11 @@ def _real_extract(self, url):
             'id': video_id,
             'formats': formats,
             **metainfo,
-            'http_headers': {'Origin': 'https://rplay.live', 'Referer': 'https://rplay.live/'},
+            'http_headers': {'Referer': 'https://rplay.live/'},
         }
 
 
-class RPlayUserIE(RPlayBaseIE):
+class RPlayUserIE(InfoExtractor):
     _VALID_URL = r'https://rplay.live/(?P<short>c|creatorhome)/(?P<id>[\d\w]+)/?(?:[#?]|$)'
     _TESTS = [{
         'url': 'https://rplay.live/creatorhome/667adc9e9aa7f739a2158ff3?page=contents',
@@ -284,10 +302,6 @@ class RPlayUserIE(RPlayBaseIE):
         'playlist_mincount': 77,
     }]
 
-    def _perform_login(self, username, password):
-        # This playlist extractor does not require login
-        return
-
     def _real_extract(self, url):
         user_id, short = self._match_valid_url(url).group('id', 'short')
         key = 'customUrl' if short == 'c' else 'userOid'
@@ -349,19 +363,15 @@ def _real_extract(self, url):
         if stream_state == 'youtube':
             return self.url_result(f'https://www.youtube.com/watch?v={live_info["liveStreamId"]}')
         elif stream_state == 'live':
-            if not self.user_id:
+            if not self.user_id and not live_info.get('allowAnonymous'):
                 self.raise_login_required(method='password')
             key2 = self._download_webpage(
                 'https://api.rplay.live/live/key2', user_id, 'getting live key',
-                headers={'Authorization': self.jwt_token},
+                headers=self.jwt_header, query=self.requestor_query) if self.user_id else ''
-                query={
+            formats = self._extract_m3u8_formats(
-                    'requestorOid': self.user_id,
+                'https://api.rplay.live/live/stream/playlist.m3u8', user_id,
-                    'loginType': self.login_type,
+                query={'creatorOid': user_id, 'key2': key2})
-                })
+
-            formats = self._extract_m3u8_formats('https://api.rplay.live/live/stream/playlist.m3u8', user_id, query={
-                'creatorOid': user_id,
-                'key2': key2,
-            })
             return {
                 'id': user_id,
                 'formats': formats,