import base64 import functools import hashlib import json import random import re import string import urllib.parse from .common import InfoExtractor from ..networking.exceptions import HTTPError from ..utils import ( ExtractorError, OnDemandPagedList, filter_dict, get_domain, int_or_none, parse_qs, unified_timestamp, update_url_query, url_or_none, urlencode_postdata, urljoin, ) from ..utils.traversal import traverse_obj class AuthManager: _AUTH_INFO = {} def __init__(self, ie: 'SheetaEmbedIE'): self._ie = ie self._auth_info = {} @property def _auth_info(self): if not self._AUTH_INFO.get(self._ie._DOMAIN): self._AUTH_INFO[self._ie._DOMAIN] = {} return self._AUTH_INFO.get(self._ie._DOMAIN) @_auth_info.setter def _auth_info(self, value): if not self._AUTH_INFO.get(self._ie._DOMAIN): self._AUTH_INFO[self._ie._DOMAIN] = {} self._AUTH_INFO[self._ie._DOMAIN].update(value) def _get_authed_info(self, query_path, item_id, dict_path, expected_code_msg, **query_kwargs): try: res = self._ie._call_api(query_path, item_id, **query_kwargs) return traverse_obj(res, dict_path) except ExtractorError as e: if not isinstance(e.cause, HTTPError) or e.cause.status not in expected_code_msg: raise e self._ie.raise_login_required( expected_code_msg[e.cause.status], metadata_available=True, method=self._auth_info.get('login_method')) return None def _get_auth_token(self): if not self._auth_info.get('auth_token'): try: self._login() return self._auth_info.get('auth_token') except Exception as e: raise ExtractorError('Unable to login due to unknown reasons') from e if self._auth_info.get('auth_token'): try: self._refresh_token() return self._auth_info.get('auth_token') except Exception as e: raise ExtractorError('Unable to refresh token due to unknown reasons') from e return None def _refresh_token(self): if not (refresh_func := self._auth_info.get('refresh_func')): return False res = self._ie._download_json( **refresh_func(self._auth_info), expected_status=(400, 403, 404), note='Refreshing token', errnote='Unable to refresh token') if error := traverse_obj( res, ('error', 'message', {lambda x: base64.b64decode(x).decode()}), ('error', 'message')): self._ie.report_warning(f'Unable to refresh token: {error!r}') elif token := traverse_obj(res, ('data', 'access_token', {str})): # niconico self._auth_info = {'auth_token': f'Bearer {token}'} return True elif token := traverse_obj(res, ('access_token', {str})): # auth0 self._auth_info = {'auth_token': f'Bearer {token}'} if refresh_token := traverse_obj(res, ('refresh_token', {str})): self._auth_info = {'refresh_token': refresh_token} self._ie.cache.store( self._ie._NETRC_MACHINE, self._auth_info['cache_key'], {self._auth_info['cache_name']: refresh_token}) return True self._ie.report_warning('Unable to find new refresh_token') else: self._ie.report_warning('Unable to refresh token') return False def _login(self): social_login_providers = traverse_obj(self._ie._call_api( f'fanclub_groups/{self._ie._FANCLUB_GROUP_ID}/login', None), ('data', 'fanclub_group', 'fanclub_social_login_providers', ..., {dict})) or [] self._ie.write_debug(f'social_login_providers = {social_login_providers!r}') for provider in social_login_providers: provider_name = traverse_obj(provider, ('social_login_provider', 'provider_name', {str})) if provider_name == 'ニコニコ': redirect_url = update_url_query(provider['url'], { 'client_id': 'FCS{:05d}'.format(provider['id']), 'redirect_uri': f'https://{self._ie._DOMAIN}/login', }) refresh_url = f'{self._ie._API_BASE_URL}/fanclub_groups/{self._ie._FANCLUB_GROUP_ID}/auth/refresh' return self._niconico_sns_login(redirect_url, refresh_url) else: raise ExtractorError(f'Unsupported social login provider: {provider_name}') return self._auth0_login() def _niconico_sns_login(self, redirect_url, refresh_url): self._auth_info = {'login_method': 'any'} mail_tel, password = self._ie._get_login_info() if not mail_tel: return cache_key = hashlib.sha1(f'{self._ie._DOMAIN}:{mail_tel}:{password}'.encode()).hexdigest() self._auth_info = {'cache_key': cache_key} cache_name = 'niconico_sns' if cached_cookies := traverse_obj(self._ie.cache.load( self._ie._NETRC_MACHINE, cache_key), (cache_name, {dict})): for name, value in cached_cookies.items(): self._ie._set_cookie(get_domain(redirect_url), name, value) if not (auth_token := self._niconico_get_token_by_cookies(redirect_url)): if cached_cookies: self._ie.cache.store(self._ie._NETRC_MACHINE, cache_key, None) self._niconico_login(mail_tel, password) if not (auth_token := self._niconico_get_token_by_cookies(redirect_url)): self._ie.report_warning('Unable to get token after login, please check if ' 'niconico channel plus is authorized to use your niconico account') return self._auth_info = { 'refresh_func': lambda data: { 'url_or_request': data['refresh_url'], 'video_id': None, 'headers': {'Authorization': data['auth_token']}, 'data': b'', }, 'refresh_url': refresh_url, 'auth_token': auth_token, } cookies = dict(traverse_obj(self._ie.cookiejar.get_cookies_for_url( redirect_url), (..., {lambda item: (item.name, item.value)}))) self._ie.cache.store(self._ie._NETRC_MACHINE, cache_key, {cache_name: cookies}) def _niconico_get_token_by_cookies(self, redirect_url): urlh = self._ie._request_webpage( redirect_url, None, note='Getting niconico auth status', expected_status=404, errnote='Unable to get niconico auth status') if not urlh.url.startswith(f'https://{self._DOMAIN}/login'): return None if not (sns_login_code := traverse_obj(parse_qs(urlh.url), ('code', 0))): self._ie.report_warning('Unable to get sns login code') return None token = traverse_obj(self._ie._call_api( f'fanclub_groups/{self._ie._FANCLUB_GROUP_ID}/sns_login', None, fatal=False, note='Fetching sns login info', errnote='Unable to fetch sns login info', data=json.dumps({ 'key_cloak_user': { 'code': sns_login_code, 'redirect_uri': f'https://{self._ie._DOMAIN}/login', }, 'fanclub_site': {'id': int(self._ie._FANCLUB_SITE_ID_AUTH)}, }).encode(), headers={ 'Content-Type': 'application/json', 'fc_use_device': 'null', 'Referer': f'https://{self._ie._DOMAIN}', }), ('data', 'access_token', {str})) if token: return f'Bearer {token}' self._ie.report_warning('Unable to get token from sns login info') return None def _niconico_login(self, mail_tel, password): login_form_strs = { 'mail_tel': mail_tel, 'password': password, } page, urlh = self._ie._download_webpage_handle( 'https://account.nicovideo.jp/login/redirector', None, note='Logging into niconico', errnote='Unable to log into niconico', data=urlencode_postdata(login_form_strs), headers={ 'Referer': 'https://account.nicovideo.jp/login', 'Content-Type': 'application/x-www-form-urlencoded', }) if urlh.url.startswith('https://account.nicovideo.jp/login'): self._ie.report_warning('Unable to log in: bad username or password') return False elif urlh.url.startswith('https://account.nicovideo.jp/mfa'): post_url = self._ie._search_regex( r']+action=(["\'])(?P.+?)\1', page, 'mfa post url', group='url') page, urlh = self._ie._download_webpage_handle( urljoin('https://account.nicovideo.jp/', post_url), None, note='Performing MFA', errnote='Unable to complete MFA', data=urlencode_postdata({ 'otp': self._ie._get_tfa_info('6 digits code'), }), headers={ 'Content-Type': 'application/x-www-form-urlencoded', }) if urlh.url.startswith('https://account.nicovideo.jp/mfa') or 'formError' in page: err_msg = self._ie._html_search_regex( r'formError\b[^>]*>(.*?)', page, 'form_error', default='There\'s an error but the message can\'t be parsed.', flags=re.DOTALL) self._ie.report_warning(f'Unable to log in: MFA challenge failed, "{err_msg}"') return False return True def _auth0_login(self): self._auth_info = {'login_method': 'password'} username, password = self._ie._get_login_info() if not username: return cache_key = hashlib.sha1(f'{self._ie._DOMAIN}:{username}:{password}'.encode()).hexdigest() cache_name = 'refresh' self._auth_info = { 'cache_key': cache_key, 'cache_name': cache_name, } login_info = self._ie._call_api(f'fanclub_sites/{self._ie._FANCLUB_SITE_ID_AUTH}/login', None)['data']['fanclub_site'] self._ie.write_debug(f'login_info = {login_info}') auth0_web_client_id = login_info['auth0_web_client_id'] auth0_domain = login_info['fanclub_group']['auth0_domain'] token_url = f'https://{auth0_domain}/oauth/token' redirect_url = f'https://{self._ie._DOMAIN}/login/login-redirect' auth0_client = base64.b64encode(json.dumps({ 'name': 'auth0-spa-js', 'version': '2.0.6', }).encode()).decode() self._auth_info = {'refresh_func': lambda data: { 'url_or_request': token_url, 'video_id': None, 'headers': {'Auth0-Client': auth0_client}, 'data': urlencode_postdata({ 'client_id': auth0_web_client_id, 'grant_type': 'refresh_token', 'refresh_token': data['refresh_token'], 'redirect_uri': redirect_url, }), }} def random_str(): return ''.join(random.choices(string.digits + string.ascii_letters, k=43)) state = base64.b64encode(random_str().encode()) nonce = base64.b64encode(random_str().encode()) code_verifier = random_str().encode() code_challenge = base64.b64encode( hashlib.sha256(code_verifier).digest()).decode().translate(self._ie._AUTH0_BASE64_TRANS) authorize_url = update_url_query(f'https://{auth0_domain}/authorize', { 'client_id': auth0_web_client_id, 'scope': 'openid profile email offline_access', 'redirect_uri': redirect_url, 'audience': f'api.{self._ie._DOMAIN}', 'prompt': 'login', 'response_type': 'code', 'response_mode': 'query', 'state': state, 'nonce': nonce, 'code_challenge': code_challenge, 'code_challenge_method': 'S256', 'auth0Client': auth0_client, }) if cached_refresh_token := traverse_obj(self._ie.cache.load( self._ie._NETRC_MACHINE, cache_key), (cache_name, {str})): self._auth_info = {'refresh_token': cached_refresh_token} if self._refresh_token(): self._ie.write_debug('cached tokens updated') return self._ie.cache.store(self._ie._NETRC_MACHINE, cache_key, None) login_form = self._ie._hidden_inputs(self._ie._download_webpage( authorize_url, None, note='Getting login form', errnote='Unable to get login form')) state_obtained = login_form['state'] login_url = f'https://{auth0_domain}/u/login?state={state_obtained}' login_form.update({ 'username': username, 'password': password, 'action': 'default', }) urlh = self._ie._request_webpage( login_url, None, note='Logging in', errnote='Unable to log in', data=urlencode_postdata(login_form), expected_status=(400, 404)) if urlh.status == 400: self._ie.report_warning('Unable to log in: bad username or password') return if not (urlh.status == 404 and urlh.url.startswith(redirect_url)): self._ie.report_warning('Unable to log in: Unknown login status') return code = parse_qs(urlh.url)['code'][0] token_json = self._ie._download_json( token_url, None, headers={'Auth0-Client': auth0_client}, note='Getting auth0 tokens', errnote='Unable to get auth0 tokens', data=urlencode_postdata({ 'client_id': auth0_web_client_id, 'code_verifier': code_verifier, 'grant_type': 'authorization_code', 'code': code, 'redirect_uri': redirect_url, })) access_token = token_json['access_token'] refresh_token = token_json['refresh_token'] auth_token = f'Bearer {access_token}' self._auth_info = { 'auth_token': auth_token, 'refresh_token': refresh_token, } self._ie.cache.store(self._NETRC_MACHINE, cache_key, {cache_name: refresh_token}) class SheetaEmbedIE(InfoExtractor): _NETRC_MACHINE = 'sheeta' IE_NAME = 'sheeta' IE_DESC = 'fan club system developed by DWANGO (ドワンゴ)' _VALID_URL = False _WEBPAGE_TESTS = [{ 'url': 'https://qlover.jp/doku/video/smy4caVHR6trSddiG9uCDiy4', 'info_dict': { 'id': 'smy4caVHR6trSddiG9uCDiy4', 'title': '名取さなの毒にも薬にもならないラジオ#39', 'ext': 'mp4', 'channel': '名取さなの毒にも薬にもならないラジオ', 'channel_id': 'qlover.jp/doku', 'channel_url': 'https://qlover.jp/doku', 'age_limit': 0, 'live_status': 'not_live', 'thumbnail': str, 'description': 'md5:75c2143a59b4b70141b77ddb485991fd', 'timestamp': 1711933200, 'duration': 1872, 'comment_count': int, 'view_count': int, 'tags': ['名取さな', 'どくラジ', '文化放送', 'ラジオ'], 'upload_date': '20240401', }, 'params': { 'skip_download': True, }, }, { 'url': 'https://itomiku-fc.jp/live/sm4P8x6oVPFBx59bNBGSgKoE', 'info_dict': { 'id': 'sm4P8x6oVPFBx59bNBGSgKoE', 'title': '【3/9(土)14:00～】「all yours」美来の日SP♪', 'ext': 'mp4', 'channel': '伊藤美来 Official Fanclub 「all yours」', 'channel_id': 'itomiku-fc.jp', 'channel_url': 'https://itomiku-fc.jp', 'age_limit': 0, 'live_status': 'was_live', 'thumbnail': str, 'description': 'md5:80a6a14db30d8506f70bec6a28a6c4ad', 'timestamp': 1709964399, 'duration': 4542, 'comment_count': int, 'view_count': int, 'tags': ['生放送', '生放送アーカイブ'], 'upload_date': '20240309', 'release_timestamp': 1709959800, 'release_date': '20240309', }, 'params': {'skip_download': True}, }, { 'url': 'https://canan8181.com/video/smxar9atjfNBn27bHhcTFLyg', 'info_dict': { 'id': 'smxar9atjfNBn27bHhcTFLyg', 'title': '💛【7月】ドネートお礼しながら感想＆どきどきトーク【感想会】', 'ext': 'mp4', 'channel': 'Canan official fanclub', 'channel_id': 'canan8181.com', 'channel_url': 'https://canan8181.com', 'age_limit': 15, 'live_status': 'was_live', 'thumbnail': str, 'description': 'md5:0cd80e51da82dbb89deae5ea14aad24d', 'timestamp': 1659182206, 'duration': 6997, 'comment_count': int, 'view_count': int, 'tags': ['安眠など♡アーカイブ＆動画（わらのおうちプラン以上）'], 'upload_date': '20220730', 'release_timestamp': 1659175200, 'release_date': '20220730', }, 'params': {'skip_download': True}, }, { 'url': 'https://11audee.jp/audio/smx3ebEZFRnHeaGzUzgi5A98', 'info_dict': { 'id': 'smx3ebEZFRnHeaGzUzgi5A98', 'title': '#相坂湯 第38回 ロコムジカちゃんの歌唱についてモノ申す！？ ある意味レアな？鼻声坂くん！', 'ext': 'm4a', 'channel': '相坂優歌 湯上がり何飲む？', 'channel_id': '11audee.jp', 'channel_url': 'https://11audee.jp', 'age_limit': 0, 'live_status': 'not_live', 'thumbnail': str, 'description': 'md5:fdf881191f8057aa6af6042fc17fb94c', 'timestamp': 1710860400, 'duration': 631, 'comment_count': int, 'view_count': int, 'tags': ['RADIO'], 'upload_date': '20240319', }, 'params': {'skip_download': True}, }, { 'url': 'https://hololive-fc.com/videos', 'info_dict': { 'id': 'hololive-fc.com/videos', 'title': '旧ホロライブ公式ファンクラブ-videos', 'age_limit': 0, 'timestamp': 1715652389, 'upload_date': '20240514', }, 'playlist_mincount': 12, }, { 'url': 'https://tokinosora-fc.com/videos?vodType=1', 'info_dict': { 'id': 'tokinosora-fc.com/videos', 'title': 'ときのそらオフィシャルファンクラブ-videos', 'age_limit': 0, 'timestamp': 1715652399, 'upload_date': '20240514', }, 'playlist_mincount': 18, }, { 'url': 'https://01audee.jp/videos?tag=RADIO&vodType=1&sort=display_date', 'info_dict': { 'id': '01audee.jp/videos', 'title': '大熊和奏 朝のささやき-videos', 'age_limit': 0, 'timestamp': 1715652369, 'upload_date': '20240514', }, 'playlist_mincount': 6, }, { 'url': 'https://qlover.jp/bokuao/lives', 'info_dict': { 'id': 'qlover.jp/bokuao/lives', 'title': '僕が見たかった青空の 「青天のヘキレキ！」-lives', 'age_limit': 0, 'timestamp': 1715652429, 'upload_date': '20240514', }, 'playlist_mincount': 1, }, { 'url': 'https://06audee.jp/lives', 'info_dict': { 'id': '06audee.jp/lives', 'title': '田中ちえ美のたなかのカナタ！-lives', 'age_limit': 0, 'timestamp': 1715652369, 'upload_date': '20240514', }, 'playlist_mincount': 5, }] _DOMAIN = None _API_BASE_URL = None _FANCLUB_GROUP_ID = None _FANCLUB_SITE_ID_AUTH = None _FANCLUB_SITE_ID_INFO = None _AUTH0_BASE64_TRANS = str.maketrans({ '+': '-', '/': '_', '=': None, }) _LIST_PAGE_SIZE = 12 auth_manager: AuthManager = None # @classmethod # def suitable(cls, url): # return ( # not any(ie.suitable(url) for ie in (ArteTVIE, ArteTVPlaylistIE)) # and super().suitable(url)) def _extract_from_url(self, url): parsed_url = urllib.parse.urlparse(url) if not self.auth_manager: self.auth_manager = AuthManager(self) if '/videos' in parsed_url.path: return self._extract_video_list_page(url) elif '/lives' in parsed_url.path: return self._extract_live_list_page(url) else: return self._extract_player_page(url) def _extract_from_webpage(self, url, webpage): if 'GTM-KXT7G5G' in webpage or 'NicoGoogleTagManagerDataLayer' in webpage: yield self._extract_from_url(url) raise self.StopExtraction def _call_api(self, path, item_id, *args, **kwargs): return self._download_json(f'{self._API_BASE_URL}/{path}', item_id, *args, **kwargs) def _find_fanclub_site_id(self, channel_id): fanclub_list_json = self._call_api( 'content_providers/channel_domain', f'channels/{channel_id}', query={'current_site_domain': urllib.parse.quote(f'https://{self._DOMAIN}/{channel_id}')}, note='Fetching channel list', errnote='Unable to fetch channel list', ) if fanclub_id := traverse_obj( fanclub_list_json, ('data', 'content_providers', 'id', {int_or_none}), get_all=False): return fanclub_id raise ExtractorError(f'Channel {channel_id} does not exist', expected=True) def _extract_base_info(self, channel_id): site_settings = self._download_json( f'https://{self._DOMAIN}/site/settings.json', None, note='Fetching site settings', errnote='Unable to fetch site settings') self.write_debug(f'site_settings = {site_settings!r}') self._API_BASE_URL = site_settings['api_base_url'] self._FANCLUB_GROUP_ID = site_settings['fanclub_group_id'] self._FANCLUB_SITE_ID_AUTH = site_settings['fanclub_site_id'] if channel_id: self._FANCLUB_SITE_ID_INFO = self._find_fanclub_site_id(channel_id) else: self._FANCLUB_SITE_ID_INFO = self._FANCLUB_SITE_ID_AUTH @property def _channel_base_info(self): return traverse_obj(self._call_api( f'fanclub_sites/{self._FANCLUB_SITE_ID_INFO}/page_base_info', f'fanclub_sites/{self._FANCLUB_SITE_ID_INFO}', note='Fetching channel base info', errnote='Unable to fetch channel base info', fatal=False, ), ('data', 'fanclub_site', {dict})) or {} @property def _channel_user_info(self): return traverse_obj(self._call_api( f'fanclub_sites/{self._FANCLUB_SITE_ID_INFO}/user_info', f'fanclub_sites/{self._FANCLUB_SITE_ID_INFO}', note='Fetching channel user info', errnote='Unable to fetch channel user info', fatal=False, data=json.dumps('null').encode(), ), ('data', 'fanclub_site', {dict})) or {} def _extract_channel_info(self, channel_id): if channel_id: full_channel_id = f'{self._DOMAIN}/{channel_id}' channel_url = f'https://{self._DOMAIN}/{channel_id}' else: full_channel_id = self._DOMAIN channel_url = f'https://{self._DOMAIN}' return { 'channel': self._channel_base_info.get('fanclub_site_name'), 'channel_id': full_channel_id, 'channel_url': channel_url, 'age_limit': traverse_obj(self._channel_user_info, ( 'content_provider', 'age_limit', {int_or_none})), } def _extract_player_page(self, url): self._DOMAIN, channel_id, content_code = re.match( r'https?://(?P[\w.-]+)(/(?P[\w.-]+))?/(?:live|video|audio)/(?Psm\w+)', url, ).group('domain', 'channel', 'code') self._extract_base_info(channel_id) data_json = self._call_api( f'video_pages/{content_code}', content_code, headers={'fc_use_device': 'null'}, note='Fetching video page info', errnote='Unable to fetch video page info', )['data']['video_page'] live_status = self._get_live_status(data_json, content_code) formats = self._get_formats(data_json, live_status, content_code) release_timestamp_str = data_json.get('live_scheduled_start_at') if live_status == 'is_upcoming': if release_timestamp_str: msg = f'This live event will begin at {release_timestamp_str} UTC' else: msg = 'This event has not started yet' self.raise_no_formats(msg, expected=True, video_id=content_code) return { 'id': content_code, 'formats': formats, 'live_status': live_status, 'release_timestamp': unified_timestamp(release_timestamp_str), **self._extract_channel_info(channel_id), **traverse_obj(data_json, { 'title': ('title', {str}), 'thumbnail': ('thumbnail_url', {url_or_none}), 'description': ('description', {str}), 'timestamp': ('display_date', {unified_timestamp}), 'duration': ('active_video_filename', 'length', {int_or_none}), 'comment_count': ('video_aggregate_info', 'number_of_comments', {int_or_none}), 'view_count': ('video_aggregate_info', 'total_views', {int_or_none}), 'tags': ('video_tags', ..., 'tag', {str}), }), '__post_extractor': self.extract_comments( content_code=content_code, comment_group_id=traverse_obj(data_json, ('video_comment_setting', 'comment_group_id'))), } def _get_comments(self, content_code, comment_group_id): item_id = f'{content_code}/comments' if not comment_group_id: return None comment_access_token = self._call_api( f'video_pages/{content_code}/comments_user_token', item_id, note='Getting comment token', errnote='Unable to get comment token', )['data']['access_token'] comment_list, urlh = self._download_json_handle( 'https://comm-api.sheeta.com/messages.history', video_id=item_id, note='Fetching comments', errnote='Unable to fetch comments', headers={'Content-Type': 'application/json'}, expected_status=404, query={ 'sort_direction': 'asc', 'limit': int_or_none(self._configuration_arg('max_comments', [''])[0]) or 120, }, data=json.dumps({ 'token': comment_access_token, 'group_id': comment_group_id, }).encode()) if urlh.status == 404: self.report_warning('Unable to fetch comments due to rate limit', content_code) return for comment in traverse_obj(comment_list, ...): yield traverse_obj(comment, { 'author': ('nickname', {str}), 'author_id': ('sender_id', {str}), 'id': ('id', {str}, {lambda x: x or None}), 'text': ('message', {str}), 'timestamp': (('updated_at', 'sent_at', 'created_at'), {unified_timestamp}), 'author_is_uploader': ('sender_id', {lambda x: x == '-1'}), }, get_all=False) def _get_live_status(self, data_json, content_code): video_type = data_json.get('type') live_finished_at = data_json.get('live_finished_at') if video_type == 'vod': if live_finished_at: live_status = 'was_live' else: live_status = 'not_live' elif video_type == 'live': if not data_json.get('live_started_at'): return 'is_upcoming' if not live_finished_at: live_status = 'is_live' else: live_status = 'was_live' video_allow_dvr_flg = traverse_obj(data_json, ('video', 'allow_dvr_flg')) video_convert_to_vod_flg = traverse_obj(data_json, ('video', 'convert_to_vod_flg')) self.write_debug( f'{content_code}: allow_dvr_flg = {video_allow_dvr_flg}, convert_to_vod_flg = {video_convert_to_vod_flg}.') if not (video_allow_dvr_flg and video_convert_to_vod_flg): raise ExtractorError( 'Live was ended, there is no video for download', video_id=content_code, expected=True) else: raise ExtractorError(f'Unknown type: {video_type!r}', video_id=content_code) self.write_debug(f'{content_code}: video_type={video_type}, live_status={live_status}') return live_status def _get_formats(self, data_json, live_status, content_code): headers = filter_dict({ 'Content-Type': 'application/json', 'fc_use_device': 'null', 'origin': f'https://{self._DOMAIN}', 'Authorization': self.auth_manager._get_auth_token(), }) formats = [] if data_json.get('video'): payload = {} if data_json.get('type') == 'live' and live_status == 'was_live': payload = {'broadcast_type': 'dvr'} session_id = self.auth_manager._get_authed_info( f'video_pages/{content_code}/session_ids', f'{content_code}/session', ('data', 'session_id', {str}), { 401: 'Members-only content', 403: 'Login required', 408: 'Outdated token', }, data=json.dumps(payload).encode(), headers=headers, note='Getting session id', errnote='Unable to get session id') if session_id: m3u8_url = data_json['video_stream']['authenticated_url'].format(session_id=session_id) formats = self._extract_m3u8_formats(m3u8_url, content_code) elif data_json.get('audio'): m3u8_url = self.auth_manager._get_authed_info( f'video_pages/{content_code}/content_access', f'{content_code}/content_access', ('data', 'resource', {url_or_none}), { 403: 'Login required', 404: 'Members-only content', 408: 'Outdated token', }, headers=headers, note='Getting content resource', errnote='Unable to get content resource') if m3u8_url: audio_type = traverse_obj(data_json, ( 'audio_filename_transcoded_list', lambda _, v: v['url'] == m3u8_url, 'video_filename_type', 'value', {str}), get_all=False) if audio_type == 'audio_free': # fully free audios are always of "audio_paid" msg = 'You have no right to access the paid content. ' if traverse_obj(data_json, 'video_free_periods'): msg += 'There may be some silent parts in this audio' else: msg += 'This audio may be completely blank' self.raise_login_required( msg, metadata_available=True, method=self.auth_manager._auth_info.get('login_method')) formats = [{ 'url': m3u8_url, 'format_id': 'audio', 'protocol': 'm3u8_native', 'ext': 'm4a', 'vcodec': 'none', 'acodec': 'aac', 'format_note': audio_type, }] else: raise ExtractorError('Unknown media type', video_id=content_code) return formats def _fetch_paged_channel_video_list(self, path, query, channel, item_id, page): response = self._call_api( path, item_id, query={ **query, 'page': (page + 1), 'per_page': self._LIST_PAGE_SIZE, }, headers={'fc_use_device': 'null'}, note=f'Fetching channel info (page {page + 1})', errnote=f'Unable to fetch channel info (page {page + 1})') for content_code in traverse_obj( response, ('data', 'video_pages', 'list', ..., 'content_code', {str})): yield self.url_result('/'.join(filter( None, [f'https://{self._DOMAIN}', channel, 'video', content_code]))) def _extract_video_list_page(self, url): """ API parameters: sort: -display_date 公開日が新しい順 (newest to oldest) display_date 公開日が古い順 (oldest to newest) -number_of_vod_views 再生数が多い順 (most play count) number_of_vod_views コメントが多い順 (most comments) vod_type (is "vodType" in "url"): 0 すべて (all) 1 会員限定 (members only) 2 一部無料 (partially free) 3 レンタル (rental) 4 生放送アーカイブ (live archives) 5 アップロード動画 (uploaded videos) 7 無料 (free) """ self._DOMAIN, channel_id = re.match( r'https?://(?P[\w.-]+)(/(?P[\w.-]+))?/videos', url, ).group('domain', 'channel') self._extract_base_info(channel_id) channel_info = self._extract_channel_info(channel_id) full_channel_id = channel_info['channel_id'] channel_name = channel_info['channel'] qs = parse_qs(url) return self.playlist_result( OnDemandPagedList( functools.partial( self._fetch_paged_channel_video_list, f'fanclub_sites/{self._FANCLUB_SITE_ID_INFO}/video_pages', filter_dict({ 'tag': traverse_obj(qs, ('tag', 0)), 'sort': traverse_obj(qs, ('sort', 0), default='-display_date'), 'vod_type': traverse_obj(qs, ('vodType', 0), default='0'), }), channel_id, f'{full_channel_id}/videos'), self._LIST_PAGE_SIZE), playlist_id=f'{full_channel_id}/videos', playlist_title=f'{channel_name}-videos') def _extract_live_list_page(self, url): """ API parameters: live_type: 1 放送中 (on air) 2 放送予定 (scheduled live streams, oldest to newest) 3 過去の放送 - すべて (all ended live streams, newest to oldest) 4 過去の放送 - 生放送アーカイブ (all archives for live streams, oldest to newest) We use "4" instead of "3" because some recently ended live streams could not be downloaded. """ self._DOMAIN, channel_id = re.match( r'https?://(?P[\w.-]+)(/(?P[\w.-]+))?/lives', url, ).group('domain', 'channel') self._extract_base_info(channel_id) channel_info = self._extract_channel_info(channel_id) full_channel_id = channel_info['channel_id'] channel_name = channel_info['channel'] return self.playlist_result( OnDemandPagedList( functools.partial( self._fetch_paged_channel_video_list, f'fanclub_sites/{self._FANCLUB_SITE_ID_INFO}/live_pages', {'live_type': 4}, channel_id, f'{full_channel_id}/lives'), self._LIST_PAGE_SIZE), playlist_id=f'{full_channel_id}/lives', playlist_title=f'{channel_name}-lives')