diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.yml b/.github/ISSUE_TEMPLATE/1_broken_site.yml index 20e5e944f..c8d3de06b 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.yml +++ b/.github/ISSUE_TEMPLATE/1_broken_site.yml @@ -2,13 +2,11 @@ name: Broken site support description: Report issue with yt-dlp on a supported site labels: [triage, site-bug] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: checkboxes id: checklist attributes: @@ -24,9 +22,7 @@ body: required: true - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar issues **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and I'm willing to share it if required - type: input @@ -47,6 +43,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true @@ -78,11 +76,3 @@ body: render: shell validations: required: true - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.yml b/.github/ISSUE_TEMPLATE/2_site_support_request.yml index 4aeff7dc6..a9564c0c2 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.yml +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.yml @@ -2,13 +2,11 @@ name: Site support request description: Request support for a new site labels: [triage, site-request] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: checkboxes id: checklist attributes: @@ -24,9 +22,7 @@ body: required: true - label: I've checked that none of provided URLs [violate any copyrights](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy) or contain any [DRM](https://en.wikipedia.org/wiki/Digital_rights_management) to the best of my knowledge required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and am willing to share it if required - type: input @@ -59,6 +55,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true @@ -90,11 +88,3 @@ body: render: shell validations: required: true - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml index 2f516ebb7..6e2380fae 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.yml +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.yml @@ -1,14 +1,12 @@ name: Site feature request -description: Request a new functionality for a supported site +description: Request new functionality for a site supported by yt-dlp labels: [triage, site-enhancement] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: checkboxes id: checklist attributes: @@ -22,9 +20,7 @@ body: required: true - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and I'm willing to share it if required - type: input @@ -55,6 +51,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true @@ -86,11 +84,3 @@ body: render: shell validations: required: true - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.yml b/.github/ISSUE_TEMPLATE/4_bug_report.yml index 201586e9d..6fc523be0 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/4_bug_report.yml @@ -2,13 +2,11 @@ name: Core bug report description: Report a bug unrelated to any particular site or extractor labels: [triage, bug] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: checkboxes id: checklist attributes: @@ -20,13 +18,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details - required: true - - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) - required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar issues **including closed ones**. DO NOT post duplicates required: true - type: textarea id: description @@ -40,6 +32,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true @@ -71,11 +65,3 @@ body: render: shell validations: required: true - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.yml b/.github/ISSUE_TEMPLATE/5_feature_request.yml index 765de86a2..57a33bb71 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.yml +++ b/.github/ISSUE_TEMPLATE/5_feature_request.yml @@ -1,14 +1,12 @@ name: Feature request -description: Request a new functionality unrelated to any particular site or extractor +description: Request a new feature unrelated to any particular site or extractor labels: [triage, enhancement] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: checkboxes id: checklist attributes: @@ -22,9 +20,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - type: textarea id: description @@ -38,6 +34,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead" @@ -65,11 +63,3 @@ body: [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc render: shell - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/6_question.yml b/.github/ISSUE_TEMPLATE/6_question.yml index 198e21bec..28ec7cbe0 100644 --- a/.github/ISSUE_TEMPLATE/6_question.yml +++ b/.github/ISSUE_TEMPLATE/6_question.yml @@ -1,14 +1,12 @@ name: Ask question -description: Ask yt-dlp related question +description: Ask a question about using yt-dlp labels: [question] body: - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. - type: markdown attributes: value: | @@ -28,9 +26,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%3Aissue%20-label%3Aspam%20%20) for similar questions **including closed ones**. DO NOT post duplicates required: true - type: textarea id: question @@ -44,6 +40,8 @@ body: id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) - label: "If using API, add `'verbose': True` to `YoutubeDL` params instead" @@ -71,11 +69,3 @@ body: [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc render: shell - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 9cdffa4b1..0131631bb 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,8 +1,5 @@ blank_issues_enabled: false contact_links: - - name: Get help from the community on Discord + - name: Get help on Discord url: https://discord.gg/H5MNcFW63r - about: Join the yt-dlp Discord for community-powered support! - - name: Matrix Bridge to the Discord server - url: https://matrix.to/#/#yt-dlp:matrix.org - about: For those who do not want to use Discord + about: Join the yt-dlp Discord server for support and discussion diff --git a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml index bff28ae4e..f1a2d3090 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml @@ -18,9 +18,7 @@ body: required: true - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar issues **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and I'm willing to share it if required - type: input diff --git a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml index 2bffe738d..31b89b683 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml @@ -18,9 +18,7 @@ body: required: true - label: I've checked that none of provided URLs [violate any copyrights](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy) or contain any [DRM](https://en.wikipedia.org/wiki/Digital_rights_management) to the best of my knowledge required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and am willing to share it if required - type: input diff --git a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml index 6c3127983..421766a75 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml @@ -1,5 +1,5 @@ name: Site feature request -description: Request a new functionality for a supported site +description: Request new functionality for a site supported by yt-dlp labels: [triage, site-enhancement] body: %(no_skip)s @@ -16,9 +16,7 @@ body: required: true - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - label: I've read about [sharing account credentials](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#are-you-willing-to-share-account-details-if-needed) and I'm willing to share it if required - type: input diff --git a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml index 5f357d96e..31a19b292 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml @@ -14,13 +14,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details - required: true - - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#video-url-contains-an-ampersand--and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command) - required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar issues **including closed ones**. DO NOT post duplicates required: true - type: textarea id: description diff --git a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml index 99107ff58..b8ab6610b 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml @@ -1,5 +1,5 @@ name: Feature request -description: Request a new functionality unrelated to any particular site or extractor +description: Request a new feature unrelated to any particular site or extractor labels: [triage, enhancement] body: %(no_skip)s @@ -16,9 +16,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar requests **including closed ones**. DO NOT post duplicates required: true - type: textarea id: description diff --git a/.github/ISSUE_TEMPLATE_tmpl/6_question.yml b/.github/ISSUE_TEMPLATE_tmpl/6_question.yml index bd742109a..062e96321 100644 --- a/.github/ISSUE_TEMPLATE_tmpl/6_question.yml +++ b/.github/ISSUE_TEMPLATE_tmpl/6_question.yml @@ -1,5 +1,5 @@ name: Ask question -description: Ask yt-dlp related question +description: Ask a question about using yt-dlp labels: [question] body: %(no_skip)s @@ -22,9 +22,7 @@ body: required: true - label: I've verified that I have **updated yt-dlp to nightly or master** ([update instructions](https://github.com/yt-dlp/yt-dlp#update-channels)) required: true - - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766) and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates - required: true - - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue) + - label: I've searched [known issues](https://github.com/yt-dlp/yt-dlp/issues/3766), [the FAQ](https://github.com/yt-dlp/yt-dlp/wiki/FAQ), and the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=is%%3Aissue%%20-label%%3Aspam%%20%%20) for similar questions **including closed ones**. DO NOT post duplicates required: true - type: textarea id: question diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4deee572f..4dcfcc48c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,14 +1,17 @@ -**IMPORTANT**: PRs without the template will be CLOSED + ### Description of your *pull request* and other information - - -ADD DESCRIPTION HERE +ADD DETAILED DESCRIPTION HERE Fixes # @@ -16,24 +19,22 @@ ### Description of your *pull request* and other information
Template ### Before submitting a *pull request* make sure you have: - [ ] At least skimmed through [contributing guidelines](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions) including [yt-dlp coding conventions](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#yt-dlp-coding-conventions) - [ ] [Searched](https://github.com/yt-dlp/yt-dlp/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests -### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check all of the following options that apply: -- [ ] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/) -- [ ] I am not the original author of this code but it is in public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence) +### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check those that apply and remove the others: +- [ ] I am the original author of the code in this PR, and I am willing to release it under [Unlicense](http://unlicense.org/) +- [ ] I am not the original author of the code in this PR, but it is in the public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence) -### What is the purpose of your *pull request*? +### What is the purpose of your *pull request*? Check those that apply and remove the others: - [ ] Fix or improvement to an extractor (Make sure to add/update tests) - [ ] New extractor ([Piracy websites will not be accepted](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy)) - [ ] Core bug fix/improvement diff --git a/README.md b/README.md index 45c56434a..0ac27c462 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ [![Release version](https://img.shields.io/github/v/release/yt-dlp/yt-dlp?color=brightgreen&label=Download&style=for-the-badge)](#installation "Installation") [![PyPI](https://img.shields.io/badge/-PyPI-blue.svg?logo=pypi&labelColor=555555&style=for-the-badge)](https://pypi.org/project/yt-dlp "PyPI") [![Donate](https://img.shields.io/badge/_-Donate-red.svg?logo=githubsponsors&labelColor=555555&style=for-the-badge)](Collaborators.md#collaborators "Donate") -[![Matrix](https://img.shields.io/matrix/yt-dlp:matrix.org?color=brightgreen&labelColor=555555&label=&logo=element&style=for-the-badge)](https://matrix.to/#/#yt-dlp:matrix.org "Matrix") [![Discord](https://img.shields.io/discord/807245652072857610?color=blue&labelColor=555555&label=&logo=discord&style=for-the-badge)](https://discord.gg/H5MNcFW63r "Discord") [![Supported Sites](https://img.shields.io/badge/-Supported_Sites-brightgreen.svg?style=for-the-badge)](supportedsites.md "Supported Sites") [![License: Unlicense](https://img.shields.io/badge/-Unlicense-blue.svg?style=for-the-badge)](LICENSE "License") diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py index 2a418ddbf..110fcc245 100644 --- a/devscripts/make_issue_template.py +++ b/devscripts/make_issue_template.py @@ -11,11 +11,13 @@ from devscripts.utils import get_filename_args, read_file, write_file -VERBOSE_TMPL = ''' +VERBOSE = ''' - type: checkboxes id: verbose attributes: label: Provide verbose output that clearly demonstrates the problem + description: | + This is mandatory unless absolutely impossible to provide. If you are unable to provide the output, please explain why. options: - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) required: true @@ -47,31 +49,23 @@ render: shell validations: required: true - - type: markdown - attributes: - value: | - > [!CAUTION] - > ### GitHub is experiencing a high volume of malicious spam comments. - > ### If you receive any replies asking you download a file, do NOT follow the download links! - > - > Note that this issue may be temporarily locked as an anti-spam measure after it is opened. '''.strip() NO_SKIP = ''' - - type: checkboxes + - type: markdown attributes: - label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE - description: Fill all fields even if you think it is irrelevant for the issue - options: - - label: I understand that I will be **blocked** if I *intentionally* remove or skip any mandatory\\* field - required: true + value: | + > [!IMPORTANT] + > Not providing the required (*) information or removing the template will result in your issue being closed and ignored. '''.strip() def main(): - fields = {'no_skip': NO_SKIP} - fields['verbose'] = VERBOSE_TMPL % fields - fields['verbose_optional'] = re.sub(r'(\n\s+validations:)?\n\s+required: true', '', fields['verbose']) + fields = { + 'no_skip': NO_SKIP, + 'verbose': VERBOSE, + 'verbose_optional': re.sub(r'(\n\s+validations:)?\n\s+required: true', '', VERBOSE), + } infile, outfile = get_filename_args(has_infile=True) write_file(outfile, read_file(infile) % fields) diff --git a/test/helper.py b/test/helper.py index c776e70b7..193019019 100644 --- a/test/helper.py +++ b/test/helper.py @@ -237,6 +237,20 @@ def sanitize(key, value): def expect_info_dict(self, got_dict, expected_dict): + ALLOWED_KEYS_SORT_ORDER = ( + # NB: Keep in sync with the docstring of extractor/common.py + 'id', 'ext', 'direct', 'display_id', 'title', 'alt_title', 'description', 'media_type', + 'uploader', 'uploader_id', 'uploader_url', 'channel', 'channel_id', 'channel_url', 'channel_is_verified', + 'channel_follower_count', 'comment_count', 'view_count', 'concurrent_view_count', + 'like_count', 'dislike_count', 'repost_count', 'average_rating', 'age_limit', 'duration', 'thumbnail', 'heatmap', + 'chapters', 'chapter', 'chapter_number', 'chapter_id', 'start_time', 'end_time', 'section_start', 'section_end', + 'categories', 'tags', 'cast', 'composers', 'artists', 'album_artists', 'creators', 'genres', + 'track', 'track_number', 'track_id', 'album', 'album_type', 'disc_number', + 'series', 'series_id', 'season', 'season_number', 'season_id', 'episode', 'episode_number', 'episode_id', + 'timestamp', 'upload_date', 'release_timestamp', 'release_date', 'release_year', 'modified_timestamp', 'modified_date', + 'playable_in_embed', 'availability', 'live_status', 'location', 'license', '_old_archive_ids', + ) + expect_dict(self, got_dict, expected_dict) # Check for the presence of mandatory fields if got_dict.get('_type') not in ('playlist', 'multi_video'): @@ -252,7 +266,13 @@ def expect_info_dict(self, got_dict, expected_dict): test_info_dict = sanitize_got_info_dict(got_dict) - missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys()) + # Check for invalid/misspelled field names being returned by the extractor + invalid_keys = sorted(test_info_dict.keys() - ALLOWED_KEYS_SORT_ORDER) + self.assertFalse(invalid_keys, f'Invalid fields returned by the extractor: {", ".join(invalid_keys)}') + + missing_keys = sorted( + test_info_dict.keys() - expected_dict.keys(), + key=lambda x: ALLOWED_KEYS_SORT_ORDER.index(x)) if missing_keys: def _repr(v): if isinstance(v, str): diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index 06840ed85..ed3ca61c4 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -9,7 +9,7 @@ import math -from yt_dlp.jsinterp import JS_Undefined, JSInterpreter +from yt_dlp.jsinterp import JS_Undefined, JSInterpreter, js_number_to_string class NaN: @@ -93,6 +93,16 @@ def test_operators(self): self._test('function f(){return 0 ?? 42;}', 0) self._test('function f(){return "life, the universe and everything" < 42;}', False) self._test('function f(){return 0 - 7 * - 6;}', 42) + self._test('function f(){return true << "5";}', 32) + self._test('function f(){return true << true;}', 2) + self._test('function f(){return "19" & "21.9";}', 17) + self._test('function f(){return "19" & false;}', 0) + self._test('function f(){return "11.0" >> "2.1";}', 2) + self._test('function f(){return 5 ^ 9;}', 12) + self._test('function f(){return 0.0 << NaN}', 0) + self._test('function f(){return null << undefined}', 0) + # TODO: Does not work due to number too large + # self._test('function f(){return 21 << 4294967297}', 42) def test_array_access(self): self._test('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}', [5, 2, 7]) @@ -431,6 +441,27 @@ def test_slice(self): self._test('function f(){return "012345678".slice(-1, 1)}', '') self._test('function f(){return "012345678".slice(-3, -1)}', '67') + def test_js_number_to_string(self): + for test, radix, expected in [ + (0, None, '0'), + (-0, None, '0'), + (0.0, None, '0'), + (-0.0, None, '0'), + (math.nan, None, 'NaN'), + (-math.nan, None, 'NaN'), + (math.inf, None, 'Infinity'), + (-math.inf, None, '-Infinity'), + (10 ** 21.5, 8, '526665530627250154000000'), + (6, 2, '110'), + (254, 16, 'fe'), + (-10, 2, '-1010'), + (-0xff, 2, '-11111111'), + (0.1 + 0.2, 16, '0.4cccccccccccd'), + (1234.1234, 10, '1234.1234'), + # (1000000000000000128, 10, '1000000000000000100') + ]: + assert js_number_to_string(test, radix) == expected + if __name__ == '__main__': unittest.main() diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 13436f088..7ae627f2c 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -201,6 +201,10 @@ 'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js', 'YWt1qdbe8SAfkoPHW5d', 'RrRjWQOJmBiP', ), + ( + 'https://www.youtube.com/s/player/9c6dfc4a/player_ias.vflset/en_US/base.js', + 'jbu7ylIosQHyJyJV', 'uwI0ESiynAmhNg', + ), ] diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index b7b19cf6e..3293a9076 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -598,7 +598,7 @@ class YoutubeDL: # NB: Keep in sync with the docstring of extractor/common.py 'url', 'manifest_url', 'manifest_stream_number', 'ext', 'format', 'format_id', 'format_note', 'width', 'height', 'aspect_ratio', 'resolution', 'dynamic_range', 'tbr', 'abr', 'acodec', 'asr', 'audio_channels', - 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns', + 'vbr', 'fps', 'vcodec', 'container', 'filesize', 'filesize_approx', 'rows', 'columns', 'hls_media_playlist_data', 'player_url', 'protocol', 'fragment_base_url', 'fragments', 'is_from_start', 'is_dash_periods', 'request_data', 'preference', 'language', 'language_preference', 'quality', 'source_preference', 'cookies', 'http_headers', 'stretched_ratio', 'no_resume', 'has_drm', 'extra_param_to_segment_url', 'extra_param_to_key_url', diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py index da2574da7..7a47f8f83 100644 --- a/yt_dlp/downloader/hls.py +++ b/yt_dlp/downloader/hls.py @@ -72,11 +72,15 @@ def check_results(): def real_download(self, filename, info_dict): man_url = info_dict['url'] - self.to_screen(f'[{self.FD_NAME}] Downloading m3u8 manifest') - urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url)) - man_url = urlh.url - s = urlh.read().decode('utf-8', 'ignore') + s = info_dict.get('hls_media_playlist_data') + if s: + self.to_screen(f'[{self.FD_NAME}] Using m3u8 manifest from extracted info') + else: + self.to_screen(f'[{self.FD_NAME}] Downloading m3u8 manifest') + urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url)) + man_url = urlh.url + s = urlh.read().decode('utf-8', 'ignore') can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None if can_download: @@ -177,6 +181,7 @@ def is_ad_fragment_end(s): if external_aes_iv: external_aes_iv = binascii.unhexlify(remove_start(external_aes_iv, '0x').zfill(32)) byte_range = {} + byte_range_offset = 0 discontinuity_count = 0 frag_index = 0 ad_frag_next = False @@ -204,6 +209,11 @@ def is_ad_fragment_end(s): }) media_sequence += 1 + # If the byte_range is truthy, reset it after appending a fragment that uses it + if byte_range: + byte_range_offset = byte_range['end'] + byte_range = {} + elif line.startswith('#EXT-X-MAP'): if format_index and discontinuity_count != format_index: continue @@ -217,10 +227,12 @@ def is_ad_fragment_end(s): if extra_segment_query: frag_url = update_url_query(frag_url, extra_segment_query) + map_byte_range = {} + if map_info.get('BYTERANGE'): splitted_byte_range = map_info.get('BYTERANGE').split('@') - sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end'] - byte_range = { + sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else 0 + map_byte_range = { 'start': sub_range_start, 'end': sub_range_start + int(splitted_byte_range[0]), } @@ -229,7 +241,7 @@ def is_ad_fragment_end(s): 'frag_index': frag_index, 'url': frag_url, 'decrypt_info': decrypt_info, - 'byte_range': byte_range, + 'byte_range': map_byte_range, 'media_sequence': media_sequence, }) media_sequence += 1 @@ -257,7 +269,7 @@ def is_ad_fragment_end(s): media_sequence = int(line[22:]) elif line.startswith('#EXT-X-BYTERANGE'): splitted_byte_range = line[17:].split('@') - sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range['end'] + sub_range_start = int(splitted_byte_range[1]) if len(splitted_byte_range) == 2 else byte_range_offset byte_range = { 'start': sub_range_start, 'end': sub_range_start + int(splitted_byte_range[0]), diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 4f2627ca7..4d1bf1b05 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -454,7 +454,10 @@ CuriosityStreamIE, CuriosityStreamSeriesIE, ) -from .cwtv import CWTVIE +from .cwtv import ( + CWTVIE, + CWTVMovieIE, +) from .cybrary import ( CybraryCourseIE, CybraryIE, @@ -505,6 +508,7 @@ from .dhm import DHMIE from .digitalconcerthall import DigitalConcertHallIE from .digiteka import DigitekaIE +from .digiview import DigiviewIE from .discogs import DiscogsReleasePlaylistIE from .disney import DisneyIE from .dispeak import DigitallySpeakingIE diff --git a/yt_dlp/extractor/acast.py b/yt_dlp/extractor/acast.py index 8f4a2cf0f..eb467cb75 100644 --- a/yt_dlp/extractor/acast.py +++ b/yt_dlp/extractor/acast.py @@ -43,14 +43,14 @@ class ACastIE(ACastBaseIE): _VALID_URL = r'''(?x: https?:// (?: - (?:(?:embed|www)\.)?acast\.com/| + (?:(?:embed|www|shows)\.)?acast\.com/| play\.acast\.com/s/ ) - (?P[^/]+)/(?P[^/#?"]+) + (?P[^/?#]+)/(?:episodes/)?(?P[^/#?"]+) )''' _EMBED_REGEX = [rf'(?x)]+\bsrc=[\'"](?P{_VALID_URL})'] _TESTS = [{ - 'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna', + 'url': 'https://shows.acast.com/sparpodcast/episodes/2.raggarmordet-rosterurdetforflutna', 'info_dict': { 'id': '2a92b283-1a75-4ad8-8396-499c641de0d9', 'ext': 'mp3', @@ -59,7 +59,7 @@ class ACastIE(ACastBaseIE): 'timestamp': 1477346700, 'upload_date': '20161024', 'duration': 2766, - 'creator': 'Third Ear Studio', + 'creators': ['Third Ear Studio'], 'series': 'Spår', 'episode': '2. Raggarmordet - Röster ur det förflutna', 'thumbnail': 'https://assets.pippa.io/shows/616ebe1886d7b1398620b943/616ebe33c7e6e70013cae7da.jpg', @@ -74,6 +74,9 @@ class ACastIE(ACastBaseIE): }, { 'url': 'https://play.acast.com/s/rattegangspodden/s04e09styckmordetihelenelund-del2-2', 'only_matching': True, + }, { + 'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna', + 'only_matching': True, }, { 'url': 'https://play.acast.com/s/sparpodcast/2a92b283-1a75-4ad8-8396-499c641de0d9', 'only_matching': True, @@ -110,7 +113,7 @@ class ACastChannelIE(ACastBaseIE): _VALID_URL = r'''(?x) https?:// (?: - (?:www\.)?acast\.com/| + (?:(?:www|shows)\.)?acast\.com/| play\.acast\.com/s/ ) (?P[^/#?]+) @@ -120,12 +123,15 @@ class ACastChannelIE(ACastBaseIE): 'info_dict': { 'id': '4efc5294-5385-4847-98bd-519799ce5786', 'title': 'Today in Focus', - 'description': 'md5:c09ce28c91002ce4ffce71d6504abaae', + 'description': 'md5:feca253de9947634605080cd9eeea2bf', }, 'playlist_mincount': 200, }, { 'url': 'http://play.acast.com/s/ft-banking-weekly', 'only_matching': True, + }, { + 'url': 'https://shows.acast.com/sparpodcast', + 'only_matching': True, }] @classmethod diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 92ddad2b7..3e7734ce1 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -201,6 +201,11 @@ class InfoExtractor: fragment_base_url * "duration" (optional, int or float) * "filesize" (optional, int) + * hls_media_playlist_data + The M3U8 media playlist data as a string. + Only use if the data must be modified during extraction and + the native HLS downloader should bypass requesting the URL. + Does not apply if ffmpeg is used as external downloader * is_from_start Is a live format that can be downloaded from the start. Boolean * preference Order number of this format. If this field is diff --git a/yt_dlp/extractor/cwtv.py b/yt_dlp/extractor/cwtv.py index cb432e616..cdb29fcee 100644 --- a/yt_dlp/extractor/cwtv.py +++ b/yt_dlp/extractor/cwtv.py @@ -1,35 +1,40 @@ +import re + from .common import InfoExtractor from ..utils import ( ExtractorError, int_or_none, parse_age_limit, parse_iso8601, + parse_qs, smuggle_url, str_or_none, update_url_query, ) +from ..utils.traversal import traverse_obj class CWTVIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?cw(?:tv(?:pr)?|seed)\.com/(?:shows/)?(?:[^/]+/)+[^?]*\?.*\b(?:play|watch)=(?P[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})' + IE_NAME = 'cwtv' + _VALID_URL = r'https?://(?:www\.)?cw(?:tv(?:pr)?|seed)\.com/(?:shows/)?(?:[^/]+/)+[^?]*\?.*\b(?:play|watch|guid)=(?P[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})' _TESTS = [{ - 'url': 'https://www.cwtv.com/shows/all-american-homecoming/ready-or-not/?play=d848488f-f62a-40fd-af1f-6440b1821aab', + 'url': 'https://www.cwtv.com/shows/continuum/a-stitch-in-time/?play=9149a1e1-4cb2-46d7-81b2-47d35bbd332b', 'info_dict': { - 'id': 'd848488f-f62a-40fd-af1f-6440b1821aab', + 'id': '9149a1e1-4cb2-46d7-81b2-47d35bbd332b', 'ext': 'mp4', - 'title': 'Ready Or Not', - 'description': 'Simone is concerned about changes taking place at Bringston; JR makes a decision about his future.', - 'thumbnail': r're:^https?://.*\.jpe?g$', - 'duration': 2547, - 'timestamp': 1720519200, + 'title': 'A Stitch in Time', + 'description': r're:(?s)City Protective Services officer Kiera Cameron is transported from 2077.+', + 'thumbnail': r're:https?://.+\.jpe?g', + 'duration': 2632, + 'timestamp': 1736928000, 'uploader': 'CWTV', - 'chapters': 'count:6', - 'series': 'All American: Homecoming', - 'season_number': 3, + 'chapters': 'count:5', + 'series': 'Continuum', + 'season_number': 1, 'episode_number': 1, - 'age_limit': 0, - 'upload_date': '20240709', - 'season': 'Season 3', + 'age_limit': 14, + 'upload_date': '20250115', + 'season': 'Season 1', 'episode': 'Episode 1', }, 'params': { @@ -42,7 +47,7 @@ class CWTVIE(InfoExtractor): 'id': '6b15e985-9345-4f60-baf8-56e96be57c63', 'ext': 'mp4', 'title': 'Legends of Yesterday', - 'description': 'Oliver and Barry Allen take Kendra Saunders and Carter Hall to a remote location to keep them hidden from Vandal Savage while they figure out how to defeat him.', + 'description': r're:(?s)Oliver and Barry Allen take Kendra Saunders and Carter Hall to a remote.+', 'duration': 2665, 'series': 'Arrow', 'season_number': 4, @@ -71,7 +76,7 @@ class CWTVIE(InfoExtractor): 'timestamp': 1444107300, 'age_limit': 14, 'uploader': 'CWTV', - 'thumbnail': r're:^https?://.*\.jpe?g$', + 'thumbnail': r're:https?://.+\.jpe?g', 'chapters': 'count:4', 'episode': 'Episode 20', 'season': 'Season 11', @@ -89,14 +94,17 @@ class CWTVIE(InfoExtractor): }, { 'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?watch=6b15e985-9345-4f60-baf8-56e96be57c63', 'only_matching': True, + }, { + 'url': 'http://www.cwtv.com/movies/play/?guid=0a8e8b5b-1356-41d5-9a6a-4eda1a6feb6c', + 'only_matching': True, }] def _real_extract(self, url): video_id = self._match_id(url) data = self._download_json( - f'https://images.cwtv.com/feed/mobileapp/video-meta/apiversion_12/guid_{video_id}', video_id) - if data.get('result') != 'ok': - raise ExtractorError(data['msg'], expected=True) + f'https://images.cwtv.com/feed/app-2/video-meta/apiversion_22/device_android/guid_{video_id}', video_id) + if traverse_obj(data, 'result') != 'ok': + raise ExtractorError(traverse_obj(data, (('error_msg', 'msg'), {str}, any)), expected=True) video_data = data['video'] title = video_data['title'] mpx_url = update_url_query( @@ -123,3 +131,50 @@ def _real_extract(self, url): 'ie_key': 'ThePlatform', 'thumbnail': video_data.get('large_thumbnail'), } + + +class CWTVMovieIE(InfoExtractor): + IE_NAME = 'cwtv:movie' + _VALID_URL = r'https?://(?:www\.)?cwtv\.com/shows/(?P[\w-]+)/?\?(?:[^#]+&)?viewContext=Movies' + _TESTS = [{ + 'url': 'https://www.cwtv.com/shows/the-crush/?viewContext=Movies+Swimlane', + 'info_dict': { + 'id': '0a8e8b5b-1356-41d5-9a6a-4eda1a6feb6c', + 'ext': 'mp4', + 'title': 'The Crush', + 'upload_date': '20241112', + 'description': 'md5:1549acd90dff4a8273acd7284458363e', + 'chapters': 'count:9', + 'timestamp': 1731398400, + 'age_limit': 16, + 'duration': 5337, + 'series': 'The Crush', + 'season': 'Season 1', + 'uploader': 'CWTV', + 'season_number': 1, + 'episode': 'Episode 1', + 'episode_number': 1, + 'thumbnail': r're:https?://.+\.jpe?g', + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }] + _UUID_RE = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}' + + def _real_extract(self, url): + display_id = self._match_id(url) + webpage = self._download_webpage(url, display_id) + app_url = ( + self._html_search_meta('al:ios:url', webpage, default=None) + or self._html_search_meta('al:android:url', webpage, default=None)) + video_id = ( + traverse_obj(parse_qs(app_url), ('video_id', 0, {lambda x: re.fullmatch(self._UUID_RE, x)}, 0)) + or self._search_regex([ + rf'CWTV\.Site\.curPlayingGUID\s*=\s*["\']({self._UUID_RE})', + rf'CWTV\.Site\.viewInAppURL\s*=\s*["\']/shows/[\w-]+/watch-in-app/\?play=({self._UUID_RE})', + ], webpage, 'video ID')) + + return self.url_result( + f'https://www.cwtv.com/shows/{display_id}/{display_id}/?play={video_id}', CWTVIE, video_id) diff --git a/yt_dlp/extractor/digiview.py b/yt_dlp/extractor/digiview.py new file mode 100644 index 000000000..f7f23864d --- /dev/null +++ b/yt_dlp/extractor/digiview.py @@ -0,0 +1,130 @@ +from .common import InfoExtractor +from .youtube import YoutubeIE +from ..utils import clean_html, int_or_none, traverse_obj, url_or_none, urlencode_postdata + + +class DigiviewIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?ladigitale\.dev/digiview/#/v/(?P[0-9a-f]+)' + _TESTS = [{ + # normal video + 'url': 'https://ladigitale.dev/digiview/#/v/67a8e50aee2ec', + 'info_dict': { + 'id': '67a8e50aee2ec', + 'ext': 'mp4', + 'title': 'Big Buck Bunny 60fps 4K - Official Blender Foundation Short Film', + 'thumbnail': 'https://i.ytimg.com/vi/aqz-KE-bpKQ/hqdefault.jpg', + 'upload_date': '20141110', + 'playable_in_embed': True, + 'duration': 635, + 'view_count': int, + 'comment_count': int, + 'channel': 'Blender', + 'license': 'Creative Commons Attribution license (reuse allowed)', + 'like_count': int, + 'tags': 'count:8', + 'live_status': 'not_live', + 'channel_id': 'UCSMOQeBJ2RAnuFungnQOxLg', + 'channel_follower_count': int, + 'channel_url': 'https://www.youtube.com/channel/UCSMOQeBJ2RAnuFungnQOxLg', + 'uploader_id': '@BlenderOfficial', + 'description': 'md5:8f3ed18a53a1bb36cbb3b70a15782fd0', + 'categories': ['Film & Animation'], + 'channel_is_verified': True, + 'heatmap': 'count:100', + 'section_end': 635, + 'uploader': 'Blender', + 'timestamp': 1415628355, + 'uploader_url': 'https://www.youtube.com/@BlenderOfficial', + 'age_limit': 0, + 'section_start': 0, + 'availability': 'public', + }, + }, { + # cut video + 'url': 'https://ladigitale.dev/digiview/#/v/67a8e51d0dd58', + 'info_dict': { + 'id': '67a8e51d0dd58', + 'ext': 'mp4', + 'title': 'Big Buck Bunny 60fps 4K - Official Blender Foundation Short Film', + 'thumbnail': 'https://i.ytimg.com/vi/aqz-KE-bpKQ/hqdefault.jpg', + 'upload_date': '20141110', + 'playable_in_embed': True, + 'duration': 5, + 'view_count': int, + 'comment_count': int, + 'channel': 'Blender', + 'license': 'Creative Commons Attribution license (reuse allowed)', + 'like_count': int, + 'tags': 'count:8', + 'live_status': 'not_live', + 'channel_id': 'UCSMOQeBJ2RAnuFungnQOxLg', + 'channel_follower_count': int, + 'channel_url': 'https://www.youtube.com/channel/UCSMOQeBJ2RAnuFungnQOxLg', + 'uploader_id': '@BlenderOfficial', + 'description': 'md5:8f3ed18a53a1bb36cbb3b70a15782fd0', + 'categories': ['Film & Animation'], + 'channel_is_verified': True, + 'heatmap': 'count:100', + 'section_end': 10, + 'uploader': 'Blender', + 'timestamp': 1415628355, + 'uploader_url': 'https://www.youtube.com/@BlenderOfficial', + 'age_limit': 0, + 'section_start': 5, + 'availability': 'public', + }, + }, { + # changed title + 'url': 'https://ladigitale.dev/digiview/#/v/67a8ea5644d7a', + 'info_dict': { + 'id': '67a8ea5644d7a', + 'ext': 'mp4', + 'title': 'Big Buck Bunny (with title changed)', + 'thumbnail': 'https://i.ytimg.com/vi/aqz-KE-bpKQ/hqdefault.jpg', + 'upload_date': '20141110', + 'playable_in_embed': True, + 'duration': 5, + 'view_count': int, + 'comment_count': int, + 'channel': 'Blender', + 'license': 'Creative Commons Attribution license (reuse allowed)', + 'like_count': int, + 'tags': 'count:8', + 'live_status': 'not_live', + 'channel_id': 'UCSMOQeBJ2RAnuFungnQOxLg', + 'channel_follower_count': int, + 'channel_url': 'https://www.youtube.com/channel/UCSMOQeBJ2RAnuFungnQOxLg', + 'uploader_id': '@BlenderOfficial', + 'description': 'md5:8f3ed18a53a1bb36cbb3b70a15782fd0', + 'categories': ['Film & Animation'], + 'channel_is_verified': True, + 'heatmap': 'count:100', + 'section_end': 15, + 'uploader': 'Blender', + 'timestamp': 1415628355, + 'uploader_url': 'https://www.youtube.com/@BlenderOfficial', + 'age_limit': 0, + 'section_start': 10, + 'availability': 'public', + }, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + video_data = self._download_json( + 'https://ladigitale.dev/digiview/inc/recuperer_video.php', video_id, + data=urlencode_postdata({'id': video_id})) + + clip_id = video_data['videoId'] + return self.url_result( + f'https://www.youtube.com/watch?v={clip_id}', + YoutubeIE, video_id, url_transparent=True, + **traverse_obj(video_data, { + 'section_start': ('debut', {int_or_none}), + 'section_end': ('fin', {int_or_none}), + 'description': ('description', {clean_html}, filter), + 'title': ('titre', {str}), + 'thumbnail': ('vignette', {url_or_none}), + 'view_count': ('vues', {int_or_none}), + }), + ) diff --git a/yt_dlp/extractor/dropbox.py b/yt_dlp/extractor/dropbox.py index 2bfeebc7c..ce8435c8c 100644 --- a/yt_dlp/extractor/dropbox.py +++ b/yt_dlp/extractor/dropbox.py @@ -82,7 +82,7 @@ def _real_extract(self, url): has_anonymous_download = self._search_regex( r'(anonymous:\tanonymous)', part, 'anonymous', default=False) transcode_url = self._search_regex( - r'\n.(https://[^\x03\x08\x12\n]+\.m3u8)', part, 'transcode url', default=None) + r'\n.?(https://[^\x03\x08\x12\n]+\.m3u8)', part, 'transcode url', default=None) if not transcode_url: continue formats, subtitles = self._extract_m3u8_formats_and_subtitles(transcode_url, video_id, 'mp4') diff --git a/yt_dlp/extractor/francetv.py b/yt_dlp/extractor/francetv.py index ab08f1c6b..c6036b306 100644 --- a/yt_dlp/extractor/francetv.py +++ b/yt_dlp/extractor/francetv.py @@ -1,3 +1,4 @@ +import json import re import urllib.parse @@ -5,6 +6,7 @@ from .dailymotion import DailymotionIE from ..networking import HEADRequest from ..utils import ( + ExtractorError, clean_html, determine_ext, filter_dict, @@ -29,6 +31,7 @@ def _make_url_result(self, video_id, url=None): class FranceTVIE(InfoExtractor): + IE_NAME = 'francetv' _VALID_URL = r'francetv:(?P[^@#]+)' _GEO_COUNTRIES = ['FR'] _GEO_BYPASS = False @@ -248,18 +251,19 @@ def _real_extract(self, url): class FranceTVSiteIE(FranceTVBaseInfoExtractor): + IE_NAME = 'francetv:site' _VALID_URL = r'https?://(?:(?:www\.)?france\.tv|mobile\.france\.tv)/(?:[^/]+/)*(?P[^/]+)\.html' _TESTS = [{ 'url': 'https://www.france.tv/france-2/13h15-le-dimanche/140921-les-mysteres-de-jesus.html', 'info_dict': { - 'id': 'c5bda21d-2c6f-4470-8849-3d8327adb2ba', + 'id': 'ec217ecc-0733-48cf-ac06-af1347b849d1', # old: c5bda21d-2c6f-4470-8849-3d8327adb2ba' 'ext': 'mp4', 'title': '13h15, le dimanche... - Les mystères de Jésus', - 'timestamp': 1514118300, - 'duration': 2880, + 'timestamp': 1502623500, + 'duration': 2580, 'thumbnail': r're:^https?://.*\.jpg$', - 'upload_date': '20171224', + 'upload_date': '20170813', }, 'params': { 'skip_download': True, @@ -282,6 +286,7 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor): 'thumbnail': r're:^https?://.*\.jpg$', 'duration': 1441, }, + 'skip': 'No longer available', }, { # geo-restricted livestream (workflow == 'token-akamai') 'url': 'https://www.france.tv/france-4/direct.html', @@ -336,19 +341,33 @@ class FranceTVSiteIE(FranceTVBaseInfoExtractor): 'only_matching': True, }] + # XXX: For parsing next.js v15+ data; see also yt_dlp.extractor.goplay + def _find_json(self, s): + return self._search_json( + r'\w+\s*:\s*', s, 'next js data', None, contains_pattern=r'\[(?s:.+)\]', default=None) + def _real_extract(self, url): display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) - video_id = self._search_regex( - r'(?:data-main-video\s*=|videoId["\']?\s*[:=])\s*(["\'])(?P(?:(?!\1).)+)\1', - webpage, 'video id', default=None, group='id') + nextjs_data = traverse_obj( + re.findall(r']*>\s*self\.__next_f\.push\(\s*(\[.+?\])\s*\);?\s*', webpage), + (..., {json.loads}, ..., {self._find_json}, ..., 'children', ..., ..., 'children', ..., ..., 'children')) + + if traverse_obj(nextjs_data, (..., ..., 'children', ..., 'isLive', {bool}, any)): + # For livestreams we need the id of the stream instead of the currently airing episode id + video_id = traverse_obj(nextjs_data, ( + ..., ..., 'children', ..., 'children', ..., 'children', ..., 'children', ..., ..., + 'children', ..., ..., 'children', ..., ..., 'children', (..., (..., ...)), + 'options', 'id', {str}, any)) + else: + video_id = traverse_obj(nextjs_data, ( + ..., ..., ..., 'children', + lambda _, v: v['video']['url'] == urllib.parse.urlparse(url).path, + 'video', ('playerReplayId', 'siId'), {str}, any)) if not video_id: - video_id = self._html_search_regex( - r'(?:href=|player\.setVideo\(\s*)"http://videos?\.francetv\.fr/video/([^@"]+@[^"]+)"', - webpage, 'video ID') + raise ExtractorError('Unable to extract video ID') return self._make_url_result(video_id, url=url) diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py index 320a47772..67c224e50 100644 --- a/yt_dlp/extractor/generic.py +++ b/yt_dlp/extractor/generic.py @@ -293,6 +293,19 @@ class GenericIE(InfoExtractor): 'timestamp': 1378272859.0, }, }, + # Live DASH MPD + { + 'url': 'https://livesim2.dashif.org/livesim2/ato_10/testpic_2s/Manifest.mpd', + 'info_dict': { + 'id': 'Manifest', + 'ext': 'mp4', + 'title': r're:Manifest \d{4}-\d{2}-\d{2} \d{2}:\d{2}$', + 'live_status': 'is_live', + }, + 'params': { + 'skip_download': 'livestream', + }, + }, # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8 { 'url': 'http://once.unicornmedia.com/now/master/playlist/bb0b18ba-64f5-4b1b-a29f-0ac252f06b68/77a785f3-5188-4806-b788-0893a61634ed/93677179-2d99-4ef4-9e17-fe70d49abfbf/content.m3u8', @@ -2436,10 +2449,9 @@ def _real_extract(self, url): subtitles = {} if format_id.endswith('mpegurl') or ext == 'm3u8': formats, subtitles = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4', headers=headers) - elif format_id.endswith(('mpd', 'dash+xml')) or ext == 'mpd': - formats, subtitles = self._extract_mpd_formats_and_subtitles(url, video_id, headers=headers) elif format_id == 'f4m' or ext == 'f4m': formats = self._extract_f4m_formats(url, video_id, headers=headers) + # Don't check for DASH/mpd here, do it later w/ first_bytes. Same number of requests either way else: formats = [{ 'format_id': format_id, @@ -2521,6 +2533,7 @@ def _real_extract(self, url): doc, mpd_base_url=full_response.url.rpartition('/')[0], mpd_url=url) + info_dict['live_status'] = 'is_live' if doc.get('type') == 'dynamic' else None self._extra_manifest_info(info_dict, url) self.report_detected('DASH manifest') return info_dict diff --git a/yt_dlp/extractor/globo.py b/yt_dlp/extractor/globo.py index d72296be6..7acbd2820 100644 --- a/yt_dlp/extractor/globo.py +++ b/yt_dlp/extractor/globo.py @@ -1,32 +1,48 @@ -import base64 -import hashlib import json -import random import re +import uuid from .common import InfoExtractor -from ..networking import HEADRequest from ..utils import ( - ExtractorError, + determine_ext, + filter_dict, float_or_none, + int_or_none, orderedSet, str_or_none, try_get, + url_or_none, ) +from ..utils.traversal import subs_list_to_dict, traverse_obj class GloboIE(InfoExtractor): - _VALID_URL = r'(?:globo:|https?://.+?\.globo\.com/(?:[^/]+/)*(?:v/(?:[^/]+/)?|videos/))(?P\d{7,})' + _VALID_URL = r'(?:globo:|https?://[^/?#]+?\.globo\.com/(?:[^/?#]+/))(?P\d{7,})' _NETRC_MACHINE = 'globo' + _VIDEO_VIEW = ''' + query getVideoView($videoId: ID!) { + video(id: $videoId) { + duration + description + relatedEpisodeNumber + relatedSeasonNumber + headline + title { + originProgramId + headline + } + } + } + ''' _TESTS = [{ - 'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/', + 'url': 'https://globoplay.globo.com/v/3607726/', 'info_dict': { 'id': '3607726', 'ext': 'mp4', 'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa', 'duration': 103.204, - 'uploader': 'G1', - 'uploader_id': '2015', + 'uploader': 'G1 ao vivo', + 'uploader_id': '4209', }, 'params': { 'skip_download': True, @@ -38,39 +54,36 @@ class GloboIE(InfoExtractor): 'ext': 'mp4', 'title': 'Acidentes de trânsito estão entre as maiores causas de queda de energia em SP', 'duration': 137.973, - 'uploader': 'Rede Globo', - 'uploader_id': '196', + 'uploader': 'Bom Dia Brasil', + 'uploader_id': '810', }, 'params': { 'skip_download': True, }, - }, { - 'url': 'http://canalbrasil.globo.com/programas/sangue-latino/videos/3928201.html', - 'only_matching': True, - }, { - 'url': 'http://globosatplay.globo.com/globonews/v/4472924/', - 'only_matching': True, - }, { - 'url': 'http://globotv.globo.com/t/programa/v/clipe-sexo-e-as-negas-adeus/3836166/', - 'only_matching': True, - }, { - 'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/', - 'only_matching': True, - }, { - 'url': 'http://canaloff.globo.com/programas/desejar-profundo/videos/4518560.html', - 'only_matching': True, }, { 'url': 'globo:3607726', 'only_matching': True, - }, { - 'url': 'https://globoplay.globo.com/v/10248083/', + }, + { + 'url': 'globo:8013907', # needs subscription to globoplay 'info_dict': { - 'id': '10248083', + 'id': '8013907', 'ext': 'mp4', - 'title': 'Melhores momentos: Equador 1 x 1 Brasil pelas Eliminatórias da Copa do Mundo 2022', - 'duration': 530.964, - 'uploader': 'SporTV', - 'uploader_id': '698', + 'title': 'Capítulo de 14⧸08⧸1989', + 'episode_number': 1, + }, + 'params': { + 'skip_download': True, + }, + }, + { + 'url': 'globo:12824146', + 'info_dict': { + 'id': '12824146', + 'ext': 'mp4', + 'title': 'Acordo de damas', + 'episode_number': 1, + 'season_number': 2, }, 'params': { 'skip_download': True, @@ -80,98 +93,70 @@ class GloboIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) - self._request_webpage( - HEADRequest('https://globo-ab.globo.com/v2/selected-alternatives?experiments=player-isolated-experiment-02&skipImpressions=true'), - video_id, 'Getting cookies') - - video = self._download_json( - f'http://api.globovideos.com/videos/{video_id}/playlist', - video_id)['videos'][0] - if not self.get_param('allow_unplayable_formats') and video.get('encrypted') is True: - self.report_drm(video_id) - - title = video['title'] + info = self._download_json( + 'https://cloud-jarvis.globo.com/graphql', video_id, + query={ + 'operationName': 'getVideoView', + 'variables': json.dumps({'videoId': video_id}), + 'query': self._VIDEO_VIEW, + }, headers={ + 'content-type': 'application/json', + 'x-platform-id': 'web', + 'x-device-id': 'desktop', + 'x-client-version': '2024.12-5', + })['data']['video'] formats = [] - security = self._download_json( - 'https://playback.video.globo.com/v2/video-session', video_id, f'Downloading security hash for {video_id}', - headers={'content-type': 'application/json'}, data=json.dumps({ - 'player_type': 'desktop', + video = self._download_json( + 'https://playback.video.globo.com/v4/video-session', video_id, + f'Downloading resource info for {video_id}', + headers={'Content-Type': 'application/json'}, + data=json.dumps(filter_dict({ + 'player_type': 'mirakulo_8k_hdr', 'video_id': video_id, 'quality': 'max', 'content_protection': 'widevine', - 'vsid': '581b986b-4c40-71f0-5a58-803e579d5fa2', - 'tz': '-3.0:00', - }).encode()) + 'vsid': f'{uuid.uuid4()}', + 'consumption': 'streaming', + 'capabilities': {'low_latency': True}, + 'tz': '-03:00', + 'Authorization': try_get(self._get_cookies('https://globo.com'), + lambda x: f'Bearer {x["GLBID"].value}'), + 'version': 1, + })).encode()) - self._request_webpage(HEADRequest(security['sources'][0]['url_template']), video_id, 'Getting locksession cookie') + if traverse_obj(video, ('resource', 'drm_protection_enabled', {bool})): + self.report_drm(video_id) - security_hash = security['sources'][0]['token'] - if not security_hash: - message = security.get('message') - if message: - raise ExtractorError( - f'{self.IE_NAME} returned error: {message}', expected=True) + main_source = video['sources'][0] - hash_code = security_hash[:2] - padding = '%010d' % random.randint(1, 10000000000) - if hash_code in ('04', '14'): - received_time = security_hash[3:13] - received_md5 = security_hash[24:] - hash_prefix = security_hash[:23] - elif hash_code in ('02', '12', '03', '13'): - received_time = security_hash[2:12] - received_md5 = security_hash[22:] - padding += '1' - hash_prefix = '05' + security_hash[:22] - - padded_sign_time = str(int(received_time) + 86400) + padding - md5_data = (received_md5 + padded_sign_time + '0xAC10FD').encode() - signed_md5 = base64.urlsafe_b64encode(hashlib.md5(md5_data).digest()).decode().strip('=') - signed_hash = hash_prefix + padded_sign_time + signed_md5 - source = security['sources'][0]['url_parts'] - resource_url = source['scheme'] + '://' + source['domain'] + source['path'] - signed_url = '{}?h={}&k=html5&a={}'.format(resource_url, signed_hash, 'F' if video.get('subscriber_only') else 'A') - - fmts, subtitles = self._extract_m3u8_formats_and_subtitles( - signed_url, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) - formats.extend(fmts) - - for resource in video['resources']: - if resource.get('type') == 'subtitle': - subtitles.setdefault(resource.get('language') or 'por', []).append({ - 'url': resource.get('url'), - }) - subs = try_get(security, lambda x: x['source']['subtitles'], expected_type=dict) or {} - for sub_lang, sub_url in subs.items(): - if sub_url: - subtitles.setdefault(sub_lang or 'por', []).append({ - 'url': sub_url, - }) - subs = try_get(security, lambda x: x['source']['subtitles_webvtt'], expected_type=dict) or {} - for sub_lang, sub_url in subs.items(): - if sub_url: - subtitles.setdefault(sub_lang or 'por', []).append({ - 'url': sub_url, - }) - - duration = float_or_none(video.get('duration'), 1000) - uploader = video.get('channel') - uploader_id = str_or_none(video.get('channel_id')) + # 4k streams are exclusively outputted in dash, so we need to filter these out + if determine_ext(main_source['url']) == 'mpd': + formats, subtitles = self._extract_mpd_formats_and_subtitles(main_source['url'], video_id, mpd_id='dash') + else: + formats, subtitles = self._extract_m3u8_formats_and_subtitles( + main_source['url'], video_id, 'mp4', m3u8_id='hls') + self._merge_subtitles(traverse_obj(main_source, ('text', ..., { + 'url': ('subtitle', 'srt', 'url', {url_or_none}), + }, all, {subs_list_to_dict(lang='en')})), target=subtitles) return { 'id': video_id, - 'title': title, - 'duration': duration, - 'uploader': uploader, - 'uploader_id': uploader_id, + **traverse_obj(info, { + 'title': ('headline', {str}), + 'duration': ('duration', {float_or_none(scale=1000)}), + 'uploader': ('title', 'headline', {str}), + 'uploader_id': ('title', 'originProgramId', {str_or_none}), + 'episode_number': ('relatedEpisodeNumber', {int_or_none}), + 'season_number': ('relatedSeasonNumber', {int_or_none}), + }), 'formats': formats, 'subtitles': subtitles, } class GloboArticleIE(InfoExtractor): - _VALID_URL = r'https?://.+?\.globo\.com/(?:[^/]+/)*(?P[^/.]+)(?:\.html)?' + _VALID_URL = r'https?://(?!globoplay).+?\.globo\.com/(?:[^/?#]+/)*(?P[^/?#.]+)(?:\.html)?' _VIDEOID_REGEXES = [ r'\bdata-video-id=["\'](\d{7,})["\']', diff --git a/yt_dlp/extractor/goplay.py b/yt_dlp/extractor/goplay.py index 32300f75c..c654c757c 100644 --- a/yt_dlp/extractor/goplay.py +++ b/yt_dlp/extractor/goplay.py @@ -12,7 +12,6 @@ from ..utils import ( ExtractorError, int_or_none, - js_to_json, remove_end, traverse_obj, ) @@ -76,6 +75,7 @@ def _real_initialize(self): if not self._id_token: raise self.raise_login_required(method='password') + # XXX: For parsing next.js v15+ data; see also yt_dlp.extractor.francetv def _find_json(self, s): return self._search_json( r'\w+\s*:\s*', s, 'next js data', None, contains_pattern=r'\[(?s:.+)\]', default=None) @@ -86,9 +86,10 @@ def _real_extract(self, url): nextjs_data = traverse_obj( re.findall(r']*>\s*self\.__next_f\.push\(\s*(\[.+?\])\s*\);?\s*', webpage), - (..., {js_to_json}, {json.loads}, ..., {self._find_json}, ...)) + (..., {json.loads}, ..., {self._find_json}, ...)) meta = traverse_obj(nextjs_data, ( - ..., lambda _, v: v['meta']['path'] == urllib.parse.urlparse(url).path, 'meta', any)) + ..., ..., 'children', ..., ..., 'children', + lambda _, v: v['video']['path'] == urllib.parse.urlparse(url).path, 'video', any)) video_id = meta['uuid'] info_dict = traverse_obj(meta, { diff --git a/yt_dlp/extractor/pbs.py b/yt_dlp/extractor/pbs.py index 7b84515e2..2f839a2e9 100644 --- a/yt_dlp/extractor/pbs.py +++ b/yt_dlp/extractor/pbs.py @@ -61,7 +61,7 @@ class PBSIE(InfoExtractor): (r'video\.wyomingpbs\.org', 'Wyoming PBS (KCWC)'), # http://www.wyomingpbs.org (r'video\.cpt12\.org', 'Colorado Public Television / KBDI 12 (KBDI)'), # http://www.cpt12.org/ (r'video\.kbyueleven\.org', 'KBYU-TV (KBYU)'), # http://www.kbyutv.org/ - (r'video\.thirteen\.org', 'Thirteen/WNET New York (WNET)'), # http://www.thirteen.org + (r'(?:video\.|www\.)thirteen\.org', 'Thirteen/WNET New York (WNET)'), # http://www.thirteen.org (r'video\.wgbh\.org', 'WGBH/Channel 2 (WGBH)'), # http://wgbh.org (r'video\.wgby\.org', 'WGBY (WGBY)'), # http://www.wgby.org (r'watch\.njtvonline\.org', 'NJTV Public Media NJ (WNJT)'), # http://www.njtvonline.org/ @@ -208,16 +208,40 @@ class PBSIE(InfoExtractor): 'description': 'md5:31b664af3c65fd07fa460d306b837d00', 'duration': 3190, }, + 'skip': 'dead URL', + }, + { + 'url': 'https://www.thirteen.org/programs/the-woodwrights-shop/carving-away-with-mary-may-tioglz/', + 'info_dict': { + 'id': '3004803331', + 'ext': 'mp4', + 'title': "The Woodwright's Shop - Carving Away with Mary May", + 'description': 'md5:7cbaaaa8b9bcc78bd8f0e31911644e28', + 'duration': 1606, + 'display_id': 'carving-away-with-mary-may-tioglz', + 'chapters': [], + 'thumbnail': 'https://image.pbs.org/video-assets/NcnTxNl-asset-mezzanine-16x9-K0Keoyv.jpg', + }, }, { 'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/', - 'md5': '6f722cb3c3982186d34b0f13374499c7', + 'md5': '372b12b670070de39438b946474df92f', 'info_dict': { 'id': '2365297690', 'ext': 'mp4', 'title': 'FRONTLINE - Losing Iraq', 'description': 'md5:5979a4d069b157f622d02bff62fbe654', 'duration': 5050, + 'chapters': [ + {'start_time': 0.0, 'end_time': 1234.0, 'title': 'After Saddam, Chaos'}, + {'start_time': 1233.0, 'end_time': 1719.0, 'title': 'The Insurgency Takes Root'}, + {'start_time': 1718.0, 'end_time': 2461.0, 'title': 'A Light Footprint'}, + {'start_time': 2460.0, 'end_time': 3589.0, 'title': 'The Surge '}, + {'start_time': 3588.0, 'end_time': 4355.0, 'title': 'The Withdrawal '}, + {'start_time': 4354.0, 'end_time': 5051.0, 'title': 'ISIS on the March '}, + ], + 'display_id': 'losing-iraq', + 'thumbnail': 'https://image.pbs.org/video-assets/pbs/frontline/138098/images/mezzanine_401.jpg', }, }, { @@ -477,6 +501,7 @@ def _extract_webpage(self, url): r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'", # frontline video embed r'class="coveplayerid">([^<]+)<', # coveplayer r']+data-coveid="(\d+)"', # coveplayer from http://www.pbs.org/wgbh/frontline/film/real-csi/ + r'\bclass="passportcoveplayer"[^>]+\bdata-media="(\d+)', # https://www.thirteen.org/programs/the-woodwrights-shop/who-wrote-the-book-of-sloyd-fggvvq/ r'', # jwplayer r"(?s)window\.PBS\.playerConfig\s*=\s*{.*?id\s*:\s*'([0-9]+)',", r']+\bdata-cove-id=["\'](\d+)"', # http://www.pbs.org/wgbh/roadshow/watch/episode/2105-indianapolis-hour-2/ diff --git a/yt_dlp/extractor/reddit.py b/yt_dlp/extractor/reddit.py index 7325e547b..d5150fd8f 100644 --- a/yt_dlp/extractor/reddit.py +++ b/yt_dlp/extractor/reddit.py @@ -198,6 +198,25 @@ class RedditIE(InfoExtractor): 'skip_download': True, 'writesubtitles': True, }, + }, { + # "gated" subreddit post + 'url': 'https://old.reddit.com/r/ketamine/comments/degtjo/when_the_k_hits/', + 'info_dict': { + 'id': 'gqsbxts133r31', + 'ext': 'mp4', + 'display_id': 'degtjo', + 'title': 'When the K hits', + 'uploader': '[deleted]', + 'channel_id': 'ketamine', + 'comment_count': int, + 'like_count': int, + 'dislike_count': int, + 'age_limit': 18, + 'duration': 34, + 'thumbnail': r're:https?://.+/.+\.(?:jpg|png)', + 'timestamp': 1570438713.0, + 'upload_date': '20191007', + }, }, { 'url': 'https://www.reddit.com/r/videos/comments/6rrwyj', 'only_matching': True, @@ -245,6 +264,15 @@ def _perform_login(self, username, password): elif not traverse_obj(login, ('json', 'data', 'cookie', {str})): raise ExtractorError('Unable to login, no cookie was returned') + def _real_initialize(self): + # Set cookie to opt-in to age-restricted subreddits + self._set_cookie('reddit.com', 'over18', '1') + # Set cookie to opt-in to "gated" subreddits + options = traverse_obj(self._get_cookies('https://www.reddit.com/'), ( + '_options', 'value', {urllib.parse.unquote}, {json.loads}, {dict})) or {} + options['pref_gated_sr_optin'] = True + self._set_cookie('reddit.com', '_options', urllib.parse.quote(json.dumps(options))) + def _get_subtitles(self, video_id): # Fallback if there were no subtitles provided by DASH or HLS manifests caption_url = f'https://v.redd.it/{video_id}/wh_ben_en.vtt' diff --git a/yt_dlp/extractor/theplatform.py b/yt_dlp/extractor/theplatform.py index 7c1769c2d..b73bea18f 100644 --- a/yt_dlp/extractor/theplatform.py +++ b/yt_dlp/extractor/theplatform.py @@ -118,8 +118,9 @@ def extract_site_specific_field(field): 'categories', lambda _, v: v.get('label') in ('category', None), 'name', {str})) or None, 'tags': traverse_obj(info, ('keywords', {lambda x: re.split(r'[;,]\s?', x) if x else None})), 'location': extract_site_specific_field('region'), - 'series': extract_site_specific_field('show'), + 'series': extract_site_specific_field('show') or extract_site_specific_field('seriesTitle'), 'season_number': int_or_none(extract_site_specific_field('seasonNumber')), + 'episode_number': int_or_none(extract_site_specific_field('episodeNumber')), 'media_type': extract_site_specific_field('programmingType') or extract_site_specific_field('type'), } diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py index c05b5bf9c..d32ae3f18 100644 --- a/yt_dlp/extractor/twitter.py +++ b/yt_dlp/extractor/twitter.py @@ -1,11 +1,12 @@ import functools import json -import random +import math import re import urllib.parse from .common import InfoExtractor from .periscope import PeriscopeBaseIE, PeriscopeIE +from ..jsinterp import js_number_to_string from ..networking.exceptions import HTTPError from ..utils import ( ExtractorError, @@ -1330,6 +1331,11 @@ def _build_graphql_query(self, media_id): }, } + def _generate_syndication_token(self, twid): + # ((Number(twid) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '') + translation = str.maketrans(dict.fromkeys('0.')) + return js_number_to_string((int(twid) / 1e15) * math.PI, 36).translate(translation) + def _call_syndication_api(self, twid): self.report_warning( 'Not all metadata or media is available via syndication endpoint', twid, only_once=True) @@ -1337,8 +1343,7 @@ def _call_syndication_api(self, twid): 'https://cdn.syndication.twimg.com/tweet-result', twid, 'Downloading syndication JSON', headers={'User-Agent': 'Googlebot'}, query={ 'id': twid, - # TODO: token = ((Number(twid) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '') - 'token': ''.join(random.choices('123456789abcdefghijklmnopqrstuvwxyz', k=10)), + 'token': self._generate_syndication_token(twid), }) if not status: raise ExtractorError('Syndication endpoint returned empty JSON response') diff --git a/yt_dlp/extractor/zdf.py b/yt_dlp/extractor/zdf.py index 703766cd7..b64a88f6c 100644 --- a/yt_dlp/extractor/zdf.py +++ b/yt_dlp/extractor/zdf.py @@ -187,12 +187,20 @@ class ZDFIE(ZDFBaseIE): 'info_dict': { 'id': '151025_magie_farben2_tex', 'ext': 'mp4', + 'duration': 2615.0, 'title': 'Die Magie der Farben (2/2)', 'description': 'md5:a89da10c928c6235401066b60a6d5c1a', - 'duration': 2615, 'timestamp': 1465021200, - 'upload_date': '20160604', 'thumbnail': 'https://www.zdf.de/assets/mauve-im-labor-100~768x432?cb=1464909117806', + 'upload_date': '20160604', + 'episode': 'Die Magie der Farben (2/2)', + 'episode_id': 'POS_954f4170-36a5-4a41-a6cf-78f1f3b1f127', + 'season': 'Staffel 1', + 'series': 'Die Magie der Farben', + 'season_number': 1, + 'series_id': 'a39900dd-cdbd-4a6a-a413-44e8c6ae18bc', + 'season_id': '5a92e619-8a0f-4410-a3d5-19c76fbebb37', + 'episode_number': 2, }, }, { 'url': 'https://www.zdf.de/funk/druck-11790/funk-alles-ist-verzaubert-102.html', @@ -200,12 +208,13 @@ class ZDFIE(ZDFBaseIE): 'info_dict': { 'ext': 'mp4', 'id': 'video_funk_1770473', - 'duration': 1278, - 'description': 'Die Neue an der Schule verdreht Ismail den Kopf.', + 'duration': 1278.0, 'title': 'Alles ist verzaubert', + 'description': 'Die Neue an der Schule verdreht Ismail den Kopf.', 'timestamp': 1635520560, - 'upload_date': '20211029', 'thumbnail': 'https://www.zdf.de/assets/teaser-funk-alles-ist-verzaubert-102~1920x1080?cb=1663848412907', + 'upload_date': '20211029', + 'episode': 'Alles ist verzaubert', }, }, { # Same as https://www.phoenix.de/sendungen/dokumentationen/gesten-der-maechtigen-i-a-89468.html?ref=suche @@ -248,22 +257,52 @@ class ZDFIE(ZDFBaseIE): 'title': 'Das Geld anderer Leute', 'description': 'md5:cb6f660850dc5eb7d1ab776ea094959d', 'duration': 2581.0, - 'timestamp': 1675160100, - 'upload_date': '20230131', + 'timestamp': 1728983700, + 'upload_date': '20241015', 'thumbnail': 'https://epg-image.zdf.de/fotobase-webdelivery/images/e2d7e55a-09f0-424e-ac73-6cac4dd65f35?layout=2400x1350', + 'series': 'SOKO Stuttgart', + 'series_id': 'f862ce9a-6dd1-4388-a698-22b36ac4c9e9', + 'season': 'Staffel 11', + 'season_number': 11, + 'season_id': 'ae1b4990-6d87-4970-a571-caccf1ba2879', + 'episode': 'Das Geld anderer Leute', + 'episode_number': 10, + 'episode_id': 'POS_7f367934-f2f0-45cb-9081-736781ff2d23', }, }, { 'url': 'https://www.zdf.de/dokumentation/terra-x/unser-gruener-planet-wuesten-doku-100.html', 'info_dict': { - 'id': '220605_dk_gruener_planet_wuesten_tex', + 'id': '220525_green_planet_makingof_1_tropen_tex', 'ext': 'mp4', - 'title': 'Unser grüner Planet - Wüsten', - 'description': 'md5:4fc647b6f9c3796eea66f4a0baea2862', - 'duration': 2613.0, - 'timestamp': 1654450200, - 'upload_date': '20220605', - 'format_note': 'uhd, main', - 'thumbnail': 'https://www.zdf.de/assets/saguaro-kakteen-102~3840x2160?cb=1655910690796', + 'title': 'Making-of Unser grüner Planet - Tropen', + 'description': 'md5:d7c6949dc7c75c73c4ad51c785fb0b79', + 'duration': 435.0, + 'timestamp': 1653811200, + 'upload_date': '20220529', + 'format_note': 'hd, main', + 'thumbnail': 'https://www.zdf.de/assets/unser-gruener-planet-making-of-1-tropen-100~3840x2160?cb=1653493335577', + 'episode': 'Making-of Unser grüner Planet - Tropen', + }, + 'skip': 'No longer available: "Leider kein Video verfügbar"', + }, { + 'url': 'https://www.zdf.de/serien/northern-lights/begegnung-auf-der-bruecke-100.html', + 'info_dict': { + 'id': '240319_2310_sendung_not', + 'ext': 'mp4', + 'title': 'Begegnung auf der Brücke', + 'description': 'md5:e53a555da87447f7f1207f10353f8e45', + 'thumbnail': 'https://epg-image.zdf.de/fotobase-webdelivery/images/c5ff1d1f-f5c8-4468-86ac-1b2f1dbecc76?layout=2400x1350', + 'upload_date': '20250203', + 'duration': 3083.0, + 'timestamp': 1738546500, + 'series_id': '1d7a1879-01ee-4468-8237-c6b4ecd633c7', + 'series': 'Northern Lights', + 'season': 'Staffel 1', + 'season_number': 1, + 'season_id': '22ac26a2-4ea2-4055-ac0b-98b755cdf718', + 'episode': 'Begegnung auf der Brücke', + 'episode_number': 1, + 'episode_id': 'POS_71049438-024b-471f-b472-4fe2e490d1fb', }, }] @@ -316,12 +355,31 @@ def _extract_entry(self, url, player, content, video_id): 'timestamp': unified_timestamp(content.get('editorialDate')), 'thumbnails': thumbnails, 'chapters': chapters or None, + 'episode': title, + **traverse_obj(content, ('programmeItem', 0, 'http://zdf.de/rels/target', { + 'series_id': ('http://zdf.de/rels/cmdm/series', 'seriesUuid', {str}), + 'series': ('http://zdf.de/rels/cmdm/series', 'seriesTitle', {str}), + 'season': ('http://zdf.de/rels/cmdm/season', 'seasonTitle', {str}), + 'season_number': ('http://zdf.de/rels/cmdm/season', 'seasonNumber', {int_or_none}), + 'season_id': ('http://zdf.de/rels/cmdm/season', 'seasonUuid', {str}), + 'episode_number': ('episodeNumber', {int_or_none}), + 'episode_id': ('contentId', {str}), + })), }) def _extract_regular(self, url, player, video_id): - content = self._call_api( - player['content'], video_id, 'content', player['apiToken'], url) - return self._extract_entry(player['content'], player, content, video_id) + player_url = player['content'] + + try: + content = self._call_api( + update_url_query(player_url, {'profile': 'player-3'}), + video_id, 'content', player['apiToken'], url) + except ExtractorError as e: + self.report_warning(f'{video_id}: {e.orig_msg}; retrying with v2 profile') + content = self._call_api( + player_url, video_id, 'content', player['apiToken'], url) + + return self._extract_entry(player_url, player, content, video_id) def _extract_mobile(self, video_id): video = self._download_v2_doc(video_id) diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py index ba059babb..ac0629715 100644 --- a/yt_dlp/jsinterp.py +++ b/yt_dlp/jsinterp.py @@ -25,7 +25,7 @@ def zeroise(x): with contextlib.suppress(TypeError): if math.isnan(x): # NB: NaN cannot be checked by membership return 0 - return x + return int(float(x)) def wrapped(a, b): return op(zeroise(a), zeroise(b)) & 0xffffffff @@ -95,6 +95,61 @@ def _js_ternary(cndn, if_true=True, if_false=False): return if_true +# Ref: https://es5.github.io/#x9.8.1 +def js_number_to_string(val: float, radix: int = 10): + if radix in (JS_Undefined, None): + radix = 10 + assert radix in range(2, 37), 'radix must be an integer at least 2 and no greater than 36' + + if math.isnan(val): + return 'NaN' + if val == 0: + return '0' + if math.isinf(val): + return '-Infinity' if val < 0 else 'Infinity' + if radix == 10: + # TODO: implement special cases + ... + + ALPHABET = b'0123456789abcdefghijklmnopqrstuvwxyz.-' + + result = collections.deque() + sign = val < 0 + val = abs(val) + fraction, integer = math.modf(val) + delta = max(math.nextafter(.0, math.inf), math.ulp(val) / 2) + + if fraction >= delta: + result.append(-2) # `.` + while fraction >= delta: + delta *= radix + fraction, digit = math.modf(fraction * radix) + result.append(int(digit)) + # if we need to round, propagate potential carry through fractional part + needs_rounding = fraction > 0.5 or (fraction == 0.5 and int(digit) & 1) + if needs_rounding and fraction + delta > 1: + for index in reversed(range(1, len(result))): + if result[index] + 1 < radix: + result[index] += 1 + break + result.pop() + + else: + integer += 1 + break + + integer, digit = divmod(int(integer), radix) + result.appendleft(digit) + while integer > 0: + integer, digit = divmod(integer, radix) + result.appendleft(digit) + + if sign: + result.appendleft(-1) # `-` + + return bytes(ALPHABET[digit] for digit in result).decode('ascii') + + # Ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence _OPERATORS = { # None => Defined in JSInterpreter._operator '?': None,