1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2025-03-09 12:50:23 -05:00
This commit is contained in:
Kieran 2025-03-07 23:03:32 +01:00 committed by GitHub
commit c15d21f9c6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 127 additions and 77 deletions

View file

@ -962,32 +962,33 @@ def test_format_note(self):
}), r'^30fps$')
def test_postprocessors(self):
filename = 'post-processor-testfile.mp4'
audiofile = filename + '.mp3'
filename = 'post-processor-testfile'
video_file = filename + '.mp4'
audio_file = filename + '.mp3'
class SimplePP(PostProcessor):
def run(self, info):
with open(audiofile, 'w') as f:
with open(audio_file, 'w') as f:
f.write('EXAMPLE')
return [info['filepath']], info
def run_pp(params, pp):
with open(filename, 'w') as f:
with open(video_file, 'w') as f:
f.write('EXAMPLE')
ydl = YoutubeDL(params)
ydl.add_post_processor(pp())
ydl.post_process(filename, {'filepath': filename})
ydl.post_process(video_file, {'filepath': video_file})
run_pp({'keepvideo': True}, SimplePP)
self.assertTrue(os.path.exists(filename), f'{filename} doesn\'t exist')
self.assertTrue(os.path.exists(audiofile), f'{audiofile} doesn\'t exist')
os.unlink(filename)
os.unlink(audiofile)
run_pp({'keepvideo': True, 'outtmpl': filename}, SimplePP)
self.assertTrue(os.path.exists(video_file), f'{video_file} doesn\'t exist')
self.assertTrue(os.path.exists(audio_file), f'{audio_file} doesn\'t exist')
os.unlink(video_file)
os.unlink(audio_file)
run_pp({'keepvideo': False}, SimplePP)
self.assertFalse(os.path.exists(filename), f'{filename} exists')
self.assertTrue(os.path.exists(audiofile), f'{audiofile} doesn\'t exist')
os.unlink(audiofile)
run_pp({'keepvideo': False, 'outtmpl': filename}, SimplePP)
self.assertFalse(os.path.exists(video_file), f'{video_file} exists')
self.assertTrue(os.path.exists(audio_file), f'{audio_file} doesn\'t exist')
os.unlink(audio_file)
class ModifierPP(PostProcessor):
def run(self, info):
@ -995,9 +996,9 @@ def run(self, info):
f.write('MODIFIED')
return [], info
run_pp({'keepvideo': False}, ModifierPP)
self.assertTrue(os.path.exists(filename), f'{filename} doesn\'t exist')
os.unlink(filename)
run_pp({'keepvideo': False, 'outtmpl': filename}, ModifierPP)
self.assertTrue(os.path.exists(video_file), f'{video_file} doesn\'t exist')
os.unlink(video_file)
def test_match_filter(self):
first = {

View file

@ -3266,7 +3266,6 @@ def replace_info_dict(new_info):
# info_dict['_filename'] needs to be set for backward compatibility
info_dict['_filename'] = full_filename = self.prepare_filename(info_dict, warn=True)
temp_filename = self.prepare_filename(info_dict, 'temp')
files_to_move = {}
# Forced printings
self.__forced_printings(info_dict, full_filename, incomplete=('format' not in info_dict))
@ -3294,13 +3293,11 @@ def check_max_downloads():
sub_files = self._write_subtitles(info_dict, temp_filename)
if sub_files is None:
return
files_to_move.update(dict(sub_files))
thumb_files = self._write_thumbnails(
'video', info_dict, temp_filename, self.prepare_filename(info_dict, 'thumbnail'))
if thumb_files is None:
return
files_to_move.update(dict(thumb_files))
infofn = self.prepare_filename(info_dict, 'infojson')
_infojson_written = self._write_info_json('video', info_dict, infofn)
@ -3374,13 +3371,12 @@ def _write_link_file(link_type):
for link_type, should_write in write_links.items()):
return
new_info, files_to_move = self.pre_process(info_dict, 'before_dl', files_to_move)
new_info, _ = self.pre_process(info_dict, 'before_dl')
replace_info_dict(new_info)
if self.params.get('skip_download'):
info_dict['filepath'] = temp_filename
info_dict['__finaldir'] = os.path.dirname(os.path.abspath(full_filename))
info_dict['__files_to_move'] = files_to_move
replace_info_dict(self.run_pp(MoveFilesAfterDownloadPP(self, False), info_dict))
info_dict['__write_download_archive'] = self.params.get('force_write_download_archive')
else:
@ -3494,9 +3490,6 @@ def correct_ext(filename, ext=new_ext):
info_dict['__files_to_merge'] = downloaded
# Even if there were no downloads, it is being merged only now
info_dict['__real_download'] = True
else:
for file in downloaded:
files_to_move[file] = None
else:
# Just a single file
dl_filename = existing_video_file(full_filename, temp_filename)
@ -3581,7 +3574,7 @@ def ffmpeg_fixup(cndn, msg, cls):
fixup()
try:
replace_info_dict(self.post_process(dl_filename, info_dict, files_to_move))
replace_info_dict(self.post_process(dl_filename, info_dict))
except PostProcessingError as err:
self.report_error(f'Postprocessing: {err}')
return
@ -3704,8 +3697,6 @@ def _delete_downloaded_files(self, *files_to_delete, info={}, msg=None):
os.remove(filename)
except OSError:
self.report_warning(f'Unable to delete file {filename}')
if filename in info.get('__files_to_move', []): # NB: Delete even if None
del info['__files_to_move'][filename]
@staticmethod
def post_extract(info_dict):
@ -3722,8 +3713,7 @@ def actual_post_extract(info_dict):
def run_pp(self, pp, infodict):
files_to_delete = []
if '__files_to_move' not in infodict:
infodict['__files_to_move'] = {}
try:
files_to_delete, infodict = pp.run(infodict)
except PostProcessingError as e:
@ -3735,10 +3725,7 @@ def run_pp(self, pp, infodict):
if not files_to_delete:
return infodict
if self.params.get('keepvideo', False):
for f in files_to_delete:
infodict['__files_to_move'].setdefault(f, '')
else:
if not self.params.get('keepvideo', False):
self._delete_downloaded_files(
*files_to_delete, info=infodict, msg='Deleting original file %s (pass -k to keep)')
return infodict
@ -3751,23 +3738,27 @@ def run_all_pps(self, key, info, *, additional_pps=None):
return info
def pre_process(self, ie_info, key='pre_process', files_to_move=None):
if files_to_move is not None:
self.report_warning('[pre_process] "files_to_move" is deprecated and may be removed in a future version')
info = dict(ie_info)
info['__files_to_move'] = files_to_move or {}
try:
info = self.run_all_pps(key, info)
except PostProcessingError as err:
msg = f'Preprocessing: {err}'
info.setdefault('__pending_error', msg)
self.report_error(msg, is_error=False)
return info, info.pop('__files_to_move', None)
return info, files_to_move
def post_process(self, filename, info, files_to_move=None):
"""Run all the postprocessors on the given file."""
if files_to_move is not None:
self.report_warning('[post_process] "files_to_move" is deprecated and may be removed in a future version')
info['filepath'] = filename
info['__files_to_move'] = files_to_move or {}
info = self.run_all_pps('post_process', info, additional_pps=info.get('__postprocessors'))
info = self.run_pp(MoveFilesAfterDownloadPP(self), info)
del info['__files_to_move']
info.pop('__multiple_thumbnails', None)
return self.run_all_pps('after_move', info)
def _make_archive_id(self, info_dict):
@ -4347,10 +4338,11 @@ def _write_subtitles(self, info_dict, filename):
sub_filename = subtitles_filename(filename, sub_lang, sub_format, info_dict.get('ext'))
sub_filename_final = subtitles_filename(sub_filename_base, sub_lang, sub_format, info_dict.get('ext'))
existing_sub = self.existing_file((sub_filename_final, sub_filename))
if existing_sub:
self.to_screen(f'[info] Video subtitle {sub_lang}.{sub_format} is already present')
sub_info['filepath'] = existing_sub
ret.append((existing_sub, sub_filename_final))
ret.append(existing_sub)
continue
self.to_screen(f'[info] Writing video subtitles to: {sub_filename}')
@ -4361,7 +4353,7 @@ def _write_subtitles(self, info_dict, filename):
with open(sub_filename, 'w', encoding='utf-8', newline='') as subfile:
subfile.write(sub_info['data'])
sub_info['filepath'] = sub_filename
ret.append((sub_filename, sub_filename_final))
ret.append(sub_filename)
continue
except OSError:
self.report_error(f'Cannot write video subtitles file {sub_filename}')
@ -4372,7 +4364,7 @@ def _write_subtitles(self, info_dict, filename):
sub_copy.setdefault('http_headers', info_dict.get('http_headers'))
self.dl(sub_filename, sub_copy, subtitle=True)
sub_info['filepath'] = sub_filename
ret.append((sub_filename, sub_filename_final))
ret.append(sub_filename)
except (DownloadError, ExtractorError, OSError, ValueError, *network_exceptions) as err:
msg = f'Unable to download video subtitles for {sub_lang!r}: {err}'
if self.params.get('ignoreerrors') is not True: # False or 'only_download'
@ -4392,6 +4384,7 @@ def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None
self.to_screen(f'[info] There are no {label} thumbnails to download')
return ret
multiple = write_all and len(thumbnails) > 1
info_dict['__multiple_thumbnails'] = multiple
if thumb_filename_base is None:
thumb_filename_base = filename
@ -4415,7 +4408,7 @@ def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None
self.to_screen('[info] {} is already present'.format((
thumb_display_id if multiple else f'{label} thumbnail').capitalize()))
t['filepath'] = existing_thumb
ret.append((existing_thumb, thumb_filename_final))
ret.append(existing_thumb)
else:
self.to_screen(f'[info] Downloading {thumb_display_id} ...')
try:
@ -4423,7 +4416,7 @@ def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None
self.to_screen(f'[info] Writing {thumb_display_id} to: {thumb_filename}')
with open(thumb_filename, 'wb') as thumbf:
shutil.copyfileobj(uf, thumbf)
ret.append((thumb_filename, thumb_filename_final))
ret.append(thumb_filename)
t['filepath'] = thumb_filename
except network_exceptions as err:
if isinstance(err, HTTPError) and err.status == 404:
@ -4433,4 +4426,5 @@ def _write_thumbnails(self, label, info_dict, filename, thumb_filename_base=None
thumbnails.pop(idx)
if ret and not write_all:
break
return ret

View file

@ -230,4 +230,8 @@ def run(self, info):
thumbnail_filename if converted or not self._already_have_thumbnail else None,
original_thumbnail if converted and not self._already_have_thumbnail else None,
info=info)
if not self._already_have_thumbnail:
info['thumbnails'][idx].pop('filepath', None)
return [], info

View file

@ -662,6 +662,10 @@ def run(self, info):
self.run_ffmpeg_multiple_files(input_files, temp_filename, opts)
os.replace(temp_filename, filename)
if not self._already_have_subtitle:
for _, subtitle in subtitles.items():
subtitle.pop('filepath', None)
files_to_delete = [] if self._already_have_subtitle else sub_filenames
return files_to_delete, info
@ -698,6 +702,7 @@ def run(self, info):
infojson_filename = info.get('infojson_filename')
options.extend(self._get_infojson_opts(info, infojson_filename))
if not infojson_filename:
info.pop('infojson_filename', None)
files_to_delete.append(info.get('infojson_filename'))
elif self._add_infojson is True:
self.to_screen('The info-json can only be attached to mkv/mka files')
@ -1015,9 +1020,6 @@ def run(self, info):
'filepath': new_file,
}
info['__files_to_move'][new_file] = replace_extension(
info['__files_to_move'][sub['filepath']], new_ext)
return sub_filenames, info
@ -1082,16 +1084,15 @@ def is_webp(cls, path):
return imghdr.what(path) == 'webp'
def fixup_webp(self, info, idx=-1):
thumbnail_filename = info['thumbnails'][idx]['filepath']
thumbnail = info['thumbnails'][idx]
thumbnail_filename = thumbnail['filepath']
_, thumbnail_ext = os.path.splitext(thumbnail_filename)
if thumbnail_ext:
if thumbnail_ext.lower() != '.webp' and imghdr.what(thumbnail_filename) == 'webp':
self.to_screen(f'Correcting thumbnail "{thumbnail_filename}" extension to webp')
webp_filename = replace_extension(thumbnail_filename, 'webp')
os.replace(thumbnail_filename, webp_filename)
info['thumbnails'][idx]['filepath'] = webp_filename
info['__files_to_move'][webp_filename] = replace_extension(
info['__files_to_move'].pop(thumbnail_filename), 'webp')
thumbnail['filepath'] = webp_filename
@staticmethod
def _options(target_ext):
@ -1129,8 +1130,6 @@ def run(self, info):
continue
thumbnail_dict['filepath'] = self.convert_thumbnail(original_thumbnail, target_ext)
files_to_delete.append(original_thumbnail)
info['__files_to_move'][thumbnail_dict['filepath']] = replace_extension(
info['__files_to_move'][original_thumbnail], target_ext)
if not has_thumbnail:
self.to_screen('There aren\'t any thumbnails to convert')

View file

@ -1,14 +1,22 @@
import os
from pathlib import Path
from .common import PostProcessor
from ..compat import shutil
from ..utils import (
PostProcessingError,
make_dir,
replace_extension,
)
class MoveFilesAfterDownloadPP(PostProcessor):
# Map of the keys that contain moveable files and the 'type' of the file
# for generating the output filename
CHILD_KEYS = {
'thumbnails': 'thumbnail',
'requested_subtitles': 'subtitle',
}
def __init__(self, downloader=None, downloaded=True):
PostProcessor.__init__(self, downloader)
@ -18,33 +26,77 @@ def __init__(self, downloader=None, downloaded=True):
def pp_key(cls):
return 'MoveFiles'
def move_file_and_write_to_info(self, info_dict, relevant_dict=None, output_file_type=None):
relevant_dict = relevant_dict or info_dict
if 'filepath' not in relevant_dict:
return
output_file_type = output_file_type or ''
current_filepath, final_filepath = self.determine_filepath(info_dict, relevant_dict, output_file_type)
move_result = self.move_file(info_dict, current_filepath, final_filepath)
if move_result:
relevant_dict['filepath'] = move_result
else:
del relevant_dict['filepath']
def determine_filepath(self, info_dict, relevant_dict, output_file_type):
current_filepath = relevant_dict['filepath']
prepared_filepath = self._downloader.prepare_filename(info_dict, output_file_type)
if (output_file_type == 'thumbnail' and info_dict['__multiple_thumbnails']) or output_file_type == 'subtitle':
desired_extension = ''.join(Path(current_filepath).suffixes[-2:])
else:
desired_extension = Path(current_filepath).suffix
return current_filepath, replace_extension(prepared_filepath, desired_extension[1:])
def move_file(self, info_dict, current_filepath, final_filepath):
if not current_filepath or not final_filepath:
return
dl_parent_folder = os.path.split(info_dict['filepath'])[0]
finaldir = info_dict.get('__finaldir', os.path.abspath(dl_parent_folder))
if not os.path.isabs(current_filepath):
current_filepath = os.path.join(finaldir, current_filepath)
if not os.path.isabs(final_filepath):
final_filepath = os.path.join(finaldir, final_filepath)
if current_filepath == final_filepath:
return final_filepath
if not os.path.exists(current_filepath):
self.report_warning(f'File "{current_filepath}" cannot be found')
return
if os.path.exists(final_filepath):
if self.get_param('overwrites', True):
self.report_warning(f'Replacing existing file "{final_filepath}"')
os.remove(final_filepath)
else:
self.report_warning(f'Cannot move file "{current_filepath}" out of temporary directory since "{final_filepath}" already exists. ')
return
make_dir(final_filepath, PostProcessingError)
self.to_screen(f'Moving file "{current_filepath}" to "{final_filepath}"')
shutil.move(current_filepath, final_filepath) # os.rename cannot move between volumes
return final_filepath
def run(self, info):
dl_path, dl_name = os.path.split(info['filepath'])
finaldir = info.get('__finaldir', dl_path)
finalpath = os.path.join(finaldir, dl_name)
if self._downloaded:
info['__files_to_move'][info['filepath']] = finalpath
# This represents the main media file (using the 'filepath' key)
self.move_file_and_write_to_info(info)
make_newfilename = lambda old: os.path.join(finaldir, os.path.basename(old))
for oldfile, newfile in info['__files_to_move'].items():
if not newfile:
newfile = make_newfilename(oldfile)
if os.path.abspath(oldfile) == os.path.abspath(newfile):
for key, output_file_type in self.CHILD_KEYS.items():
if key not in info:
continue
if not os.path.exists(oldfile):
self.report_warning(f'File "{oldfile}" cannot be found')
continue
if os.path.exists(newfile):
if self.get_param('overwrites', True):
self.report_warning(f'Replacing existing file "{newfile}"')
os.remove(newfile)
else:
self.report_warning(
f'Cannot move file "{oldfile}" out of temporary directory since "{newfile}" already exists. ')
continue
make_dir(newfile, PostProcessingError)
self.to_screen(f'Moving file "{oldfile}" to "{newfile}"')
shutil.move(oldfile, newfile) # os.rename cannot move between volumes
info['filepath'] = finalpath
if isinstance(info[key], (dict, list)):
iterable = info[key].values() if isinstance(info[key], dict) else info[key]
for file_dict in iterable:
self.move_file_and_write_to_info(info, file_dict, output_file_type)
return [], info