rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 02:48:38 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Getting file name from yt-dlp error when video file name is too long, trimming that long file name.

Browse source
This commit is contained in:
Roland 2024-10-03 21:58:39 -06:00
parent 82e4e4797a
commit 584fdadcb7
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
8 changed files with 36 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

9
main.py
View file

@ -21,6 +21,7 @@ from wrolpi.common import logger, check_media_directory, set_log_level, limit_co
from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initialize_configs_contexts
from wrolpi.dates import Seconds
from wrolpi.downloader import import_downloads_config, download_manager
from wrolpi.errors import WROLModeEnabled
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, INTERNET_SERVER
from wrolpi.version import get_version_string
@ -203,7 +204,6 @@ async def handle_server_shutdown_reset(app: Sanic, loop):
reset_shared_contexts(app)
# Start periodic tasks after configs are ready.
@api_app.after_server_start
async def start_single_tasks(app: Sanic):
"""Recurring/Single tasks that are started in only one Sanic process."""
@ -214,6 +214,9 @@ async def start_single_tasks(app: Sanic):
logger.debug(f'start_single_tasks started')
# Do not start downloads until configs have been imported.
download_manager.disable()
# Import configs, ignore errors so the service will start. Configs will refuse to save if they failed to import.
with suppress(Exception):
get_wrolpi_config().import_config()
@ -236,8 +239,6 @@ async def start_single_tasks(app: Sanic):
if get_wrolpi_config().download_on_startup:
download_manager.enable()
else:
download_manager.disable()
from modules.zim.lib import flag_outdated_zim_files
try:
@ -304,6 +305,8 @@ async def perpetual_have_internet_worker():
else:
# Check more often until the internet is back.
flags.have_internet.clear()
except WROLModeEnabled:
flags.have_internet.clear()
except Exception as e:
logger.error('Failed to check if internet is up', exc_info=e)

4
modules/videos/channel/lib.py
View file

@ -274,10 +274,6 @@ async def tag_channel(tag_name: str | None, directory: pathlib.Path | None, chan
channel = Channel.find_by_id(channel_id, session)
# Do not move Channel files into a directory with files.
if directory and directory != channel.directory and directory.is_dir() and next(directory.iterdir(), None):
raise ChannelDirectoryConflict('Channel directory already exists and is not empty')
# May also clear the tag if `tag_name` is None.
channel.set_tag(tag_name)
channel.flush()

1
modules/videos/channel/test/test_api.py
View file

@ -584,6 +584,7 @@ async def test_move_channel_after_terminal_move(async_client, test_session, test
vid1 = video_factory(channel_id=channel.id)
test_session.commit()
# TODO this does not seem to work
assert str(channel.directory) == str(test_directory / 'videos/Channel Name')
assert str(vid1.video_path).startswith(str(test_directory / 'videos/Channel Name/'))

47
modules/videos/downloader.py
View file

@ -485,7 +485,7 @@ class VideoDownloader(Downloader, ABC):
@staticmethod
def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]:
"""Get the full path of a video file from its URL."""
"""Get the full path of a video file from its URL using yt-dlp."""
if not out_dir.is_dir():
raise ValueError(f'Output directory does not exist! {out_dir=}')
@ -506,28 +506,35 @@ class VideoDownloader(Downloader, ABC):
try:
entry = extract_info(url, ydl=ydl, process=True)
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
final_filename = trim_file_name(final_filename)
except DownloadError as e:
if ' Cannot write ' in str(e):
# yt-dlp does not handle long file names well, get the name from the error (lol)
last_line = str(e).splitlines()[-1]
filename = last_line.split(' file ')[-1].strip()
full_path, _ = split_path_stem_and_suffix(filename, full=True)
if not full_path.startswith('/'):
logger.error(f'Failed to extract filename from {last_line}')
raise
# Trim long filename, add video suffix.
final_filename = pathlib.Path(f'{full_path}.{PREFERRED_VIDEO_EXTENSION}')
final_filename = trim_file_name(final_filename)
# Get entry info json.
options['outtmpl'] = str(final_filename)
ydl = YoutubeDL(options)
ydl.params['logger'] = ydl_logger
ydl.add_default_info_extractors()
entry = extract_info(url, ydl=ydl, process=True)
else:
if ' Cannot write ' not in str(e):
raise
# yt-dlp does not handle long file names well, get the name from the error (lol)
last_line = str(e).splitlines()[-1]
full_path = last_line.split(' file ')[-1].strip()
if not full_path.startswith('/'):
logger.error(f'Failed to extract filename from {last_line}')
raise
# Split filename from parent.
full_path = pathlib.Path(full_path)
parent = full_path.parent
filename, _ = split_path_stem_and_suffix(full_path.name)
# Trim long filename, add video suffix, add back into parent directory.
filename = escape_file_name(filename)
final_filename = parent / f'{filename}.{PREFERRED_VIDEO_EXTENSION}'
final_filename = trim_file_name(final_filename)
logger.debug(f'Video file name was too long. Trimmed to: {final_filename.name}')
# Get entry info json.
options['outtmpl'] = str(final_filename)
ydl = YoutubeDL(options)
ydl.params['logger'] = ydl_logger
ydl.add_default_info_extractors()
entry = extract_info(url, ydl=ydl, process=True)
logger.debug(f'Downloading {url} to {repr(str(final_filename))}')
return final_filename, entry

1
modules/videos/lib.py
View file

@ -311,7 +311,6 @@ class ChannelsConfig(ConfigFile):
link_channel_and_downloads(session)
self.successful_import = True
channel_import_logger.info('Importing channels config complete')
except Exception as e:
self.successful_import = False
message = f'Failed to import {self.get_relative_file()} config!'

1
wrolpi/common.py
View file

@ -2095,6 +2095,7 @@ def get_title_from_html(html: str, url: str = None) -> str:
logger_.debug(f'Unable to extract title {url}')
@wrol_mode_check
def can_connect_to_server(hostname: str, port: int = 80) -> bool:
"""Return True only if `hostname` responds."""
# Thanks https://stackoverflow.com/questions/20913411/test-if-an-internet-connection-is-present-in-python

2
wrolpi/downloader.py
View file

@ -347,7 +347,7 @@ class Downloader:
# Output all logs from the process.
# TODO is there a way to stream this output while the process is running?
logger.debug(f'Download exited with {proc.returncode}')
logs = {'stdout': stdout, 'stderr': stderr}
logs = {'stdout': stdout or '', 'stderr': stderr or ''}
return proc.returncode, logs, stdout

11
wrolpi/test/test_common.py
View file

@ -308,17 +308,6 @@ def test_escape_file_name(name, expected):
raise AssertionError(f'Escape of {repr(name)}: {repr(result)} != {repr(expected)}')
@pytest.mark.parametrize('name,expected', [
('Channel_20240714_aaa.jpg', 'Channel_20240714_aaa.jpg'),
# Really long filename. The suffix is preserved.
('Channel_20240714_' + ('a' * 244) + '.jpg', 'Channel_20240714_' + ('a' * 159) + '.jpg'),
])
def test_trim_file_name(name, expected):
result = common.trim_file_name(name)
assert result == expected, f'Expected length of {len(expected)} but got length of {len(result)}'
assert len(result) <= 255, 'File name cannot be longer than 255 characters'
def test_truncate_object_bytes():
"""
Objects can be truncated (lists will be shortened) so they will fit in tsvector columns.