diff --git a/main.py b/main.py index 0bc26489..7d34348d 100755 --- a/main.py +++ b/main.py @@ -21,6 +21,7 @@ from wrolpi.common import logger, check_media_directory, set_log_level, limit_co from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initialize_configs_contexts from wrolpi.dates import Seconds from wrolpi.downloader import import_downloads_config, download_manager +from wrolpi.errors import WROLModeEnabled from wrolpi.vars import PROJECT_DIR, DOCKERIZED, INTERNET_SERVER from wrolpi.version import get_version_string @@ -203,7 +204,6 @@ async def handle_server_shutdown_reset(app: Sanic, loop): reset_shared_contexts(app) -# Start periodic tasks after configs are ready. @api_app.after_server_start async def start_single_tasks(app: Sanic): """Recurring/Single tasks that are started in only one Sanic process.""" @@ -214,6 +214,9 @@ async def start_single_tasks(app: Sanic): logger.debug(f'start_single_tasks started') + # Do not start downloads until configs have been imported. + download_manager.disable() + # Import configs, ignore errors so the service will start. Configs will refuse to save if they failed to import. with suppress(Exception): get_wrolpi_config().import_config() @@ -236,8 +239,6 @@ async def start_single_tasks(app: Sanic): if get_wrolpi_config().download_on_startup: download_manager.enable() - else: - download_manager.disable() from modules.zim.lib import flag_outdated_zim_files try: @@ -304,6 +305,8 @@ async def perpetual_have_internet_worker(): else: # Check more often until the internet is back. flags.have_internet.clear() + except WROLModeEnabled: + flags.have_internet.clear() except Exception as e: logger.error('Failed to check if internet is up', exc_info=e) diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index a55cd1e5..6d65a188 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -274,10 +274,6 @@ async def tag_channel(tag_name: str | None, directory: pathlib.Path | None, chan channel = Channel.find_by_id(channel_id, session) - # Do not move Channel files into a directory with files. - if directory and directory != channel.directory and directory.is_dir() and next(directory.iterdir(), None): - raise ChannelDirectoryConflict('Channel directory already exists and is not empty') - # May also clear the tag if `tag_name` is None. channel.set_tag(tag_name) channel.flush() diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index f17a153a..9c9c4aae 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -584,6 +584,7 @@ async def test_move_channel_after_terminal_move(async_client, test_session, test vid1 = video_factory(channel_id=channel.id) test_session.commit() + # TODO this does not seem to work assert str(channel.directory) == str(test_directory / 'videos/Channel Name') assert str(vid1.video_path).startswith(str(test_directory / 'videos/Channel Name/')) diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index d43f17d5..cad44104 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -485,7 +485,7 @@ class VideoDownloader(Downloader, ABC): @staticmethod def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]: - """Get the full path of a video file from its URL.""" + """Get the full path of a video file from its URL using yt-dlp.""" if not out_dir.is_dir(): raise ValueError(f'Output directory does not exist! {out_dir=}') @@ -506,28 +506,35 @@ class VideoDownloader(Downloader, ABC): try: entry = extract_info(url, ydl=ydl, process=True) final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute() - final_filename = trim_file_name(final_filename) except DownloadError as e: - if ' Cannot write ' in str(e): - # yt-dlp does not handle long file names well, get the name from the error (lol) - last_line = str(e).splitlines()[-1] - filename = last_line.split(' file ')[-1].strip() - full_path, _ = split_path_stem_and_suffix(filename, full=True) - if not full_path.startswith('/'): - logger.error(f'Failed to extract filename from {last_line}') - raise - # Trim long filename, add video suffix. - final_filename = pathlib.Path(f'{full_path}.{PREFERRED_VIDEO_EXTENSION}') - final_filename = trim_file_name(final_filename) - # Get entry info json. - options['outtmpl'] = str(final_filename) - ydl = YoutubeDL(options) - ydl.params['logger'] = ydl_logger - ydl.add_default_info_extractors() - entry = extract_info(url, ydl=ydl, process=True) - else: + if ' Cannot write ' not in str(e): raise + # yt-dlp does not handle long file names well, get the name from the error (lol) + last_line = str(e).splitlines()[-1] + full_path = last_line.split(' file ')[-1].strip() + if not full_path.startswith('/'): + logger.error(f'Failed to extract filename from {last_line}') + raise + + # Split filename from parent. + full_path = pathlib.Path(full_path) + parent = full_path.parent + filename, _ = split_path_stem_and_suffix(full_path.name) + + # Trim long filename, add video suffix, add back into parent directory. + filename = escape_file_name(filename) + final_filename = parent / f'{filename}.{PREFERRED_VIDEO_EXTENSION}' + final_filename = trim_file_name(final_filename) + logger.debug(f'Video file name was too long. Trimmed to: {final_filename.name}') + + # Get entry info json. + options['outtmpl'] = str(final_filename) + ydl = YoutubeDL(options) + ydl.params['logger'] = ydl_logger + ydl.add_default_info_extractors() + entry = extract_info(url, ydl=ydl, process=True) + logger.debug(f'Downloading {url} to {repr(str(final_filename))}') return final_filename, entry diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 95001b8c..af566c0e 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -311,7 +311,6 @@ class ChannelsConfig(ConfigFile): link_channel_and_downloads(session) self.successful_import = True - channel_import_logger.info('Importing channels config complete') except Exception as e: self.successful_import = False message = f'Failed to import {self.get_relative_file()} config!' diff --git a/wrolpi/common.py b/wrolpi/common.py index e8f8da8c..581b38c1 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -2095,6 +2095,7 @@ def get_title_from_html(html: str, url: str = None) -> str: logger_.debug(f'Unable to extract title {url}') +@wrol_mode_check def can_connect_to_server(hostname: str, port: int = 80) -> bool: """Return True only if `hostname` responds.""" # Thanks https://stackoverflow.com/questions/20913411/test-if-an-internet-connection-is-present-in-python diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 49f0cad0..02234fe7 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -347,7 +347,7 @@ class Downloader: # Output all logs from the process. # TODO is there a way to stream this output while the process is running? logger.debug(f'Download exited with {proc.returncode}') - logs = {'stdout': stdout, 'stderr': stderr} + logs = {'stdout': stdout or '', 'stderr': stderr or ''} return proc.returncode, logs, stdout diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 870500d4..8f5cae8f 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -308,17 +308,6 @@ def test_escape_file_name(name, expected): raise AssertionError(f'Escape of {repr(name)}: {repr(result)} != {repr(expected)}') -@pytest.mark.parametrize('name,expected', [ - ('Channel_20240714_aaa.jpg', 'Channel_20240714_aaa.jpg'), - # Really long filename. The suffix is preserved. - ('Channel_20240714_' + ('a' * 244) + '.jpg', 'Channel_20240714_' + ('a' * 159) + '.jpg'), -]) -def test_trim_file_name(name, expected): - result = common.trim_file_name(name) - assert result == expected, f'Expected length of {len(expected)} but got length of {len(result)}' - assert len(result) <= 255, 'File name cannot be longer than 255 characters' - - def test_truncate_object_bytes(): """ Objects can be truncated (lists will be shortened) so they will fit in tsvector columns.