diff --git a/main.py b/main.py index 4ab8fcb1..6dcacca1 100755 --- a/main.py +++ b/main.py @@ -12,7 +12,7 @@ from wrolpi import root_api # noqa from wrolpi import tags from wrolpi.api_utils import api_app from wrolpi.common import logger, check_media_directory, set_log_level, limit_concurrent, \ - cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, can_connect_to_server + cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, can_connect_to_server, wrol_mode_enabled from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initialize_configs_contexts from wrolpi.dates import Seconds from wrolpi.downloader import import_downloads_config, download_manager, \ @@ -174,6 +174,9 @@ async def initialize_configs(app: Sanic): try: initialize_configs_contexts(app) + await asyncio.sleep(0.5) + wrol_mode_enabled() + logger.info('initialize_configs succeeded') except Exception as e: logger.error('initialize_configs failed with', exc_info=e) raise diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index dc1012e0..423d40d2 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -202,6 +202,7 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT): Video.comments_failed == False, # Need a URL to try to get comments. FileGroup.url != None, # noqa + FileGroup.censored != True, # noqa # We want old videos (time for comments to accumulate), or those which we don't know the published date. or_( one_month_ago > FileGroup.published_datetime, @@ -214,12 +215,14 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT): logger.info(f'Found {len(video_urls)} videos missing comments') - def add_video_to_skip_list(video_url: str): + def add_video_to_skip_list(video_url: str, censored: bool = False): """Do not attempt to get comments of the provided Video again.""" with get_db_session(commit=True) as session_: - for video_ in session_.query(Video).join(FileGroup).filter_by(url=video_url).all(): + for video_, fg in session_.query(Video, FileGroup).join(FileGroup).filter_by(url=video_url).all(): video_.comments_failed = True + fg.censored = censored download_manager.add_to_skip_list(video_url) + logger.debug(f'add_video_to_skip_list: {video_url=} {censored=}') for url in video_urls: logger.info(f'Getting comments for: {url}') @@ -258,19 +261,28 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT): except Exception as e: if 'no longer available' in str(e): logger.error(f'Giving up on downloading comments because video does not exist: {url=}', exc_info=e) - add_video_to_skip_list(url) + add_video_to_skip_list(url, censored=True) elif 'Comments are turned off' in str(e): logger.error(f'Giving up on downloading comments because comments are disabled: {url=}', exc_info=e) add_video_to_skip_list(url) elif 'Private video' in str(e): logger.error(f'Giving up on downloading comments because video is private: {url=}', exc_info=e) - add_video_to_skip_list(url) + add_video_to_skip_list(url, censored=True) elif 'Video unavailable' in str(e): logger.error(f'Giving up on downloading comments because video is unavailable: {url=}', exc_info=e) - add_video_to_skip_list(url) - elif ' members ' in str(e): + add_video_to_skip_list(url, censored=True) + elif 'video has been removed' in str(e): + logger.error(f'Giving up on downloading comments because video is removed: {url=}', exc_info=e) + add_video_to_skip_list(url, censored=True) + elif ' members' in str(e): logger.error(f'Giving up on downloading comments because video is members-only: {url=}', exc_info=e) - add_video_to_skip_list(url) + add_video_to_skip_list(url, censored=True) + elif 'rumble.com' in url and "'bool' object has no attribute 'get'" in str(e): + logger.error(f'Rumble error in yt-dlp', exc_info=e) + add_video_to_skip_list(url, censored=False) + elif 'not a valid URL' in str(e): + logger.error(f'URL is not valid: {url=}', exc_info=e) + add_video_to_skip_list(url, censored=True) else: logger.error(f'Got error when attempting to download video comments: {url=}', exc_info=e) continue @@ -286,7 +298,7 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT): else: logger.error(f'Attempting to replace comments for non-existent video! {video}') if len(videos) == 1: - download_manager.add_to_skip_list(video.url) + download_manager.add_to_skip_list(url) video.comments_failed = True # Sleep a random amount of time, so we don't spam. diff --git a/wrolpi/api_utils.py b/wrolpi/api_utils.py index be0da6cb..26550727 100644 --- a/wrolpi/api_utils.py +++ b/wrolpi/api_utils.py @@ -105,6 +105,8 @@ def json_error_handler(request: Request, exception: Exception): if PYTEST: logger.error('Unexpected error', exc_info=exception) + else: + logger.debug('Unexpected error', exc_info=exception) # Some unknown error, use internal error code. return json_response(body, HTTPStatus.INTERNAL_SERVER_ERROR)