rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-25 18:40:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Adding FileGroup.censored when video can no longer be downloaded.

Browse source
This commit is contained in:
Roland 2024-06-22 10:46:45 -06:00
parent 91efa9b2f3
commit a6647cc8f7
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
3 changed files with 26 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

5
main.py
View file

@ -12,7 +12,7 @@ from wrolpi import root_api # noqa
from wrolpi import tags
from wrolpi.api_utils import api_app
from wrolpi.common import logger, check_media_directory, set_log_level, limit_concurrent, \
cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, can_connect_to_server
cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, can_connect_to_server, wrol_mode_enabled
from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initialize_configs_contexts
from wrolpi.dates import Seconds
from wrolpi.downloader import import_downloads_config, download_manager, \
@ -174,6 +174,9 @@ async def initialize_configs(app: Sanic):
try:
initialize_configs_contexts(app)
await asyncio.sleep(0.5)
wrol_mode_enabled()
logger.info('initialize_configs succeeded')
except Exception as e:
logger.error('initialize_configs failed with', exc_info=e)
raise

28
modules/videos/video/lib.py
View file

@ -202,6 +202,7 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
Video.comments_failed == False,
# Need a URL to try to get comments.
FileGroup.url != None, # noqa
FileGroup.censored != True, # noqa
# We want old videos (time for comments to accumulate), or those which we don't know the published date.
or_(
one_month_ago > FileGroup.published_datetime,
@ -214,12 +215,14 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
logger.info(f'Found {len(video_urls)} videos missing comments')
def add_video_to_skip_list(video_url: str):
def add_video_to_skip_list(video_url: str, censored: bool = False):
"""Do not attempt to get comments of the provided Video again."""
with get_db_session(commit=True) as session_:
for video_ in session_.query(Video).join(FileGroup).filter_by(url=video_url).all():
for video_, fg in session_.query(Video, FileGroup).join(FileGroup).filter_by(url=video_url).all():
video_.comments_failed = True
fg.censored = censored
download_manager.add_to_skip_list(video_url)
logger.debug(f'add_video_to_skip_list: {video_url=} {censored=}')
for url in video_urls:
logger.info(f'Getting comments for: {url}')
@ -258,19 +261,28 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
except Exception as e:
if 'no longer available' in str(e):
logger.error(f'Giving up on downloading comments because video does not exist: {url=}', exc_info=e)
add_video_to_skip_list(url)
add_video_to_skip_list(url, censored=True)
elif 'Comments are turned off' in str(e):
logger.error(f'Giving up on downloading comments because comments are disabled: {url=}', exc_info=e)
add_video_to_skip_list(url)
elif 'Private video' in str(e):
logger.error(f'Giving up on downloading comments because video is private: {url=}', exc_info=e)
add_video_to_skip_list(url)
add_video_to_skip_list(url, censored=True)
elif 'Video unavailable' in str(e):
logger.error(f'Giving up on downloading comments because video is unavailable: {url=}', exc_info=e)
add_video_to_skip_list(url)
elif ' members ' in str(e):
add_video_to_skip_list(url, censored=True)
elif 'video has been removed' in str(e):
logger.error(f'Giving up on downloading comments because video is removed: {url=}', exc_info=e)
add_video_to_skip_list(url, censored=True)
elif ' members' in str(e):
logger.error(f'Giving up on downloading comments because video is members-only: {url=}', exc_info=e)
add_video_to_skip_list(url)
add_video_to_skip_list(url, censored=True)
elif 'rumble.com' in url and "'bool' object has no attribute 'get'" in str(e):
logger.error(f'Rumble error in yt-dlp', exc_info=e)
add_video_to_skip_list(url, censored=False)
elif 'not a valid URL' in str(e):
logger.error(f'URL is not valid: {url=}', exc_info=e)
add_video_to_skip_list(url, censored=True)
else:
logger.error(f'Got error when attempting to download video comments: {url=}', exc_info=e)
continue
@ -286,7 +298,7 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
else:
logger.error(f'Attempting to replace comments for non-existent video! {video}')
if len(videos) == 1:
download_manager.add_to_skip_list(video.url)
download_manager.add_to_skip_list(url)
video.comments_failed = True
# Sleep a random amount of time, so we don't spam.

2
wrolpi/api_utils.py
View file

@ -105,6 +105,8 @@ def json_error_handler(request: Request, exception: Exception):
if PYTEST:
logger.error('Unexpected error', exc_info=exception)
else:
logger.debug('Unexpected error', exc_info=exception)
# Some unknown error, use internal error code.
return json_response(body, HTTPStatus.INTERNAL_SERVER_ERROR)