From dbdf035474d0df21016e3f6309c95c37fc13c855 Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 14 Oct 2019 15:19:06 -0600 Subject: [PATCH] Refresh cleans up video records. --- wrolpi/plugins/videos/api.py | 35 ++++++++++++++----- wrolpi/plugins/videos/common.py | 7 ++-- wrolpi/plugins/videos/downloader.py | 20 ++++++++--- .../videos/templates/channels_settings.html | 2 +- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/wrolpi/plugins/videos/api.py b/wrolpi/plugins/videos/api.py index f218f763..ab66b018 100644 --- a/wrolpi/plugins/videos/api.py +++ b/wrolpi/plugins/videos/api.py @@ -1,4 +1,5 @@ import json +import os import pathlib import cherrypy @@ -36,10 +37,20 @@ class SettingsAPI(object): def json_statuses_streamer(func): - """Wraps status lines in JSON objects. Adds a new line after each JSON string.""" + """Wraps status lines in JSON objects. Adds a new line after each JSON string. + + Example: + >>> func = json_statuses_streamer(lambda : ['foo', 'bar']) + >>> func() + '{"status": "foo"}\n' + '{"status": "bar"}\n' + '{"success": "stream-complete"}\n' + """ + def wrap(*a, **kw): yield from (json.dumps({'status': i}) + '\n' for i in func(*a, **kw)) yield json.dumps({'success': 'stream-complete'}) + '\n' + return wrap @@ -195,12 +206,13 @@ def get_channel_form(form_data: dict): def refresh_channel_videos(db, channel): existing_paths = {i['video_path'] for i in channel['videos']} directory = resolve_project_path(channel['directory']) - if not pathlib.Path(directory).is_dir(): + if not directory.is_dir(): logger.warn(f'Channel {channel["name"]} directory "{directory}" does not exist, skipping...') logger.warn(f'Have you downloaded any videos for channel {channel["name"]}?') return - possible_new_paths = {str(i) for i in generate_video_paths(directory)} + channel_directory = resolve_project_path(channel['directory']).absolute() + possible_new_paths = {str(i) for i in generate_video_paths(directory, relative_to=channel_directory)} new_videos = possible_new_paths.difference(existing_paths) for video_path in new_videos: @@ -228,14 +240,21 @@ def refresh_videos(db: DictDB): # Remove any videos that don't exist yield 'Verifying videos in DB exist in file system' curs = db.get_cursor() - curs.execute('SELECT id, video_path FROM video') + curs.execute('SELECT DISTINCT ON (video_path) video_path, video.id, directory AS channel_directory FROM ' + 'video LEFT JOIN channel ON video.channel_id = channel.id') existing_videos = curs.fetchall() - to_delete = [] + to_keep = [] for video in existing_videos: - if not pathlib.Path(video['video_path']).is_file(): - to_delete.append(video['id']) + channel_directory = resolve_project_path(video['channel_directory']) + video_path = resolve_project_path(channel_directory / video['video_path']) + if video_path.is_file(): + to_keep.append(video['id']) + + yield 'Deleting video records no longer in file system' + curs.execute('SELECT id FROM video WHERE id != ALL(%s)', (to_keep,)) + to_delete = [i for (i,) in curs.fetchall()] if to_delete: - yield 'Deleting video files no longer in file system' + logger.info(f'Deleting video records: {to_delete}') curs.execute('DELETE FROM video WHERE id = ANY(%s)', (to_delete,)) for channel in Channel.get_where(): diff --git a/wrolpi/plugins/videos/common.py b/wrolpi/plugins/videos/common.py index c28a2a4d..9370331d 100644 --- a/wrolpi/plugins/videos/common.py +++ b/wrolpi/plugins/videos/common.py @@ -145,7 +145,7 @@ def any_extensions(filename: str, extensions=None): match_video_extensions = partial(any_extensions, extensions={'mp4', 'webm', 'flv'}) -def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, pathlib.Path]: +def generate_video_paths(directory: Union[str, pathlib.Path], relative_to=None) -> Tuple[str, pathlib.Path]: """ Generate a list of video paths in the provided directory. """ @@ -154,7 +154,10 @@ def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, path for child in directory.iterdir(): if child.is_file() and match_video_extensions(str(child)): child = child.absolute() - yield child + if relative_to: + yield child.relative_to(relative_to) + else: + yield child elif child.is_dir(): yield from generate_video_paths(child) diff --git a/wrolpi/plugins/videos/downloader.py b/wrolpi/plugins/videos/downloader.py index db162f54..5e6193b6 100755 --- a/wrolpi/plugins/videos/downloader.py +++ b/wrolpi/plugins/videos/downloader.py @@ -166,7 +166,7 @@ def replace_extension(path: pathlib.Path, new_ext) -> pathlib.Path: return path -def find_meta_files(path: pathlib.Path) -> Tuple[pathlib.Path, pathlib.Path, pathlib.Path, pathlib.Path]: +def find_meta_files(path: pathlib.Path, relative_to=None) -> Tuple[pathlib.Path, pathlib.Path, pathlib.Path, pathlib.Path]: """Find all files that share a file's full path, except their extensions. It is assumed that file with the same name, but different extension is related to that file. @@ -181,8 +181,13 @@ def find_meta_files(path: pathlib.Path) -> Tuple[pathlib.Path, pathlib.Path, pat meta_file_exts = ('.jpg', '.description', '.en.vtt', '.info.json') for meta_ext in meta_file_exts: meta_path = replace_extension(path, meta_ext) - meta_path = meta_path if meta_path.exists() else None - yield meta_path + if meta_path.exists(): + if relative_to: + yield meta_path.relative_to(relative_to) + else: + yield meta_path + else: + yield None NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.{11})_)?(.*)\.' @@ -190,9 +195,14 @@ NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.{11})_)?(.*)\.' def insert_video(db: DictDB, video_path: pathlib.Path, channel: Table, idempotency: str = None): - """Find an insert a video file's information into the DB.""" + """Find and insert a video file's information into the DB.""" Video = db['video'] - poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path) + channel_dir = resolve_project_path(str(channel['directory'])).absolute() + poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path, relative_to=channel_dir) + + # Video paths should be relative to the channel's directory + video_path = video_path.relative_to(channel_dir) + name_match = NAME_PARSER.match(video_path.name) _ = upload_date = source_id = title = ext = None if name_match: diff --git a/wrolpi/plugins/videos/templates/channels_settings.html b/wrolpi/plugins/videos/templates/channels_settings.html index 3ef17dbf..b66ad83f 100644 --- a/wrolpi/plugins/videos/templates/channels_settings.html +++ b/wrolpi/plugins/videos/templates/channels_settings.html @@ -37,7 +37,7 @@
- {{ post_form_button('refresh', 'dark', 'Refresh Video Files') }} + {{ post_form_button('refresh', 'dark', 'Refresh Video Files.') }}