Refresh cleans up video records.

This commit is contained in:
Roland 2019-10-14 15:19:06 -06:00
parent 0ce9f5fca7
commit dbdf035474
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
4 changed files with 48 additions and 16 deletions

View file

@ -1,4 +1,5 @@
import json
import os
import pathlib
import cherrypy
@ -36,10 +37,20 @@ class SettingsAPI(object):
def json_statuses_streamer(func):
"""Wraps status lines in JSON objects. Adds a new line after each JSON string."""
"""Wraps status lines in JSON objects. Adds a new line after each JSON string.
Example:
>>> func = json_statuses_streamer(lambda : ['foo', 'bar'])
>>> func()
'{"status": "foo"}\n'
'{"status": "bar"}\n'
'{"success": "stream-complete"}\n'
"""
def wrap(*a, **kw):
yield from (json.dumps({'status': i}) + '\n' for i in func(*a, **kw))
yield json.dumps({'success': 'stream-complete'}) + '\n'
return wrap
@ -195,12 +206,13 @@ def get_channel_form(form_data: dict):
def refresh_channel_videos(db, channel):
existing_paths = {i['video_path'] for i in channel['videos']}
directory = resolve_project_path(channel['directory'])
if not pathlib.Path(directory).is_dir():
if not directory.is_dir():
logger.warn(f'Channel {channel["name"]} directory "{directory}" does not exist, skipping...')
logger.warn(f'Have you downloaded any videos for channel {channel["name"]}?')
return
possible_new_paths = {str(i) for i in generate_video_paths(directory)}
channel_directory = resolve_project_path(channel['directory']).absolute()
possible_new_paths = {str(i) for i in generate_video_paths(directory, relative_to=channel_directory)}
new_videos = possible_new_paths.difference(existing_paths)
for video_path in new_videos:
@ -228,14 +240,21 @@ def refresh_videos(db: DictDB):
# Remove any videos that don't exist
yield 'Verifying videos in DB exist in file system'
curs = db.get_cursor()
curs.execute('SELECT id, video_path FROM video')
curs.execute('SELECT DISTINCT ON (video_path) video_path, video.id, directory AS channel_directory FROM '
'video LEFT JOIN channel ON video.channel_id = channel.id')
existing_videos = curs.fetchall()
to_delete = []
to_keep = []
for video in existing_videos:
if not pathlib.Path(video['video_path']).is_file():
to_delete.append(video['id'])
channel_directory = resolve_project_path(video['channel_directory'])
video_path = resolve_project_path(channel_directory / video['video_path'])
if video_path.is_file():
to_keep.append(video['id'])
yield 'Deleting video records no longer in file system'
curs.execute('SELECT id FROM video WHERE id != ALL(%s)', (to_keep,))
to_delete = [i for (i,) in curs.fetchall()]
if to_delete:
yield 'Deleting video files no longer in file system'
logger.info(f'Deleting video records: {to_delete}')
curs.execute('DELETE FROM video WHERE id = ANY(%s)', (to_delete,))
for channel in Channel.get_where():

View file

@ -145,7 +145,7 @@ def any_extensions(filename: str, extensions=None):
match_video_extensions = partial(any_extensions, extensions={'mp4', 'webm', 'flv'})
def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, pathlib.Path]:
def generate_video_paths(directory: Union[str, pathlib.Path], relative_to=None) -> Tuple[str, pathlib.Path]:
"""
Generate a list of video paths in the provided directory.
"""
@ -154,7 +154,10 @@ def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, path
for child in directory.iterdir():
if child.is_file() and match_video_extensions(str(child)):
child = child.absolute()
yield child
if relative_to:
yield child.relative_to(relative_to)
else:
yield child
elif child.is_dir():
yield from generate_video_paths(child)

View file

@ -166,7 +166,7 @@ def replace_extension(path: pathlib.Path, new_ext) -> pathlib.Path:
return path
def find_meta_files(path: pathlib.Path) -> Tuple[pathlib.Path, pathlib.Path, pathlib.Path, pathlib.Path]:
def find_meta_files(path: pathlib.Path, relative_to=None) -> Tuple[pathlib.Path, pathlib.Path, pathlib.Path, pathlib.Path]:
"""Find all files that share a file's full path, except their extensions. It is assumed that file with the
same name, but different extension is related to that file.
@ -181,8 +181,13 @@ def find_meta_files(path: pathlib.Path) -> Tuple[pathlib.Path, pathlib.Path, pat
meta_file_exts = ('.jpg', '.description', '.en.vtt', '.info.json')
for meta_ext in meta_file_exts:
meta_path = replace_extension(path, meta_ext)
meta_path = meta_path if meta_path.exists() else None
yield meta_path
if meta_path.exists():
if relative_to:
yield meta_path.relative_to(relative_to)
else:
yield meta_path
else:
yield None
NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.{11})_)?(.*)\.'
@ -190,9 +195,14 @@ NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.{11})_)?(.*)\.'
def insert_video(db: DictDB, video_path: pathlib.Path, channel: Table, idempotency: str = None):
"""Find an insert a video file's information into the DB."""
"""Find and insert a video file's information into the DB."""
Video = db['video']
poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path)
channel_dir = resolve_project_path(str(channel['directory'])).absolute()
poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path, relative_to=channel_dir)
# Video paths should be relative to the channel's directory
video_path = video_path.relative_to(channel_dir)
name_match = NAME_PARSER.match(video_path.name)
_ = upload_date = source_id = title = ext = None
if name_match:

View file

@ -37,7 +37,7 @@
<div class="row">
<div class="col-4">
<form id="refresh" action="/api/videos/settings/refresh">
{{ post_form_button('refresh', 'dark', 'Refresh Video Files') }}
{{ post_form_button('refresh', 'dark', 'Refresh Video Files.') }}
</form>
</div>
<div class="col-8">