From d0d69519cd673cbccb71ef2a2f300a8a4ebe691e Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 6 Dec 2022 14:28:35 -0700 Subject: [PATCH] Bumping version for new feature. Skipping video epoch test on circleci. Created script which finds and cleans up (renames/deletes) orphaned video meta files. Reworking orphaned video query, some video files were leaking through. Modifying File table to allow finding of orphaned video files. Extracting epoch from video info_json for more accurate upload_date. Extracting epoch from video info_json for more accurate upload_date. --- .../6489cd50c889_file_stem_and_directory.py | 43 ++++++ modules/videos/__init__.py | 7 +- modules/videos/downloader.py | 2 + modules/videos/lib.py | 64 +++++++-- modules/videos/test/test_api.py | 1 - modules/videos/test/test_lib.py | 84 +++++++++++ scripts/find_and_rename_orphan_video_files.py | 131 ++++++++++++++++++ wrolpi/files/models.py | 20 ++- wrolpi/files/test/test_lib.py | 23 +-- wrolpi/version.py | 2 +- 10 files changed, 347 insertions(+), 30 deletions(-) create mode 100644 alembic/versions/6489cd50c889_file_stem_and_directory.py create mode 100755 scripts/find_and_rename_orphan_video_files.py diff --git a/alembic/versions/6489cd50c889_file_stem_and_directory.py b/alembic/versions/6489cd50c889_file_stem_and_directory.py new file mode 100644 index 00000000..20dc7db1 --- /dev/null +++ b/alembic/versions/6489cd50c889_file_stem_and_directory.py @@ -0,0 +1,43 @@ +"""File stem and directory + +Revision ID: 6489cd50c889 +Revises: 13542a230c78 +Create Date: 2022-12-04 15:39:55.485355 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '6489cd50c889' +down_revision = '13542a230c78' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('ALTER TABLE file ADD COLUMN directory TEXT') + session.execute('ALTER TABLE file ADD COLUMN full_stem TEXT') + session.execute('ALTER TABLE file ADD COLUMN stem TEXT') + session.execute('CREATE INDEX file_directory_idx ON file(directory)') + session.execute('CREATE INDEX file_full_stem_idx ON file(full_stem)') + session.execute('CREATE INDEX file_stem_idx ON file(stem)') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('DROP INDEX IF EXISTS file_directory_idx') + session.execute('DROP INDEX IF EXISTS file_full_stem_idx') + session.execute('DROP INDEX IF EXISTS file_stem_idx') + session.execute('ALTER TABLE file DROP COLUMN IF EXISTS directory') + session.execute('ALTER TABLE file DROP COLUMN IF EXISTS full_stem') + session.execute('ALTER TABLE file DROP COLUMN IF EXISTS stem') diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py index 13cca563..c4ce9285 100644 --- a/modules/videos/__init__.py +++ b/modules/videos/__init__.py @@ -32,9 +32,10 @@ def video_modeler(groups: Dict[str, List[File]], session: Session): session.flush(group) poster_file = next((i for i in group if i.mimetype.split('/')[0] == 'image'), None) - caption_file = next((i for i in group if i.path.name.endswith('.en.vtt') or i.path.name.endswith('.en.srt')), - None) info_json_file = next((i for i in group if i.path.name.endswith('.info.json')), None) + # Prefer WebVTT over SRT. (SRT cannot be displayed for HTML video). + caption_file = next((i for i in group if i.path.name.endswith('.en.vtt')), None) + caption_file = caption_file or next((i for i in group if i.path.name.endswith('.en.srt')), None) if poster_file: poster_file.associated = True @@ -57,7 +58,7 @@ def video_modeler(groups: Dict[str, List[File]], session: Session): caption_file != video.caption_file or \ info_json_file != video.info_json_file or \ video.size != size: - # Files were changed. Re-index. + # Files might have been changed. Re-index. video.video_file.indexed = False video.poster_file = poster_file diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 18fe6e40..f5a3285c 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -290,6 +290,8 @@ class VideoDownloader(Downloader, ABC): '-o', file_name_format, '--no-cache-dir', '--compat-options', 'no-live-chat', + # Use experimental feature to merge files. + '--ppa', 'Merger+ffmpeg_o1:-strict -2', url, ) return_code, logs = await self.process_runner(url, cmd, out_dir) diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 3e080f3b..514b6e43 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -3,16 +3,17 @@ import html import pathlib import re from collections import defaultdict -from typing import Tuple, Optional, Union +from typing import Tuple, Optional, Union, Generator from sqlalchemy import or_ from sqlalchemy.orm import Session from yt_dlp import YoutubeDL from wrolpi import before_startup -from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, after_refresh, register_after_refresh +from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, register_after_refresh from wrolpi.dates import from_timestamp, Seconds from wrolpi.db import get_db_curs, get_db_session, optional_session +from wrolpi.errors import UnknownDirectory from wrolpi.files.models import File from wrolpi.vars import PYTEST from .common import is_valid_poster, convert_image, \ @@ -27,10 +28,11 @@ DEFAULT_DOWNLOAD_FREQUENCY = Seconds.week class VideoInfoJSON(object): channel_source_id = None - title = None duration = None - view_count = None + epoch = None + title = None url = None + view_count = None def process_video_info_json(video: Video) -> VideoInfoJSON: @@ -42,10 +44,18 @@ def process_video_info_json(video: Video) -> VideoInfoJSON: title = info_json.get('fulltitle') or info_json.get('title') video_info_json.title = html.unescape(title) if title else None - video_info_json.duration = info_json.get('duration') - video_info_json.view_count = info_json.get('view_count') - video_info_json.url = info_json.get('webpage_url') or info_json.get('url') - video_info_json.channel_source_id = info_json.get('channel_id') + video_info_json.duration = info_json.get('duration') or None + video_info_json.view_count = info_json.get('view_count') or None + video_info_json.url = info_json.get('webpage_url') or info_json.get('url') or None + video_info_json.channel_source_id = info_json.get('channel_id') or None + + # Get `epoch` from top of JSON. Convert to datetime. + if epoch := info_json.get('epoch'): + if '.' in str(epoch): + epoch = float(epoch) + else: + epoch = int(epoch) + video_info_json.epoch = from_timestamp(epoch) if epoch else None return video_info_json @@ -71,12 +81,16 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session found, it will be generated from the video file. """ info_json_path = video.info_json_file.path if video.info_json_file else video.info_json_path - if info_json_path and (not video.title or not video.duration or not video.view_count or not video.url): + json_data_missing = bool(video.title) and bool(video.duration) and bool(video.view_count) and bool(video.url) \ + and bool(video.upload_date) + if info_json_path and json_data_missing is False: # These properties can be found in the info json. video_info_json = process_video_info_json(video) video.title = video_info_json.title video.duration = video_info_json.duration video.url = video_info_json.url + # Epoch is more precise than upload_date. + video.upload_date = video_info_json.epoch # View count will probably be overwritten by more recent data when this Video's Channel is # updated. video.view_count = video.view_count or video_info_json.view_count @@ -91,6 +105,7 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session # These are the least trusted, so anything already on the video should be trusted. _, upload_date, source_id, title = parse_video_file_name(video_path) video.title = video.title or html.unescape(title) + # Prefer epoch over upload_date from file. video.upload_date = video.upload_date or upload_date video.source_id = video.source_id or source_id @@ -572,3 +587,34 @@ def parse_video_file_name(video_path: pathlib.Path) -> \ # Return the stem as a last resort title = pathlib.Path(video_path).stem.strip() return None, None, None, title + + +def find_orphaned_video_files(directory: pathlib.Path) -> Generator[pathlib.Path, None, None]: + """Finds all files which should be associated with a video file, but a video file does not match their stem. + + Example: + video1.mp4, video1.info.json # Video file with info json. + video2.info.json # Orphaned video json. + """ + with get_db_curs() as curs: + if not directory.is_dir(): + raise UnknownDirectory() + + directory = str(directory).rstrip('/') + + curs.execute(f''' + SELECT f1.path + FROM ( + -- Get array of mimetypes for each full_stem group. + select f2.full_stem, array_agg(f2.mimetype) as mimetypes + from file f2 + where f2.full_stem like '{directory}/%' + group by 1) AS stem_groups + -- Get all files that match the full_stem. + LEFT JOIN file f1 ON stem_groups.full_stem = f1.full_stem + -- Get all groups which DO NOT contain one of the video mimetypes. + WHERE NOT (select array_agg(f3.mimetype) from file f3 where f3.mimetype like 'video/%') && stem_groups.mimetypes + ''') + results = map(lambda i: pathlib.Path(i[0]), curs.fetchall()) + results = filter(lambda i: i.is_file(), results) + yield from results diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py index 03f05c8e..53e0546c 100644 --- a/modules/videos/test/test_api.py +++ b/modules/videos/test/test_api.py @@ -19,7 +19,6 @@ async def test_refresh_videos_index(test_session, test_directory, video_factory) video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True) test_session.commit() - test_session.query(Video).delete() test_session.query(File).delete() test_session.commit() diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index c0f6b4d0..7439534f 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -1,14 +1,21 @@ +import json import pathlib +import shutil import pytest import sqlalchemy +from modules.videos import lib from modules.videos.common import apply_info_json from modules.videos.lib import parse_video_file_name, validate_video, get_statistics from modules.videos.models import Video from modules.videos.video.lib import search_videos +from wrolpi.dates import strptime from wrolpi.files import lib as files_lib +from wrolpi.files.lib import refresh_files from wrolpi.files.models import File +from wrolpi.test.common import skip_circleci +from wrolpi.vars import PROJECT_DIR def test_search_censored_videos(test_session, simple_channel, video_factory): @@ -188,3 +195,80 @@ async def test_get_statistics(test_session, video_factory, channel_factory): assert 'videos' in result['statistics'] assert 'channels' in result['statistics'] assert 'historical' in result['statistics'] + + +@skip_circleci +@pytest.mark.asyncio +async def test_video_epoch(test_session, test_directory, video_file): + """The epoch in a Video's info_json is more precise and should be used when available. + + Fallback to the upload_date in the Video file's name. + """ + # The date from a video file's name is assumed to be midnight in UTC. + upload_date = '20221206' + + destination = test_directory / f'test channel_{upload_date}_the title.mp4' + video_file.rename(destination) + video_file = destination + + await refresh_files() + assert test_session.query(File).count() == 1 + assert test_session.query(Video).count() == 1 + + video: Video = test_session.query(Video).one() + # Upload date is converted to local time. + assert video.title == 'the title' + assert video.upload_date == strptime('2022-12-05 17:00:00'), \ + 'Video upload_date should be the date from the file name.' + assert not video.info_json_file, 'Video should not have info json file' + + # Write info_json with more precise epoch. + info_json_path = video_file.with_suffix('.info.json') + info_json = {'epoch': 1656783193} + info_json_path.write_text(json.dumps(info_json)) + # Invalidate video so the JSON will be processed again. + video.validated = False + test_session.commit() + + # Epoch should replace file upload_date. + await refresh_files() + assert test_session.query(File).count() == 2 + assert video.info_json_file, 'Video info json was not discovered.' + assert video.upload_date == strptime('2022-07-02 11:33:13'), 'Epoch from info_json did not replace file upload_date' + assert video_file.is_file() and info_json_path.is_file(), 'Files should contain the old date.' + + +@pytest.mark.asyncio +async def test_orphaned_files(test_session, make_files_structure, test_directory, video_factory): + # A Video without associated files is not orphaned. + vid1 = video_factory(with_video_file=True) + # The video files will be removed... + vid2 = video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpeg', with_info_json=True) + vid3 = video_factory(with_video_file=True, with_poster_ext='jpg') + # This will be ignored because it is not in the "videos" subdirectory. + shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', test_directory / 'video4.en.vtt') + test_session.commit() + + # Remove vid2 video. Caption, poster, info_json are now orphaned. + vid2.video_path.unlink() + vid2_caption_path, vid2_poster_path, vid2_info_json_path = vid2.caption_path, vid2.poster_path, vid2.info_json_path + # Remove vid3 video. Poster is now orphaned. + vid3.video_path.unlink() + vid3_poster_path = vid3.poster_path + await refresh_files() + test_session.commit() + + # 6 files when two video files are deleted. + # (vid1, vid2[caption,poster,json], vid3 poster, vid4) + assert test_session.query(File).count() == 6 + assert test_session.query(Video).count() == 1 + + videos_directory = test_directory / 'videos' + results = lib.find_orphaned_video_files(videos_directory) + + assert sorted(list(results)) == sorted([ + vid2_caption_path, + vid2_info_json_path, + vid2_poster_path, + vid3_poster_path, + ]) diff --git a/scripts/find_and_rename_orphan_video_files.py b/scripts/find_and_rename_orphan_video_files.py new file mode 100755 index 00000000..d1000dc0 --- /dev/null +++ b/scripts/find_and_rename_orphan_video_files.py @@ -0,0 +1,131 @@ +#! /usr/bin/env python3 +import argparse +import os +import pathlib +import sys +from typing import List, Dict + +sys.path.append(os.getcwd()) + +from modules.videos import lib as videos_lib +from wrolpi.common import get_media_directory +from wrolpi.files.lib import get_mimetype, split_path_stem_and_suffix + + +def group_orphaned_files(orphaned_paths: List[pathlib.Path]) -> Dict[str, List[pathlib.Path]]: + """Group files by the `source_id` in their filename.""" + groups = dict() + for orphan in orphaned_paths: + channel, upload_date, source_id, name = videos_lib.parse_video_file_name(orphan) + if source_id: + try: + groups[source_id].append(orphan) + except KeyError: + groups[source_id] = [orphan, ] + return groups + + +def find_videos(directory: pathlib.Path, source_id: str): + """Find all video files that match the `source_id`.""" + paths = directory.glob(f'*{source_id}*') + videos = list(filter(lambda i: get_mimetype(i).startswith('video/'), paths)) + return videos + + +def main(videos_directory: pathlib.Path, move: bool = False, delete_existing: bool = False, + delete_orphan: bool = False): + # Get a list of orphaned files from the DB. (A file refresh is required for this to work). + orphaned_files = videos_lib.find_orphaned_video_files(videos_directory) + orphaned_files = list(orphaned_files) + + # Protect video files absolutely. + if any(get_mimetype(i).startswith('video/') for i in orphaned_files): + print(f'Refusing to process video files!', file=sys.stderr) + sys.exit(1) + + # Group files by the `source_id` in their filename. + orphaned_file_groups = group_orphaned_files(orphaned_files) + + for source_id, group in orphaned_file_groups.items(): + first = group[0] + parent = first.parent + stem, _ = split_path_stem_and_suffix(first) + + video_files = find_videos(parent, source_id) + video_stems = [split_path_stem_and_suffix(i)[0] for i in video_files] + + if not video_stems: + if not delete_orphan: + for orphan in group: + print(f'Could not find video file with matching source_id ({source_id}) for {orphan}', + file=sys.stderr) + else: + for orphan in group: + print(f'Deleting: {orphan}') + orphan.unlink() + continue + + if len(set(video_stems)) > 1: + group = list(map(str, group)) + print(f'Multiple videos match the source_id ({source_id}). Do not know which to rename to.', + file=sys.stderr) + print(f'Files: {group}', file=sys.stderr) + continue + + video_stem = video_stems[0] + for orphan in group: + orphan_suffix = split_path_stem_and_suffix(orphan.name)[1].lstrip('.') + if not orphan_suffix: + print(f'Unable to parse file name {orphan}', file=sys.stderr) + continue + + destination = pathlib.Path(f'{parent / video_stem}.{orphan_suffix}') + + try: + if destination.is_file(): + if delete_existing: + print(f'Deleting: {orphan}') + orphan.unlink() + else: + print(f'Destination already exists! {orphan}', file=sys.stderr) + continue + + print(f'{repr(str(orphan))} => {repr(str(destination))}') + + if move: + orphan.rename(destination) + + if orphan.is_file() or not destination.is_file(): + # Something is very wrong, quit. + print(f'Failed to rename orphan! {orphan}', file=sys.stderr) + sys.exit(2) + except OSError as e: + if 'File name too long' in str(e): + print(f'Failed to rename orphan, name is too long! {destination}') + continue + raise + + +if __name__ == '__main__': + media_directory = get_media_directory() + + parser = argparse.ArgumentParser() + parser.add_argument('--directory', default=media_directory / 'videos', type=pathlib.Path) + parser.add_argument('-m', '--move', default=False, action='store_true', + help='Rename files when a match is found.') + parser.add_argument('-d', '--delete-existing', default=False, action='store_true', + help='Delete files if their matching video file already has that file.') + parser.add_argument('-o', '--delete-orphan', default=False, action='store_true', + help='Delete files which have no matching video file.') + args = parser.parse_args() + + if not args.directory.is_dir(): + print('Invalid directory', file=sys.stderr) + sys.exit(1) + + main( + args.directory, + args.move, + args.delete_existing, + args.delete_orphan, + ) diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index e1764bd3..ea93d2b9 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -22,14 +22,17 @@ class File(ModelHelper, Base): path: pathlib.Path = Column(MediaPathType, primary_key=True) associated = Column(Boolean, default=False) + directory: pathlib.Path = Column(MediaPathType) # aka the parent. + full_stem = Column(String) # f'{directory}/{stem}' idempotency = Column(TZDateTime, default=lambda: now()) indexed = Column(Boolean, default=False) - mimetype = Column(String) - model = Column(String) + mimetype = Column(String) # Gotten by python-magic or file command. + model = Column(String) # Filled out by the modeler. Will be the table name (video, archive, etc.). modification_datetime = Column(TZDateTime) size = Column(Integer) - suffix = Column(String) - title = Column(String) + stem = Column(String) # The file name, but without the suffix (aka extension). See: split_path_stem_and_suffix + suffix = Column(String) # aka the extension. + title = Column(String) # Provided by the modeler, or it will be the entire file name (with suffix). a_text = deferred(Column(String)) b_text = deferred(Column(String)) @@ -53,12 +56,15 @@ class File(ModelHelper, Base): path = self.path.relative_to(get_media_directory()) d = dict( associated=self.associated, + directory=self.directory, + full_stem=self.full_stem, key=path, # React browser expects this. mimetype=self.mimetype, model=self.model, modified=self.modification_datetime, # React browser expects this. path=path, size=self.size, + stem=self.stem, suffix=self.suffix, title=self.title, ) @@ -122,11 +128,13 @@ class File(ModelHelper, Base): if not self.mimetype: from wrolpi.files.lib import get_mimetype self.mimetype = get_mimetype(self.path) + self.stem, self.suffix = split_path_stem_and_suffix(self.path) + self.directory = self.path.parent + self.full_stem = f'{self.directory}/{self.stem}' self.title = self.title or self.path.name stat = self.path.stat() self.size = stat.st_size self.modification_datetime = from_timestamp(stat.st_mtime) - name, self.suffix = split_path_stem_and_suffix(self.path) changed = old_mimetype != self.mimetype \ or old_size != self.size \ @@ -135,7 +143,7 @@ class File(ModelHelper, Base): if changed: self.indexed = False # Use title as the a_text temporarily, this may be overwritten by indexer. - self.a_text = self.a_text or name + self.a_text = self.a_text or self.stem return changed @validates('path') diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 3d99aea0..f9f0dce1 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -275,7 +275,7 @@ async def test_mime_type(test_session, make_files_structure, test_directory): @pytest.mark.asyncio -async def test_files_indexer(test_session, make_files_structure): +async def test_files_indexer(test_session, make_files_structure, test_directory): """An Indexer is provided for each file based on it's mimetype or contents.""" source_files: List[str] = [ 'a bzip file.bzip', @@ -318,15 +318,18 @@ async def test_files_indexer(test_session, make_files_structure): assert info_json_file.path.suffix == '.json' and info_json_file.indexer == indexers.DefaultIndexer assert video_file.path.suffix == '.mp4' and video_file.indexer == videos.VideoIndexer - # Path suffix is copied to File.suffix. - assert bzip_file.suffix == '.bzip' - assert gzip_file.suffix == '.gzip' - assert text_file.suffix == '.txt' - assert zip_file.suffix == '.zip' - assert image_file.suffix == '.jpeg' - assert unknown_file.suffix == '' - assert info_json_file.suffix == '.info.json' - assert video_file.suffix == '.mp4' + # Suffix, directory and name are correct. + def assert_file_properties(file: File, suffix, directory, stem): + assert file.suffix == suffix and file.directory == directory and file.stem == stem + + assert_file_properties(bzip_file, '.bzip', test_directory, 'a bzip file') + assert_file_properties(gzip_file, '.gzip', test_directory, 'a gzip file') + assert_file_properties(text_file, '.txt', test_directory, 'a text file') + assert_file_properties(zip_file, '.zip', test_directory, 'a zip file') + assert_file_properties(image_file, '.jpeg', test_directory / 'images', 'an image file') + assert_file_properties(unknown_file, '', test_directory, 'unknown file') + assert_file_properties(info_json_file, '.info.json', test_directory / 'videos', 'a video file') + assert_file_properties(video_file, '.mp4', test_directory / 'videos', 'a video file') # File are indexed by their titles and contents. files, total = lib.search_files('file', 10, 0) diff --git a/wrolpi/version.py b/wrolpi/version.py index 41f80ac3..162a046a 100644 --- a/wrolpi/version.py +++ b/wrolpi/version.py @@ -1,4 +1,4 @@ -__version__ = '0.7.5-beta' +__version__ = '0.7.6-beta' import subprocess