Bumping version for new feature.

Skipping video epoch test on circleci.

Created script which finds and cleans up (renames/deletes) orphaned video meta files.

Reworking orphaned video query, some video files were leaking through.

Modifying File table to allow finding of orphaned video files.

Extracting epoch from video info_json for more accurate upload_date.

Extracting epoch from video info_json for more accurate upload_date.

modules/videos/__init__.py

@@ -32,9 +32,10 @@ def video_modeler(groups: Dict[str, List[File]], session: Session):
 
         session.flush(group)
         poster_file = next((i for i in group if i.mimetype.split('/')[0] == 'image'), None)
-        caption_file = next((i for i in group if i.path.name.endswith('.en.vtt') or i.path.name.endswith('.en.srt')),
-                            None)
         info_json_file = next((i for i in group if i.path.name.endswith('.info.json')), None)
+        # Prefer WebVTT over SRT.  (SRT cannot be displayed for HTML video).
+        caption_file = next((i for i in group if i.path.name.endswith('.en.vtt')), None)
+        caption_file = caption_file or next((i for i in group if i.path.name.endswith('.en.srt')), None)
 
         if poster_file:
             poster_file.associated = True
@@ -57,7 +58,7 @@ def video_modeler(groups: Dict[str, List[File]], session: Session):
                 caption_file != video.caption_file or \
                 info_json_file != video.info_json_file or \
                 video.size != size:
-            # Files were changed.  Re-index.
+            # Files might have been changed.  Re-index.
             video.video_file.indexed = False
 
         video.poster_file = poster_file

modules/videos/downloader.py

@@ -290,6 +290,8 @@ class VideoDownloader(Downloader, ABC):
                 '-o', file_name_format,
                 '--no-cache-dir',
                 '--compat-options', 'no-live-chat',
+                # Use experimental feature to merge files.
+                '--ppa', 'Merger+ffmpeg_o1:-strict -2',
                 url,
             )
             return_code, logs = await self.process_runner(url, cmd, out_dir)

modules/videos/lib.py

@@ -3,16 +3,17 @@ import html
 import pathlib
 import re
 from collections import defaultdict
-from typing import Tuple, Optional, Union
+from typing import Tuple, Optional, Union, Generator
 
 from sqlalchemy import or_
 from sqlalchemy.orm import Session
 from yt_dlp import YoutubeDL
 
 from wrolpi import before_startup
-from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, after_refresh, register_after_refresh
+from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, register_after_refresh
 from wrolpi.dates import from_timestamp, Seconds
 from wrolpi.db import get_db_curs, get_db_session, optional_session
+from wrolpi.errors import UnknownDirectory
 from wrolpi.files.models import File
 from wrolpi.vars import PYTEST
 from .common import is_valid_poster, convert_image, \
@@ -27,10 +28,11 @@ DEFAULT_DOWNLOAD_FREQUENCY = Seconds.week
 
 class VideoInfoJSON(object):
     channel_source_id = None
-    title = None
     duration = None
-    view_count = None
+    epoch = None
+    title = None
     url = None
+    view_count = None
 
 
 def process_video_info_json(video: Video) -> VideoInfoJSON:
@@ -42,10 +44,18 @@ def process_video_info_json(video: Video) -> VideoInfoJSON:
         title = info_json.get('fulltitle') or info_json.get('title')
         video_info_json.title = html.unescape(title) if title else None
 
-        video_info_json.duration = info_json.get('duration')
-        video_info_json.view_count = info_json.get('view_count')
-        video_info_json.url = info_json.get('webpage_url') or info_json.get('url')
-        video_info_json.channel_source_id = info_json.get('channel_id')
+        video_info_json.duration = info_json.get('duration') or None
+        video_info_json.view_count = info_json.get('view_count') or None
+        video_info_json.url = info_json.get('webpage_url') or info_json.get('url') or None
+        video_info_json.channel_source_id = info_json.get('channel_id') or None
+
+        # Get `epoch` from top of JSON.  Convert to datetime.
+        if epoch := info_json.get('epoch'):
+            if '.' in str(epoch):
+                epoch = float(epoch)
+            else:
+                epoch = int(epoch)
+        video_info_json.epoch = from_timestamp(epoch) if epoch else None
 
     return video_info_json
 
@@ -71,12 +81,16 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
     found, it will be generated from the video file.
     """
     info_json_path = video.info_json_file.path if video.info_json_file else video.info_json_path
-    if info_json_path and (not video.title or not video.duration or not video.view_count or not video.url):
+    json_data_missing = bool(video.title) and bool(video.duration) and bool(video.view_count) and bool(video.url) \
+                        and bool(video.upload_date)
+    if info_json_path and json_data_missing is False:
         # These properties can be found in the info json.
         video_info_json = process_video_info_json(video)
         video.title = video_info_json.title
         video.duration = video_info_json.duration
         video.url = video_info_json.url
+        # Epoch is more precise than upload_date.
+        video.upload_date = video_info_json.epoch
         # View count will probably be overwritten by more recent data when this Video's Channel is
         # updated.
         video.view_count = video.view_count or video_info_json.view_count
@@ -91,6 +105,7 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
         # These are the least trusted, so anything already on the video should be trusted.
         _, upload_date, source_id, title = parse_video_file_name(video_path)
         video.title = video.title or html.unescape(title)
+        # Prefer epoch over upload_date from file.
         video.upload_date = video.upload_date or upload_date
         video.source_id = video.source_id or source_id
 
@@ -572,3 +587,34 @@ def parse_video_file_name(video_path: pathlib.Path) -> \
     # Return the stem as a last resort
     title = pathlib.Path(video_path).stem.strip()
     return None, None, None, title
+
+
+def find_orphaned_video_files(directory: pathlib.Path) -> Generator[pathlib.Path, None, None]:
+    """Finds all files which should be associated with a video file, but a video file does not match their stem.
+
+    Example:
+        video1.mp4, video1.info.json  # Video file with info json.
+        video2.info.json  # Orphaned video json.
+    """
+    with get_db_curs() as curs:
+        if not directory.is_dir():
+            raise UnknownDirectory()
+
+        directory = str(directory).rstrip('/')
+
+        curs.execute(f'''
+        SELECT f1.path
+        FROM (
+                 -- Get array of mimetypes for each full_stem group.
+                 select f2.full_stem, array_agg(f2.mimetype) as mimetypes
+                 from file f2
+                 where f2.full_stem like '{directory}/%'
+                 group by 1) AS stem_groups
+                 -- Get all files that match the full_stem.
+                 LEFT JOIN file f1 ON stem_groups.full_stem = f1.full_stem
+        -- Get all groups which DO NOT contain one of the video mimetypes.
+        WHERE NOT (select array_agg(f3.mimetype) from file f3 where f3.mimetype like 'video/%') && stem_groups.mimetypes
+        ''')
+        results = map(lambda i: pathlib.Path(i[0]), curs.fetchall())
+        results = filter(lambda i: i.is_file(), results)
+        yield from results

modules/videos/test/test_api.py

@@ -19,7 +19,6 @@ async def test_refresh_videos_index(test_session, test_directory, video_factory)
     video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True)
     test_session.commit()
 
-    test_session.query(Video).delete()
     test_session.query(File).delete()
     test_session.commit()
 

modules/videos/test/test_lib.py

@@ -1,14 +1,21 @@
+import json
 import pathlib
+import shutil
 
 import pytest
 import sqlalchemy
 
+from modules.videos import lib
 from modules.videos.common import apply_info_json
 from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
 from modules.videos.models import Video
 from modules.videos.video.lib import search_videos
+from wrolpi.dates import strptime
 from wrolpi.files import lib as files_lib
+from wrolpi.files.lib import refresh_files
 from wrolpi.files.models import File
+from wrolpi.test.common import skip_circleci
+from wrolpi.vars import PROJECT_DIR
 
 
 def test_search_censored_videos(test_session, simple_channel, video_factory):
@@ -188,3 +195,80 @@ async def test_get_statistics(test_session, video_factory, channel_factory):
     assert 'videos' in result['statistics']
     assert 'channels' in result['statistics']
     assert 'historical' in result['statistics']
+
+
+@skip_circleci
+@pytest.mark.asyncio
+async def test_video_epoch(test_session, test_directory, video_file):
+    """The epoch in a Video's info_json is more precise and should be used when available.
+
+    Fallback to the upload_date in the Video file's name.
+    """
+    # The date from a video file's name is assumed to be midnight in UTC.
+    upload_date = '20221206'
+
+    destination = test_directory / f'test channel_{upload_date}_the title.mp4'
+    video_file.rename(destination)
+    video_file = destination
+
+    await refresh_files()
+    assert test_session.query(File).count() == 1
+    assert test_session.query(Video).count() == 1
+
+    video: Video = test_session.query(Video).one()
+    # Upload date is converted to local time.
+    assert video.title == 'the title'
+    assert video.upload_date == strptime('2022-12-05 17:00:00'), \
+        'Video upload_date should be the date from the file name.'
+    assert not video.info_json_file, 'Video should not have info json file'
+
+    # Write info_json with more precise epoch.
+    info_json_path = video_file.with_suffix('.info.json')
+    info_json = {'epoch': 1656783193}
+    info_json_path.write_text(json.dumps(info_json))
+    # Invalidate video so the JSON will be processed again.
+    video.validated = False
+    test_session.commit()
+
+    # Epoch should replace file upload_date.
+    await refresh_files()
+    assert test_session.query(File).count() == 2
+    assert video.info_json_file, 'Video info json was not discovered.'
+    assert video.upload_date == strptime('2022-07-02 11:33:13'), 'Epoch from info_json did not replace file upload_date'
+    assert video_file.is_file() and info_json_path.is_file(), 'Files should contain the old date.'
+
+
+@pytest.mark.asyncio
+async def test_orphaned_files(test_session, make_files_structure, test_directory, video_factory):
+    # A Video without associated files is not orphaned.
+    vid1 = video_factory(with_video_file=True)
+    # The video files will be removed...
+    vid2 = video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpeg', with_info_json=True)
+    vid3 = video_factory(with_video_file=True, with_poster_ext='jpg')
+    # This will be ignored because it is not in the "videos" subdirectory.
+    shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', test_directory / 'video4.en.vtt')
+    test_session.commit()
+
+    # Remove vid2 video. Caption, poster, info_json are now orphaned.
+    vid2.video_path.unlink()
+    vid2_caption_path, vid2_poster_path, vid2_info_json_path = vid2.caption_path, vid2.poster_path, vid2.info_json_path
+    # Remove vid3 video.  Poster is now orphaned.
+    vid3.video_path.unlink()
+    vid3_poster_path = vid3.poster_path
+    await refresh_files()
+    test_session.commit()
+
+    # 6 files when two video files are deleted.
+    # (vid1, vid2[caption,poster,json], vid3 poster, vid4)
+    assert test_session.query(File).count() == 6
+    assert test_session.query(Video).count() == 1
+
+    videos_directory = test_directory / 'videos'
+    results = lib.find_orphaned_video_files(videos_directory)
+
+    assert sorted(list(results)) == sorted([
+        vid2_caption_path,
+        vid2_info_json_path,
+        vid2_poster_path,
+        vid3_poster_path,
+    ])