mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
d0d69519cd
Skipping video epoch test on circleci. Created script which finds and cleans up (renames/deletes) orphaned video meta files. Reworking orphaned video query, some video files were leaking through. Modifying File table to allow finding of orphaned video files. Extracting epoch from video info_json for more accurate upload_date. Extracting epoch from video info_json for more accurate upload_date.
145 lines
5 KiB
Python
145 lines
5 KiB
Python
import json
|
|
import pathlib
|
|
from abc import ABC
|
|
from typing import List, Dict
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modules.videos.models import Video, Channel
|
|
from wrolpi.captions import read_captions, extract_captions
|
|
from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent
|
|
from wrolpi.db import get_db_curs
|
|
from wrolpi.files.indexers import Indexer, register_indexer
|
|
from wrolpi.files.lib import split_path_stem_and_suffix
|
|
from wrolpi.files.models import File
|
|
from wrolpi.vars import PYTEST
|
|
from .downloader import video_downloader # Import downloaders so they are registered.
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
__all__ = ['video_modeler', 'VideoIndexer']
|
|
|
|
|
|
@register_modeler
|
|
def video_modeler(groups: Dict[str, List[File]], session: Session):
|
|
local_groups = groups.copy()
|
|
videos = []
|
|
for stem, group in local_groups.items():
|
|
video_file = next((i for i in group if i.mimetype.startswith('video/') and i.path.suffix != '.part'), None)
|
|
if not video_file:
|
|
# Not a video group.
|
|
continue
|
|
|
|
session.flush(group)
|
|
poster_file = next((i for i in group if i.mimetype.split('/')[0] == 'image'), None)
|
|
info_json_file = next((i for i in group if i.path.name.endswith('.info.json')), None)
|
|
# Prefer WebVTT over SRT. (SRT cannot be displayed for HTML video).
|
|
caption_file = next((i for i in group if i.path.name.endswith('.en.vtt')), None)
|
|
caption_file = caption_file or next((i for i in group if i.path.name.endswith('.en.srt')), None)
|
|
|
|
if poster_file:
|
|
poster_file.associated = True
|
|
poster_file.do_stats()
|
|
if caption_file:
|
|
caption_file.associated = True
|
|
caption_file.do_stats()
|
|
if info_json_file:
|
|
info_json_file.associated = True
|
|
info_json_file.do_stats()
|
|
|
|
video = session.query(Video).filter_by(video_file=video_file).one_or_none()
|
|
if not video:
|
|
video = Video(video_file=video_file)
|
|
session.add(video)
|
|
|
|
size = video_file.path.stat().st_size
|
|
|
|
if poster_file != video.poster_file or \
|
|
caption_file != video.caption_file or \
|
|
info_json_file != video.info_json_file or \
|
|
video.size != size:
|
|
# Files might have been changed. Re-index.
|
|
video.video_file.indexed = False
|
|
|
|
video.poster_file = poster_file
|
|
video.caption_file = caption_file
|
|
video.info_json_file = info_json_file
|
|
video.size = size
|
|
video_file.model = Video.__tablename__
|
|
|
|
videos.append(video)
|
|
|
|
if videos:
|
|
session.flush(videos)
|
|
for video in videos:
|
|
video.video_file.do_index()
|
|
video.validate(session)
|
|
|
|
# Remove this group, it will not be processed by other modelers.
|
|
stem, _ = split_path_stem_and_suffix(video.video_path)
|
|
del groups[stem]
|
|
|
|
|
|
@register_after_refresh
|
|
@limit_concurrent(1)
|
|
def video_cleanup():
|
|
# Claim all Videos in a Channel's directory for that Channel.
|
|
logger.info('Claiming Videos for their Channels')
|
|
with get_db_curs(commit=True) as curs:
|
|
curs.execute('''
|
|
UPDATE video v
|
|
SET channel_id = c.id
|
|
FROM channel c
|
|
WHERE
|
|
v.video_path LIKE c.directory || '/%'::VARCHAR
|
|
AND v.channel_id IS NULL
|
|
''')
|
|
|
|
|
|
EXTRACT_SUBTITLES = False
|
|
|
|
|
|
@register_indexer('video')
|
|
class VideoIndexer(Indexer, ABC):
|
|
"""Handles video files like mp4/ogg."""
|
|
|
|
@staticmethod
|
|
def get_description(file):
|
|
video_path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path
|
|
info_json_path = video_path.with_suffix('.info.json')
|
|
if info_json_path.is_file():
|
|
with info_json_path.open('rt') as fh:
|
|
try:
|
|
return json.load(fh)['description']
|
|
except Exception:
|
|
if not PYTEST:
|
|
logger.warning(f'Video info json file exists, but cannot get description. {file}')
|
|
return None
|
|
|
|
@classmethod
|
|
def create_index(cls, file):
|
|
"""
|
|
Index the video file and it's associated files.
|
|
|
|
a = title.
|
|
b = <empty>
|
|
c = description from info json.
|
|
d = captions from info json, or extracted from video file.
|
|
"""
|
|
a = cls.get_title(file)
|
|
c = cls.get_description(file)
|
|
|
|
# Detect the associated caption files.
|
|
d = None
|
|
if (en_vtt := file.path.with_suffix('.en.vtt')).is_file():
|
|
d = read_captions(en_vtt)
|
|
elif (vtt := file.path.with_suffix('.vtt')).is_file():
|
|
d = read_captions(vtt)
|
|
elif (en_srt := file.path.with_suffix('.en.srt')).is_file():
|
|
d = read_captions(en_srt)
|
|
elif (srt := file.path.with_suffix('.srt')).is_file():
|
|
d = read_captions(srt)
|
|
elif EXTRACT_SUBTITLES:
|
|
d = extract_captions(file.path) or ''
|
|
|
|
return a, None, c, d
|