rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-25 18:40:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Bumping version for new feature.

Browse source 
Skipping video epoch test on circleci.

Created script which finds and cleans up (renames/deletes) orphaned video meta files.

Reworking orphaned video query, some video files were leaking through.

Modifying File table to allow finding of orphaned video files.

Extracting epoch from video info_json for more accurate upload_date.

Extracting epoch from video info_json for more accurate upload_date.
This commit is contained in:
Roland 2022-12-06 14:28:35 -07:00
parent b216a9c678
commit d0d69519cd
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
10 changed files with 347 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

43
alembic/versions/6489cd50c889_file_stem_and_directory.py Normal file
View file

@ -0,0 +1,43 @@
"""File stem and directory
Revision ID: 6489cd50c889
Revises: 13542a230c78
Create Date: 2022-12-04 15:39:55.485355
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '6489cd50c889'
down_revision = '13542a230c78'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE file ADD COLUMN directory TEXT')
session.execute('ALTER TABLE file ADD COLUMN full_stem TEXT')
session.execute('ALTER TABLE file ADD COLUMN stem TEXT')
session.execute('CREATE INDEX file_directory_idx ON file(directory)')
session.execute('CREATE INDEX file_full_stem_idx ON file(full_stem)')
session.execute('CREATE INDEX file_stem_idx ON file(stem)')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('DROP INDEX IF EXISTS file_directory_idx')
session.execute('DROP INDEX IF EXISTS file_full_stem_idx')
session.execute('DROP INDEX IF EXISTS file_stem_idx')
session.execute('ALTER TABLE file DROP COLUMN IF EXISTS directory')
session.execute('ALTER TABLE file DROP COLUMN IF EXISTS full_stem')
session.execute('ALTER TABLE file DROP COLUMN IF EXISTS stem')

7
modules/videos/__init__.py
View file

@ -32,9 +32,10 @@ def video_modeler(groups: Dict[str, List[File]], session: Session):
session.flush(group)
poster_file = next((i for i in group if i.mimetype.split('/')[0] == 'image'), None)
caption_file = next((i for i in group if i.path.name.endswith('.en.vtt') or i.path.name.endswith('.en.srt')),
None)
info_json_file = next((i for i in group if i.path.name.endswith('.info.json')), None)
# Prefer WebVTT over SRT. (SRT cannot be displayed for HTML video).
caption_file = next((i for i in group if i.path.name.endswith('.en.vtt')), None)
caption_file = caption_file or next((i for i in group if i.path.name.endswith('.en.srt')), None)
if poster_file:
poster_file.associated = True
@ -57,7 +58,7 @@ def video_modeler(groups: Dict[str, List[File]], session: Session):
caption_file != video.caption_file or \
info_json_file != video.info_json_file or \
video.size != size:
# Files were changed. Re-index.
# Files might have been changed. Re-index.
video.video_file.indexed = False
video.poster_file = poster_file

2
modules/videos/downloader.py
View file

@ -290,6 +290,8 @@ class VideoDownloader(Downloader, ABC):
'-o', file_name_format,
'--no-cache-dir',
'--compat-options', 'no-live-chat',
# Use experimental feature to merge files.
'--ppa', 'Merger+ffmpeg_o1:-strict -2',
url,
)
return_code, logs = await self.process_runner(url, cmd, out_dir)

64
modules/videos/lib.py
View file

@ -3,16 +3,17 @@ import html
import pathlib
import re
from collections import defaultdict
from typing import Tuple, Optional, Union
from typing import Tuple, Optional, Union, Generator
from sqlalchemy import or_
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
from wrolpi import before_startup
from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, after_refresh, register_after_refresh
from wrolpi.common import ConfigFile, get_media_directory, sanitize_link, register_after_refresh
from wrolpi.dates import from_timestamp, Seconds
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.errors import UnknownDirectory
from wrolpi.files.models import File
from wrolpi.vars import PYTEST
from .common import is_valid_poster, convert_image, \
@ -27,10 +28,11 @@ DEFAULT_DOWNLOAD_FREQUENCY = Seconds.week
class VideoInfoJSON(object):
channel_source_id = None
title = None
duration = None
view_count = None
epoch = None
title = None
url = None
view_count = None
def process_video_info_json(video: Video) -> VideoInfoJSON:
@ -42,10 +44,18 @@ def process_video_info_json(video: Video) -> VideoInfoJSON:
title = info_json.get('fulltitle') or info_json.get('title')
video_info_json.title = html.unescape(title) if title else None
video_info_json.duration = info_json.get('duration')
video_info_json.view_count = info_json.get('view_count')
video_info_json.url = info_json.get('webpage_url') or info_json.get('url')
video_info_json.channel_source_id = info_json.get('channel_id')
video_info_json.duration = info_json.get('duration') or None
video_info_json.view_count = info_json.get('view_count') or None
video_info_json.url = info_json.get('webpage_url') or info_json.get('url') or None
video_info_json.channel_source_id = info_json.get('channel_id') or None
# Get `epoch` from top of JSON. Convert to datetime.
if epoch := info_json.get('epoch'):
if '.' in str(epoch):
epoch = float(epoch)
else:
epoch = int(epoch)
video_info_json.epoch = from_timestamp(epoch) if epoch else None
return video_info_json
@ -71,12 +81,16 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
found, it will be generated from the video file.
"""
info_json_path = video.info_json_file.path if video.info_json_file else video.info_json_path
if info_json_path and (not video.title or not video.duration or not video.view_count or not video.url):
json_data_missing = bool(video.title) and bool(video.duration) and bool(video.view_count) and bool(video.url) \
and bool(video.upload_date)
if info_json_path and json_data_missing is False:
# These properties can be found in the info json.
video_info_json = process_video_info_json(video)
video.title = video_info_json.title
video.duration = video_info_json.duration
video.url = video_info_json.url
# Epoch is more precise than upload_date.
video.upload_date = video_info_json.epoch
# View count will probably be overwritten by more recent data when this Video's Channel is
# updated.
video.view_count = video.view_count or video_info_json.view_count
@ -91,6 +105,7 @@ def validate_video(video: Video, channel_generate_poster: bool, session: Session
# These are the least trusted, so anything already on the video should be trusted.
_, upload_date, source_id, title = parse_video_file_name(video_path)
video.title = video.title or html.unescape(title)
# Prefer epoch over upload_date from file.
video.upload_date = video.upload_date or upload_date
video.source_id = video.source_id or source_id
@ -572,3 +587,34 @@ def parse_video_file_name(video_path: pathlib.Path) -> \
# Return the stem as a last resort
title = pathlib.Path(video_path).stem.strip()
return None, None, None, title
def find_orphaned_video_files(directory: pathlib.Path) -> Generator[pathlib.Path, None, None]:
"""Finds all files which should be associated with a video file, but a video file does not match their stem.
Example:
video1.mp4, video1.info.json # Video file with info json.
video2.info.json # Orphaned video json.
"""
with get_db_curs() as curs:
if not directory.is_dir():
raise UnknownDirectory()
directory = str(directory).rstrip('/')
curs.execute(f'''
SELECT f1.path
FROM (
-- Get array of mimetypes for each full_stem group.
select f2.full_stem, array_agg(f2.mimetype) as mimetypes
from file f2
where f2.full_stem like '{directory}/%'
group by 1) AS stem_groups
-- Get all files that match the full_stem.
LEFT JOIN file f1 ON stem_groups.full_stem = f1.full_stem
-- Get all groups which DO NOT contain one of the video mimetypes.
WHERE NOT (select array_agg(f3.mimetype) from file f3 where f3.mimetype like 'video/%') && stem_groups.mimetypes
''')
results = map(lambda i: pathlib.Path(i[0]), curs.fetchall())
results = filter(lambda i: i.is_file(), results)
yield from results

1
modules/videos/test/test_api.py
View file

@ -19,7 +19,6 @@ async def test_refresh_videos_index(test_session, test_directory, video_factory)
video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True)
test_session.commit()
test_session.query(Video).delete()
test_session.query(File).delete()
test_session.commit()

84
modules/videos/test/test_lib.py
View file

@ -1,14 +1,21 @@
import json
import pathlib
import shutil
import pytest
import sqlalchemy
from modules.videos import lib
from modules.videos.common import apply_info_json
from modules.videos.lib import parse_video_file_name, validate_video, get_statistics
from modules.videos.models import Video
from modules.videos.video.lib import search_videos
from wrolpi.dates import strptime
from wrolpi.files import lib as files_lib
from wrolpi.files.lib import refresh_files
from wrolpi.files.models import File
from wrolpi.test.common import skip_circleci
from wrolpi.vars import PROJECT_DIR
def test_search_censored_videos(test_session, simple_channel, video_factory):
@ -188,3 +195,80 @@ async def test_get_statistics(test_session, video_factory, channel_factory):
assert 'videos' in result['statistics']
assert 'channels' in result['statistics']
assert 'historical' in result['statistics']
@skip_circleci
@pytest.mark.asyncio
async def test_video_epoch(test_session, test_directory, video_file):
"""The epoch in a Video's info_json is more precise and should be used when available.
Fallback to the upload_date in the Video file's name.
"""
# The date from a video file's name is assumed to be midnight in UTC.
upload_date = '20221206'
destination = test_directory / f'test channel_{upload_date}_the title.mp4'
video_file.rename(destination)
video_file = destination
await refresh_files()
assert test_session.query(File).count() == 1
assert test_session.query(Video).count() == 1
video: Video = test_session.query(Video).one()
# Upload date is converted to local time.
assert video.title == 'the title'
assert video.upload_date == strptime('2022-12-05 17:00:00'), \
'Video upload_date should be the date from the file name.'
assert not video.info_json_file, 'Video should not have info json file'
# Write info_json with more precise epoch.
info_json_path = video_file.with_suffix('.info.json')
info_json = {'epoch': 1656783193}
info_json_path.write_text(json.dumps(info_json))
# Invalidate video so the JSON will be processed again.
video.validated = False
test_session.commit()
# Epoch should replace file upload_date.
await refresh_files()
assert test_session.query(File).count() == 2
assert video.info_json_file, 'Video info json was not discovered.'
assert video.upload_date == strptime('2022-07-02 11:33:13'), 'Epoch from info_json did not replace file upload_date'
assert video_file.is_file() and info_json_path.is_file(), 'Files should contain the old date.'
@pytest.mark.asyncio
async def test_orphaned_files(test_session, make_files_structure, test_directory, video_factory):
# A Video without associated files is not orphaned.
vid1 = video_factory(with_video_file=True)
# The video files will be removed...
vid2 = video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpeg', with_info_json=True)
vid3 = video_factory(with_video_file=True, with_poster_ext='jpg')
# This will be ignored because it is not in the "videos" subdirectory.
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', test_directory / 'video4.en.vtt')
test_session.commit()
# Remove vid2 video. Caption, poster, info_json are now orphaned.
vid2.video_path.unlink()
vid2_caption_path, vid2_poster_path, vid2_info_json_path = vid2.caption_path, vid2.poster_path, vid2.info_json_path
# Remove vid3 video. Poster is now orphaned.
vid3.video_path.unlink()
vid3_poster_path = vid3.poster_path
await refresh_files()
test_session.commit()
# 6 files when two video files are deleted.
# (vid1, vid2[caption,poster,json], vid3 poster, vid4)
assert test_session.query(File).count() == 6
assert test_session.query(Video).count() == 1
videos_directory = test_directory / 'videos'
results = lib.find_orphaned_video_files(videos_directory)
assert sorted(list(results)) == sorted([
vid2_caption_path,
vid2_info_json_path,
vid2_poster_path,
vid3_poster_path,
])

131
scripts/find_and_rename_orphan_video_files.py Executable file
View file

@ -0,0 +1,131 @@
#! /usr/bin/env python3
import argparse
import os
import pathlib
import sys
from typing import List, Dict
sys.path.append(os.getcwd())
from modules.videos import lib as videos_lib
from wrolpi.common import get_media_directory
from wrolpi.files.lib import get_mimetype, split_path_stem_and_suffix
def group_orphaned_files(orphaned_paths: List[pathlib.Path]) -> Dict[str, List[pathlib.Path]]:
"""Group files by the `source_id` in their filename."""
groups = dict()
for orphan in orphaned_paths:
channel, upload_date, source_id, name = videos_lib.parse_video_file_name(orphan)
if source_id:
try:
groups[source_id].append(orphan)
except KeyError:
groups[source_id] = [orphan, ]
return groups
def find_videos(directory: pathlib.Path, source_id: str):
"""Find all video files that match the `source_id`."""
paths = directory.glob(f'*{source_id}*')
videos = list(filter(lambda i: get_mimetype(i).startswith('video/'), paths))
return videos
def main(videos_directory: pathlib.Path, move: bool = False, delete_existing: bool = False,
delete_orphan: bool = False):
# Get a list of orphaned files from the DB. (A file refresh is required for this to work).
orphaned_files = videos_lib.find_orphaned_video_files(videos_directory)
orphaned_files = list(orphaned_files)
# Protect video files absolutely.
if any(get_mimetype(i).startswith('video/') for i in orphaned_files):
print(f'Refusing to process video files!', file=sys.stderr)
sys.exit(1)
# Group files by the `source_id` in their filename.
orphaned_file_groups = group_orphaned_files(orphaned_files)
for source_id, group in orphaned_file_groups.items():
first = group[0]
parent = first.parent
stem, _ = split_path_stem_and_suffix(first)
video_files = find_videos(parent, source_id)
video_stems = [split_path_stem_and_suffix(i)[0] for i in video_files]
if not video_stems:
if not delete_orphan:
for orphan in group:
print(f'Could not find video file with matching source_id ({source_id}) for {orphan}',
file=sys.stderr)
else:
for orphan in group:
print(f'Deleting: {orphan}')
orphan.unlink()
continue
if len(set(video_stems)) > 1:
group = list(map(str, group))
print(f'Multiple videos match the source_id ({source_id}). Do not know which to rename to.',
file=sys.stderr)
print(f'Files: {group}', file=sys.stderr)
continue
video_stem = video_stems[0]
for orphan in group:
orphan_suffix = split_path_stem_and_suffix(orphan.name)[1].lstrip('.')
if not orphan_suffix:
print(f'Unable to parse file name {orphan}', file=sys.stderr)
continue
destination = pathlib.Path(f'{parent / video_stem}.{orphan_suffix}')
try:
if destination.is_file():
if delete_existing:
print(f'Deleting: {orphan}')
orphan.unlink()
else:
print(f'Destination already exists! {orphan}', file=sys.stderr)
continue
print(f'{repr(str(orphan))} => {repr(str(destination))}')
if move:
orphan.rename(destination)
if orphan.is_file() or not destination.is_file():
# Something is very wrong, quit.
print(f'Failed to rename orphan! {orphan}', file=sys.stderr)
sys.exit(2)
except OSError as e:
if 'File name too long' in str(e):
print(f'Failed to rename orphan, name is too long! {destination}')
continue
raise
if __name__ == '__main__':
media_directory = get_media_directory()
parser = argparse.ArgumentParser()
parser.add_argument('--directory', default=media_directory / 'videos', type=pathlib.Path)
parser.add_argument('-m', '--move', default=False, action='store_true',
help='Rename files when a match is found.')
parser.add_argument('-d', '--delete-existing', default=False, action='store_true',
help='Delete files if their matching video file already has that file.')
parser.add_argument('-o', '--delete-orphan', default=False, action='store_true',
help='Delete files which have no matching video file.')
args = parser.parse_args()
if not args.directory.is_dir():
print('Invalid directory', file=sys.stderr)
sys.exit(1)
main(
args.directory,
args.move,
args.delete_existing,
args.delete_orphan,
)

20
wrolpi/files/models.py
View file

@ -22,14 +22,17 @@ class File(ModelHelper, Base):
path: pathlib.Path = Column(MediaPathType, primary_key=True)
associated = Column(Boolean, default=False)
directory: pathlib.Path = Column(MediaPathType) # aka the parent.
full_stem = Column(String) # f'{directory}/{stem}'
idempotency = Column(TZDateTime, default=lambda: now())
indexed = Column(Boolean, default=False)
mimetype = Column(String)
model = Column(String)
mimetype = Column(String) # Gotten by python-magic or file command.
model = Column(String) # Filled out by the modeler. Will be the table name (video, archive, etc.).
modification_datetime = Column(TZDateTime)
size = Column(Integer)
suffix = Column(String)
title = Column(String)
stem = Column(String) # The file name, but without the suffix (aka extension). See: split_path_stem_and_suffix
suffix = Column(String) # aka the extension.
title = Column(String) # Provided by the modeler, or it will be the entire file name (with suffix).
a_text = deferred(Column(String))
b_text = deferred(Column(String))
@ -53,12 +56,15 @@ class File(ModelHelper, Base):
path = self.path.relative_to(get_media_directory())
d = dict(
associated=self.associated,
directory=self.directory,
full_stem=self.full_stem,
key=path, # React browser expects this.
mimetype=self.mimetype,
model=self.model,
modified=self.modification_datetime, # React browser expects this.
path=path,
size=self.size,
stem=self.stem,
suffix=self.suffix,
title=self.title,
)
@ -122,11 +128,13 @@ class File(ModelHelper, Base):
if not self.mimetype:
from wrolpi.files.lib import get_mimetype
self.mimetype = get_mimetype(self.path)
self.stem, self.suffix = split_path_stem_and_suffix(self.path)
self.directory = self.path.parent
self.full_stem = f'{self.directory}/{self.stem}'
self.title = self.title or self.path.name
stat = self.path.stat()
self.size = stat.st_size
self.modification_datetime = from_timestamp(stat.st_mtime)
name, self.suffix = split_path_stem_and_suffix(self.path)
changed = old_mimetype != self.mimetype \
or old_size != self.size \
@ -135,7 +143,7 @@ class File(ModelHelper, Base):
if changed:
self.indexed = False
# Use title as the a_text temporarily, this may be overwritten by indexer.
self.a_text = self.a_text or name
self.a_text = self.a_text or self.stem
return changed
@validates('path')

23
wrolpi/files/test/test_lib.py
View file

@ -275,7 +275,7 @@ async def test_mime_type(test_session, make_files_structure, test_directory):
@pytest.mark.asyncio
async def test_files_indexer(test_session, make_files_structure):
async def test_files_indexer(test_session, make_files_structure, test_directory):
"""An Indexer is provided for each file based on it's mimetype or contents."""
source_files: List[str] = [
'a bzip file.bzip',
@ -318,15 +318,18 @@ async def test_files_indexer(test_session, make_files_structure):
assert info_json_file.path.suffix == '.json' and info_json_file.indexer == indexers.DefaultIndexer
assert video_file.path.suffix == '.mp4' and video_file.indexer == videos.VideoIndexer
# Path suffix is copied to File.suffix.
assert bzip_file.suffix == '.bzip'
assert gzip_file.suffix == '.gzip'
assert text_file.suffix == '.txt'
assert zip_file.suffix == '.zip'
assert image_file.suffix == '.jpeg'
assert unknown_file.suffix == ''
assert info_json_file.suffix == '.info.json'
assert video_file.suffix == '.mp4'
# Suffix, directory and name are correct.
def assert_file_properties(file: File, suffix, directory, stem):
assert file.suffix == suffix and file.directory == directory and file.stem == stem
assert_file_properties(bzip_file, '.bzip', test_directory, 'a bzip file')
assert_file_properties(gzip_file, '.gzip', test_directory, 'a gzip file')
assert_file_properties(text_file, '.txt', test_directory, 'a text file')
assert_file_properties(zip_file, '.zip', test_directory, 'a zip file')
assert_file_properties(image_file, '.jpeg', test_directory / 'images', 'an image file')
assert_file_properties(unknown_file, '', test_directory, 'unknown file')
assert_file_properties(info_json_file, '.info.json', test_directory / 'videos', 'a video file')
assert_file_properties(video_file, '.mp4', test_directory / 'videos', 'a video file')
# File are indexed by their titles and contents.
files, total = lib.search_files('file', 10, 0)

2
wrolpi/version.py
View file

@ -1,4 +1,4 @@
__version__ = '0.7.5-beta'
__version__ = '0.7.6-beta'
import subprocess