diff --git a/alembic/versions/70db5ba8763d_files_rework.py b/alembic/versions/70db5ba8763d_files_rework.py new file mode 100644 index 00000000..d20f60c9 --- /dev/null +++ b/alembic/versions/70db5ba8763d_files_rework.py @@ -0,0 +1,216 @@ +"""Files rework. Unifying files for Videos and Archives. + +Revision ID: 70db5ba8763d +Revises: aac864072193 +Create Date: 2022-07-27 20:11:30.990396 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '70db5ba8763d' +down_revision = 'aac864072193' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + # Recreate the file table. Use file.path as the primary key. + session.execute('DROP TABLE IF EXISTS file') + session.execute('''CREATE TABLE file ( + path TEXT PRIMARY KEY, + idempotency TIMESTAMP WITH TIME ZONE, + indexed BOOLEAN DEFAULT FALSE, + mimetype TEXT, + suffix TEXT, + model TEXT, + associated BOOLEAN DEFAULT FALSE, + modification_datetime TIMESTAMP WITH TIME ZONE, + size BIGINT, + title TEXT, + a_text TEXT, + b_text TEXT, + c_text TEXT, + d_text TEXT, + textsearch tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") || + setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") || + setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") || + setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char") + ) STORED + )''') + session.execute('CREATE INDEX file_mimetype_idx ON file USING BTREE(mimetype)') + session.execute('CREATE INDEX file_textsearch_idx ON file USING GIN(textsearch)') + session.execute('CREATE INDEX file_modification_datetime_idx ON file USING BTREE(modification_datetime)') + session.execute('CREATE INDEX file_size_idx ON file USING BTREE(size)') + session.execute('CREATE INDEX file_model_idx ON file USING BTREE(model)') + session.execute('CREATE INDEX file_idempotency_idx ON file USING BTREE(idempotency)') + session.execute('CREATE INDEX file_suffix_idx ON file USING BTREE(suffix)') + + # Videos are fundamentally different, replace the video table. + session.execute('DROP TABLE IF EXISTS video') + session.execute(''' + CREATE TABLE video ( + id BIGSERIAL PRIMARY KEY, + title TEXT, + source_id TEXT, + channel_id INTEGER REFERENCES channel(id), + upload_date TIMESTAMPTZ, + duration INTEGER, + favorite TIMESTAMPTZ, + size BIGINT, + viewed TIMESTAMPTZ, + view_count INTEGER, + url TEXT, + censored BOOLEAN DEFAULT FALSE, + validated BOOLEAN DEFAULT FALSE, + modification_datetime TIMESTAMPTZ, + video_path TEXT REFERENCES file(path), + info_json_path TEXT REFERENCES file(path), + poster_path TEXT REFERENCES file(path), + caption_path TEXT REFERENCES file(path) + )''') + session.execute(''' + CREATE INDEX video_caption_path_idx ON video(caption_path); + CREATE INDEX video_info_json_path_idx ON video(info_json_path); + CREATE INDEX video_poster_path_idx ON video(poster_path); + CREATE INDEX video_video_path_idx ON video(video_path); + CREATE INDEX video_url_idx ON video(url); + ''') + + session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS idempotency') + + session.execute('DELETE FROM archive') # noqa + session.execute('DELETE FROM domains') # noqa + session.execute('DROP TABLE IF EXISTS archive') + session.execute('DROP TABLE IF EXISTS domains') + session.execute(''' + CREATE TABLE domains ( + id SERIAL PRIMARY KEY, + domain TEXT NOT NULL, + directory TEXT + )''') + session.execute(''' + CREATE TABLE archive ( + id SERIAL PRIMARY KEY, + domain_id INTEGER REFERENCES domains(id), + title TEXT, + archive_datetime timestamptz, + url TEXT, + validated BOOLEAN DEFAULT FALSE, + singlefile_path TEXT REFERENCES file(path), + readability_path TEXT REFERENCES file(path), + readability_json_path TEXT REFERENCES file(path), + readability_txt_path TEXT REFERENCES file(path), + screenshot_path TEXT REFERENCES file(path) + )''') + session.execute(''' + CREATE INDEX archive_singlefile_path ON archive(singlefile_path); + CREATE INDEX archive_readability_path ON archive(readability_path); + CREATE INDEX archive_readability_json_path ON archive(readability_json_path); + CREATE INDEX archive_readability_txt_path ON archive(readability_txt_path); + CREATE INDEX archive_screenshot_path ON archive(screenshot_path); + ''') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute(''' + DROP INDEX IF EXISTS archive_singlefile_path; + DROP INDEX IF EXISTS archive_readability_path; + DROP INDEX IF EXISTS archive_readability_json_path; + DROP INDEX IF EXISTS archive_readability_txt_path; + DROP INDEX IF EXISTS archive_screenshot_path; + ''') + + session.execute('DELETE FROM archive') # noqa + session.execute(''' + ALTER TABLE archive DROP COLUMN IF EXISTS singlefile_path; + ALTER TABLE archive DROP COLUMN IF EXISTS readability_path; + ALTER TABLE archive DROP COLUMN IF EXISTS readability_json_path; + ALTER TABLE archive DROP COLUMN IF EXISTS readability_txt_path; + ALTER TABLE archive DROP COLUMN IF EXISTS screenshot_path; + ALTER TABLE archive DROP COLUMN IF EXISTS associated; + ''') + session.execute(''' + ALTER TABLE archive ADD COLUMN IF NOT EXISTS singlefile_path TEXT; + ALTER TABLE archive ADD COLUMN IF NOT EXISTS readability_path TEXT; + ALTER TABLE archive ADD COLUMN IF NOT EXISTS readability_json_path TEXT; + ALTER TABLE archive ADD COLUMN IF NOT EXISTS readability_txt_path TEXT; + ALTER TABLE archive ADD COLUMN IF NOT EXISTS screenshot_path TEXT; + ALTER TABLE archive ADD COLUMN IF NOT EXISTS contents TEXT; + ALTER TABLE archive ADD COLUMN IF NOT EXISTS status TEXT; + ''') + session.execute('ALTER TABLE archive ALTER COLUMN domain_id SET NOT NULL') + + session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS idempotency TEXT') + + session.execute('DELETE FROM video') # noqa + session.execute(''' + ALTER TABLE video DROP COLUMN IF EXISTS caption_path; + ALTER TABLE video DROP COLUMN IF EXISTS info_json_path; + ALTER TABLE video DROP COLUMN IF EXISTS poster_path; + ALTER TABLE video DROP COLUMN IF EXISTS video_path; + ALTER TABLE video DROP COLUMN IF EXISTS video_path; + ''') + session.execute(''' + ALTER TABLE video ADD COLUMN IF NOT EXISTS idempotency TEXT; + ALTER TABLE video ADD COLUMN IF NOT EXISTS caption_path TEXT; + ALTER TABLE video ADD COLUMN IF NOT EXISTS description_path TEXT; + ALTER TABLE video ADD COLUMN IF NOT EXISTS info_json_path TEXT; + ALTER TABLE video ADD COLUMN IF NOT EXISTS poster_path TEXT; + ALTER TABLE video ADD COLUMN IF NOT EXISTS video_path TEXT; + ALTER TABLE video ADD COLUMN IF NOT EXISTS caption TEXT; + ''') + session.execute(''' + DROP INDEX IF EXISTS video_caption_path_idx; + DROP INDEX IF EXISTS video_info_json_path_idx; + DROP INDEX IF EXISTS video_poster_path_idx; + DROP INDEX IF EXISTS video_video_path_idx; + DROP INDEX IF EXISTS video_url_idx; + ''') + session.execute(''' + ALTER TABLE video + ADD COLUMN textsearch tsvector + GENERATED ALWAYS AS ( + setweight(to_tsvector('english'::regconfig, title), 'A') || + setweight(to_tsvector('english'::regconfig, caption), 'D') + ) STORED + ''') + session.execute('CREATE INDEX IF NOT EXISTS video_textsearch_idx ON video USING GIN(textsearch)') + + session.execute('DROP INDEX IF EXISTS file_mimetype_idx') + session.execute('DROP INDEX IF EXISTS file_textsearch_idx') + session.execute('DROP INDEX IF EXISTS file_modification_datetime_idx') + session.execute('DROP INDEX IF EXISTS file_size_idx') + session.execute('DROP INDEX IF EXISTS file_model_idx') + session.execute('DROP INDEX IF EXISTS file_idempotency_idx') + session.execute('DROP INDEX IF EXISTS file_suffix_idx') + + # Restore the old file table. + session.execute('DROP TABLE IF EXISTS file') + session.execute('''CREATE TABLE file ( + id SERIAL PRIMARY KEY, + path TEXT, + title TEXT, + mimetype TEXT, + size BIGINT, + modification_datetime TIMESTAMP WITH TIME ZONE, + idempotency TEXT DEFAULT NULL, + textsearch tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english'::regconfig, title), 'A'::"char") + ) STORED + )''') + session.execute('CREATE UNIQUE INDEX file_path_idx ON file USING BTREE(path)') + session.execute('CREATE INDEX file_mimetype_idx ON file USING BTREE(mimetype)') + session.execute('CREATE INDEX file_textsearch_idx ON file USING GIN(textsearch)') diff --git a/app/package.json b/app/package.json index e4229e37..1367aa79 100644 --- a/app/package.json +++ b/app/package.json @@ -11,7 +11,7 @@ "react-dropzone": "^11.4.2", "react-keyed-file-browser": "^1.14.0", "react-qr-code": "^2.0.7", - "react-router-dom": "^5.2.0", + "react-router-dom": "^6.3.0", "react-scripts": "^3.4.1", "react-semantic-toasts": "^0.6.5", "react-timezone-select": "0.8.4", diff --git a/app/public/index.html b/app/public/index.html index a811237f..1faaf9b3 100644 --- a/app/public/index.html +++ b/app/public/index.html @@ -7,9 +7,9 @@ - + WROLPi - Bring your personal internet, off-grid - +
''' - # Make some test files to refresh. - example_dir = archive_directory / 'example.com' - example_dir.mkdir() - (example_dir / '2021-10-05 16:20:10.346823.html').touch() # renames to 2021-10-05-16-20-10_NA.html - (example_dir / '2021-10-05 16:20:10.346823.png').touch() - (example_dir / '2021-10-05 16:20:10.346823-readability.txt').touch() - (example_dir / '2021-10-05 16:20:10.346823-readability.json').touch() - (example_dir / '2021-10-05 16:20:10.346823-readability.html').touch() - with (example_dir / '2021-10-05 16:20:10.346823-readability.json').open('wt') as fh: - json.dump({'url': 'foo'}, fh) - - # These should log an error because they are missing the singlefile path. - (example_dir / '2021-10-05 16:20:10.readability.html').touch() - (example_dir / '2021-10-05 16:20:11.readability.json').touch() - # Bad files should also be ignored. - (example_dir / 'random file').touch() - (example_dir / '2021-10-05 16:20:10.346827-something.html').mkdir() - (example_dir / '2021-10-05 16:20:10 no extension').touch() + make_files_structure({ + # These are all for an individual Archive. + 'archive/example.com/2021-10-05 16:20:10.346823.html': singlefile_contents, # renames to 2021-10-05-16-20-10_NA + 'archive/example.com/2021-10-05 16:20:10.346823.png': None, + 'archive/example.com/2021-10-05 16:20:10.346823-readability.txt': 'article text contents', + 'archive/example.com/2021-10-05 16:20:10.346823-readability.json': '{"url": "https://example.com"}', + 'archive/example.com/2021-10-05 16:20:10.346823-readability.html': '', + # These should log an error because they are missing the singlefile path. + 'archive/example.com/2021-10-05 16:20:10.readability.html': '', + 'archive/example.com/2021-10-05 16:20:10.readability.json': None, + # Bad files should also be ignored. + 'archive/example.com/random file': 'hello', + 'archive/example.com/2021-10-05 16:20:10 no extension': None, + }) # The single archive is found. - _refresh_archives() - with get_db_session() as session: - assert session.query(Archive).count() == 1 + test_client.post('/api/files/refresh') + assert test_session.query(Archive).count() == 1 # Running the refresh does not result in a new archive. - _refresh_archives() - with get_db_session() as session: - assert session.query(Archive).count() == 1 + test_client.post('/api/files/refresh') + assert test_session.query(Archive).count() == 1 # Archives file format was changed, lets check the new formats are found. - (example_dir / '2021-10-05-16-20-11_The Title.html').touch() - (example_dir / '2021-10-05-16-20-11_The Title.readability.json').write_text(json.dumps({'url': 'bar'})) - _refresh_archives() + make_files_structure({ + 'archive/example.com/2021-10-05-16-20-11_The Title.html': '', + 'archive/example.com/2021-10-05-16-20-11_The Title.readability.json': '{"url": "https://example.com"}', + }) + test_client.post('/api/files/refresh') # The old formatted archive above is renamed. - assert (example_dir / '2021-10-05-16-20-10_NA.html').is_file() - with get_db_session() as session: - assert session.query(Archive).count() == 2 + assert (test_directory / 'archive/example.com/2021-10-05-16-20-10_NA.html').is_file() + assert test_session.query(Archive).count() == 2 + + content = json.dumps({'search_str': 'text', 'model': 'archive'}) + request, response = test_client.post('/api/files/search', content=content) + assert response.status_code == HTTPStatus.OK + assert response.json['files'], 'No files matched "text"' + assert response.json['files'][0]['model'] == 'archive', 'Returned file was not an archive' + assert response.json['files'][0].get('path') == 'archive/example.com/2021-10-05-16-20-10_NA.html', \ + 'Could not find readability text file containing "text"' + assert response.json['files'][0]['archive']['archive_datetime'], 'Archive has no datetime' + + +def test_refresh_archives_invalid_file(test_session, test_directory, test_client, make_files_structure): + """ + Invalid Archives should be removed during refresh. + """ + *_, bogus_path = make_files_structure({ + '2022-09-04-16-20-11_The Title.html': '', + '2022-09-04-16-20-11_The Title.readability.json': '{"url": "https://example.com"}', + 'bogus file': None, + }) + test_client.post('/api/files/refresh') + assert test_session.query(File).count() == 3 # Three files total. + assert test_session.query(Archive).count() == 1 # Only one real archive. + + # Create an invalid Archive, it should be removed during refresh. + test_session.add(Archive(singlefile_path=bogus_path)) + # Claim the bogus file as an Archive, this should be removed. + bogus_file: File = test_session.query(File).filter_by(path=bogus_path).one() + bogus_file.model = 'archive' + test_session.commit() + assert test_session.query(File).count() == 3 + assert test_session.query(Archive).count() == 2 + + test_client.post('/api/files/refresh') + assert test_session.query(File).count() == 3, 'More files in DB than we created' + assert test_session.query(Archive).count() == 1, 'The bogus Archive was not removed' + assert bogus_file.model is None, 'Model was not removed' def test_group_archive_files(test_directory): @@ -488,9 +518,10 @@ def test_migrate_archive_files(test_session, archive_directory, archive_factory) archive1 = archive_factory('example.com', title='the title') archive2 = archive_factory('example.com') archive3 = archive_factory('example.org') - assert archive1.singlefile_path.path.is_file() - assert archive2.singlefile_path.path.is_file() - assert archive3.singlefile_path.path.is_file() + test_session.commit() + assert archive1.singlefile_path.is_file() + assert archive2.singlefile_path.is_file() + assert archive3.singlefile_path.is_file() example_com_directory = archive_directory / 'example.com' example_org_directory = archive_directory / 'example.org' @@ -532,6 +563,25 @@ def test_migrate_archive_files(test_session, archive_directory, archive_factory) 'example.org/2000-01-01-00-00-00_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.readability.json')), ] # New Archive files still exist. - assert all(i.is_file() for i in archive1.my_paths()) - assert all(i.is_file() for i in archive2.my_paths()) - assert all(i.is_file() for i in archive3.my_paths()) + assert all(i.is_file() for i in archive1.my_files()) + assert all(i.is_file() for i in archive2.my_files()) + assert all(i.is_file() for i in archive3.my_files()) + + +def test_archive_files(test_directory): + af = lib.ArchiveFiles( + singlefile=test_directory / 'archive/singlefile.html', + ) + assert repr(af) == \ + "" + + +def test_archive_order(test_session, test_directory, archive_factory): + archive1 = archive_factory() + archive2 = archive_factory() + archive2.archive_datetime += timedelta(seconds=10) + test_session.commit() + + assert archive2 > archive1 + assert archive1 < archive2 diff --git a/modules/files/conftest.py b/modules/files/conftest.py deleted file mode 100644 index 9921ab2a..00000000 --- a/modules/files/conftest.py +++ /dev/null @@ -1,21 +0,0 @@ -import pathlib -from typing import List, Callable - -import pytest - - -@pytest.fixture -def make_files_structure(test_directory) -> Callable[[List[str]], List[pathlib.Path]]: - def create_files(paths: List[str]) -> List[pathlib.Path]: - files = [] - for name in paths: - path = test_directory / name - if name.endswith('/'): - path.mkdir() - else: - path.parent.mkdir(parents=True, exist_ok=True) - path.touch() - files.append(path) - return files - - return create_files diff --git a/modules/files/lib.py b/modules/files/lib.py deleted file mode 100644 index 83fd55ae..00000000 --- a/modules/files/lib.py +++ /dev/null @@ -1,185 +0,0 @@ -import asyncio -import re -import subprocess -from functools import wraps -from pathlib import Path -from typing import List -from uuid import uuid4 - -import psycopg2 -from sqlalchemy.orm import Session - -from modules.files.models import File -from wrolpi.cmd import which -from wrolpi.common import get_media_directory, wrol_mode_check, walk, chunks, logger -from wrolpi.dates import from_timestamp -from wrolpi.db import get_db_session, get_db_curs, get_ranked_models -from wrolpi.errors import InvalidFile -from wrolpi.vars import PYTEST - -logger = logger.getChild(__name__) - -__all__ = ['list_files', 'delete_file', 'split_file_name', 'upsert_file', 'refresh_files', 'search', 'get_mimetype'] - - -def filter_parent_directories(directories: List[Path]) -> List[Path]: - """ - Remove parent directories if their children are in the list. - - >>> filter_parent_directories([Path('foo'), Path('foo/bar'), Path('baz')]) - [Path('foo/bar'), Path('baz')] - """ - unique_children = set() - for directory in sorted(directories): - for parent in directory.parents: - # Remove any parent of this child. - if parent in unique_children: - unique_children.remove(parent) - unique_children.add(directory) - - # Restore the original order. - new_directories = [i for i in directories if i in unique_children] - return new_directories - - -def list_files(directories: List[str]) -> List[Path]: - """ - List all files down to the directories provided. This includes all parent directories of the directories. - """ - media_directory = get_media_directory() - - # Always display the media_directory files. - paths = list(media_directory.iterdir()) - - if directories: - directories = [media_directory / i for i in directories if i] - directories = filter_parent_directories(directories) - for directory in directories: - directory = media_directory / directory - for parent in directory.parents: - is_relative_to = str(media_directory).startswith(str(parent)) - if parent == Path('.') or is_relative_to: - continue - paths.extend(parent.iterdir()) - paths.extend(directory.iterdir()) - - return paths - - -@wrol_mode_check -def delete_file(file: str): - """Delete a file in the media directory.""" - file = get_media_directory() / file - if file.is_dir() or not file.is_file(): - raise InvalidFile(f'Invalid file {file}') - file.unlink() - - -FILE_NAME_REGEX = re.compile(r'[_ ]') - - -def split_file_name(path: Path) -> List[str]: - """ - Split a file name into words. - - >>> split_file_name(Path('foo.mp4')) - ['foo'] - >>> split_file_name(Path('foo bar.mp4')) - ['foo', 'bar'] - """ - words = [str(i) for i in FILE_NAME_REGEX.split(path.stem)] - return words - - -FILE_BIN = which('file', '/usr/bin/file') - - -def get_mimetype(path: Path) -> str: - """Get the mimetype using the builtin `file` command.""" - cmd = (FILE_BIN, '--mime-type', str(path.absolute())) - output = subprocess.check_output(cmd) - output = output.decode() - mimetype = output.split(' ')[-1].strip() - return mimetype - - -def upsert_file(path: Path, session: Session) -> File: - """Update/insert a File in the DB. Gather metadata about it.""" - file = session.query(File).filter_by(path=path).one_or_none() - if not file: - file = File(path=path) - session.add(file) - if not file.mimetype: - file.mimetype = get_mimetype(path) - if not file.title: - file.title = split_file_name(path) - if not file.size or not file.modification_datetime: - stat = path.stat() - file.size = stat.st_size - file.modification_datetime = file.modification_datetime or from_timestamp(stat.st_mtime) - - return file - - -def _refresh_files(): - """Find and index all files""" - logger.info('Refreshing Files') - # Mark all files as unverified. Any records that are unverified after refresh will be deleted. - with get_db_curs(commit=True) as curs: - curs.execute('UPDATE file SET idempotency=null') # noqa - - paths = filter(lambda i: i.is_file(), walk(get_media_directory())) - idempotency = str(uuid4()) - for chunk in chunks(paths, 20): - with get_db_session(commit=True) as session: - for path in chunk: - file = upsert_file(path, session) - file.idempotency = idempotency - - # Remove any records where the file no longer exists. - with get_db_curs(commit=True) as curs: - curs.execute('DELETE FROM file WHERE idempotency IS NULL') - - logger.info('Done refreshing Files') - - -@wraps(_refresh_files) -def refresh_files(): - """Schedule a refresh task if not testing. If testing, do a synchronous refresh.""" - if PYTEST: - return _refresh_files() - - async def _(): - return _refresh_files() - - asyncio.create_task(_()) - - -def search(search_str: str, limit: int, offset: int): - """ - Search the "title" of each file. The title is the words in the file's stem. - """ - with get_db_curs() as curs: - stmt = ''' - SELECT id, ts_rank_cd(textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total - FROM file - WHERE textsearch @@ websearch_to_tsquery(%(search_str)s) - OFFSET %(offset)s LIMIT %(limit)s - ''' - params = dict(search_str=search_str, offset=offset, limit=limit) - curs.execute(stmt, params) - - try: - results = list(map(dict, curs.fetchall())) - except psycopg2.ProgrammingError: - # No files. - return [], 0 - - total = results[0]['total'] if results else 0 - ranked_ids = [i['id'] for i in results] - - with get_db_session() as session: - results = get_ranked_models(ranked_ids, File, session) - results = [i.__json__() for i in results] - - return results, total diff --git a/modules/files/models.py b/modules/files/models.py deleted file mode 100644 index 8578925a..00000000 --- a/modules/files/models.py +++ /dev/null @@ -1,36 +0,0 @@ -from sqlalchemy import Integer, Column, String, Computed -from sqlalchemy.orm import deferred - -from wrolpi.common import Base, ModelHelper, get_media_directory, tsvector -from wrolpi.dates import TZDateTime -from wrolpi.media_path import MediaPathType - - -class File(ModelHelper, Base): - __tablename__ = 'file' - id = Column(Integer, primary_key=True) - path = Column(MediaPathType) - - idempotency = Column(String) - mimetype = Column(String) - modification_datetime = Column(TZDateTime) - size = Column(Integer) - title = deferred(Column(String)) # used by textsearch column - - textsearch = deferred( - Column(tsvector, Computed('''setweight(to_tsvector('english'::regconfig, title), 'A'::"char")'''))) - - def __repr__(self): - return f'' - - def __json__(self): - path = self.path.path.relative_to(get_media_directory()) - d = dict( - id=self.id, - mimetype=self.mimetype, - modified=self.modification_datetime, # React browser expects this. - path=path, - size=self.size, - key=path, - ) - return d diff --git a/modules/files/test/test_lib.py b/modules/files/test/test_lib.py deleted file mode 100644 index 25322462..00000000 --- a/modules/files/test/test_lib.py +++ /dev/null @@ -1,207 +0,0 @@ -import shutil -from pathlib import Path -from typing import List, Iterable - -import pytest -from sqlalchemy.orm import Session - -from modules.files import lib -from modules.files.models import File -from wrolpi.errors import InvalidFile -from wrolpi.media_path import MediaPathType -from wrolpi.vars import PROJECT_DIR - - -def assert_files(session: Session, expected): - files = {str(i.path.relative) for i in session.query(File).all()} - assert files == set(expected) - - -def test_list_files(make_files_structure, test_directory): - files = [ - 'archives/bar.txt', - 'archives/baz/bar.txt', - 'archives/baz/foo.txt', - 'archives/foo.txt', - 'empty directory/', - 'videos/other video.mp4', - 'videos/some video.mp4', - ] - make_files_structure(files) - - def check_list_files(path, expected): - result = lib.list_files(path) - result = sorted(result) - assert [str(i.relative_to(test_directory)) for i in result] == expected - - check_list_files([], ['archives', 'empty directory', 'videos']) - # Falsey directories are ignored. - check_list_files([None, ''], ['archives', 'empty directory', 'videos']) - check_list_files(['empty directory'], ['archives', 'empty directory', 'videos']) - check_list_files(['empty directory'], ['archives', 'empty directory', 'videos']) - # Trailing slash is ignored. - check_list_files(['empty directory/'], ['archives', 'empty directory', 'videos']) - - check_list_files(['archives'], [ - 'archives', - 'archives/bar.txt', - 'archives/baz', - 'archives/foo.txt', - 'empty directory', - 'videos', - ]) - check_list_files(['archives/baz'], [ - 'archives', - 'archives/bar.txt', - 'archives/baz', - 'archives/baz/bar.txt', - 'archives/baz/foo.txt', - 'archives/foo.txt', - 'empty directory', - 'videos', - ]) - # Including duplicate directories does not change results. - check_list_files(['archives/baz', 'archives'], [ - 'archives', - 'archives/bar.txt', - 'archives/baz', - 'archives/baz/bar.txt', - 'archives/baz/foo.txt', - 'archives/foo.txt', - 'empty directory', - 'videos', - ]) - check_list_files(['videos'], [ - 'archives', - 'empty directory', - 'videos', - 'videos/other video.mp4', - 'videos/some video.mp4', - ]) - # All files and directories are listed. - check_list_files(['videos', 'archives/baz'], [ - 'archives', - 'archives/bar.txt', - 'archives/baz', - 'archives/baz/bar.txt', - 'archives/baz/foo.txt', - 'archives/foo.txt', - 'empty directory', - 'videos', - 'videos/other video.mp4', - 'videos/some video.mp4', - ]) - - -@pytest.mark.parametrize( - 'directories,expected', [ - ([Path('foo')], [Path('foo')]), - ([Path('foo'), Path('bar')], [Path('foo'), Path('bar')]), - ([Path('foo'), Path('foo/bar'), Path('baz')], [Path('foo/bar'), Path('baz')]), - ([Path('foo/bar'), Path('foo'), Path('baz')], [Path('foo/bar'), Path('baz')]), - ([Path('baz'), Path('foo/bar'), Path('foo')], [Path('baz'), Path('foo/bar')]), - ([Path('foo/bar'), Path('foo'), Path('foo/bar/baz')], [Path('foo/bar/baz')]), - ] -) -def test_filter_parent_directories(directories, expected): - assert lib.filter_parent_directories(directories) == expected - - -def test_delete_file(make_files_structure, test_directory): - """ - File in the media directory can be deleted. - """ - files = [ - 'archives/foo.txt', - 'bar.txt', - 'baz/', - ] - make_files_structure(files) - - lib.delete_file('bar.txt') - assert (test_directory / 'archives/foo.txt').is_file() - assert not (test_directory / 'bar.txt').is_file() - assert (test_directory / 'baz').is_dir() - - lib.delete_file('archives/foo.txt') - assert not (test_directory / 'archives/foo.txt').is_file() - assert not (test_directory / 'bar.txt').is_file() - - with pytest.raises(InvalidFile): - lib.delete_file('baz') - with pytest.raises(InvalidFile): - lib.delete_file('does not exist') - - assert (test_directory / 'baz').is_dir() - - -@pytest.mark.parametrize( - 'path,expected', - [ - ('foo.mp4', ['foo']), - ('foo bar.txt', ['foo', 'bar']), - ('foo_bar.mp4', ['foo', 'bar']), - ('foo-bar', ['foo-bar']), - ('123 foo bar', ['123', 'foo', 'bar']), - ('123foo bar', ['123foo', 'bar']), - ] -) -def test_split_file_name(path, expected): - assert lib.split_file_name(Path(path)) == expected - - -def test_refresh_files(test_session, make_files_structure, test_directory): - files = make_files_structure([ - 'foo.txt', - 'bar.txt', - 'baz.txt', - ]) - foo, bar, baz = files - - def get_relative_strs(files_: Iterable[MediaPathType]) -> List[str]: - return sorted([str(i.path.path.relative_to(test_directory)) for i in files_]) - - lib.refresh_files() - results = test_session.query(File) - assert get_relative_strs(results) == ['bar.txt', 'baz.txt', 'foo.txt'] - - baz.unlink() - - lib.refresh_files() - results = test_session.query(File) - assert get_relative_strs(results) == ['bar.txt', 'foo.txt'] - - foo.unlink() - bar.unlink() - - lib.refresh_files() - results = test_session.query(File) - assert get_relative_strs(results) == [] - - -def test_mime_type(test_session, make_files_structure, test_directory): - """Files module uses the `file` command to get the mimetype of each file.""" - from PIL import Image - - foo, bar, baz, empty = make_files_structure([ - 'dir/foo text.txt', - 'dir/bar.jpeg', - 'dir/baz.mp4', - 'dir/empty', - ]) - foo.write_text('some text') - Image.new('RGB', (25, 25), color='grey').save(bar) - shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', baz) - - lib.refresh_files() - assert_files(test_session, ['dir/foo text.txt', 'dir/bar.jpeg', 'dir/baz.mp4', 'dir/empty']) - - foo = test_session.query(File).filter_by(path=f'{test_directory}/dir/foo text.txt').one() - bar = test_session.query(File).filter_by(path=f'{test_directory}/dir/bar.jpeg').one() - baz = test_session.query(File).filter_by(path=f'{test_directory}/dir/baz.mp4').one() - empty = test_session.query(File).filter_by(path=f'{test_directory}/dir/empty').one() - - assert foo.mimetype == 'text/plain' - assert bar.mimetype == 'image/jpeg' - assert baz.mimetype == 'video/mp4' - assert empty.mimetype == 'inode/x-empty' diff --git a/modules/inventory/api.py b/modules/inventory/api.py index 56be56a3..17abc179 100644 --- a/modules/inventory/api.py +++ b/modules/inventory/api.py @@ -8,7 +8,7 @@ from sanic_ext.extensions.openapi import openapi from wrolpi.common import run_after, recursive_map from wrolpi.errors import ValidationError from wrolpi.root_api import get_blueprint, json_response -from . import common, inventory, schema +from modules.inventory import common, inventory, schema NAME = 'inventory' @@ -16,25 +16,25 @@ bp = get_blueprint('Inventory', '/api/inventory') @bp.get('/categories') -def _get_categories(_: Request): +def get_categories(_: Request): categories = inventory.get_categories() return json_response(dict(categories=categories)) @bp.get('/brands') -def _get_brands(_: Request): +def get_brands(_: Request): brands = inventory.get_brands() return json_response(dict(brands=brands)) @bp.get('/') -def inventories_get(_: Request): +def get_inventories(_: Request): inventories = inventory.get_inventories() return json_response(dict(inventories=inventories)) @bp.get('/') -def inventory_get(_: Request, inventory_id: int): +def get_inventory(_: Request, inventory_id: int): by_category = common.get_inventory_by_category(inventory_id) by_subcategory = common.get_inventory_by_subcategory(inventory_id) by_name = common.get_inventory_by_name(inventory_id) diff --git a/modules/inventory/inventory.py b/modules/inventory/inventory.py index dceaf13c..6705dd6b 100644 --- a/modules/inventory/inventory.py +++ b/modules/inventory/inventory.py @@ -123,7 +123,7 @@ def get_categories() -> List[Tuple[str, str]]: """ categories = DEFAULT_CATEGORIES.copy() with get_db_curs() as curs: - curs.execute('SELECT DISTINCT subcategory, category FROM item ORDER BY 1, 2') + curs.execute("SELECT DISTINCT COALESCE(subcategory, ''), COALESCE(category, '') FROM item ORDER BY 1, 2") try: db_categories = curs.fetchall() db_categories = [tuple(i) for i in db_categories] diff --git a/modules/inventory/test/test_api.py b/modules/inventory/test/test_api.py index 08d7fb21..1561ec35 100644 --- a/modules/inventory/test/test_api.py +++ b/modules/inventory/test/test_api.py @@ -84,6 +84,15 @@ def test_inventory_api(test_session, test_client): inventory = response.json['inventories'][0] inventory_id = inventory['id'] + request, response = test_client.get(f'/api/inventory/{inventory_id}') + assert response.status_code == HTTPStatus.OK + + request, response = test_client.get('/api/inventory/categories') + assert response.status_code == HTTPStatus.OK + + request, response = test_client.get('/api/inventory/brands') + assert response.status_code == HTTPStatus.OK + inventory = dict(name='new name') request, response = test_client.put(f'/api/inventory/{inventory_id}', content=json.dumps(inventory)) assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code diff --git a/modules/map/models.py b/modules/map/models.py index 72043f6f..00a0e819 100644 --- a/modules/map/models.py +++ b/modules/map/models.py @@ -35,5 +35,5 @@ class MapFile(Base, ModelHelper): 'path': self.path, 'imported': self.imported, 'size': self.size, - 'time_to_import': dates.seconds_to_timestamp(seconds_to_import(self.path.path, self.size)), + 'time_to_import': dates.seconds_to_timestamp(seconds_to_import(self.path, self.size)), } diff --git a/modules/map/test/test_lib.py b/modules/map/test/test_lib.py new file mode 100644 index 00000000..2edf067a --- /dev/null +++ b/modules/map/test/test_lib.py @@ -0,0 +1,21 @@ +from unittest import mock + +from modules.map import lib + + +def test_get_map_paths(test_directory): + pbf_dir = test_directory / 'map/pbf' + pbf_dir.mkdir(parents=True) + + pbf = pbf_dir / 'map.osm.pbf' + pbf.touch() + dump = pbf_dir / 'map.dump' + dump.touch() + + with mock.patch('modules.map.lib.subprocess.check_output') as mock_check_output: + mock_check_output.return_value = b'OpenStreetMap Protocolbuffer Binary Format' + assert lib.get_map_paths() == [pbf, ] + + with mock.patch('modules.map.lib.subprocess.check_output') as mock_check_output: + mock_check_output.return_value = b'PostgreSQL custom database dump' + assert lib.get_map_paths() == [dump, ] diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py index 8bfe14d6..d901db1b 100644 --- a/modules/videos/__init__.py +++ b/modules/videos/__init__.py @@ -1 +1,123 @@ -from modules.videos import downloader +import json +import pathlib +from abc import ABC +from typing import List, Dict + +from sqlalchemy.orm import Session + +from modules.videos.models import Video, Channel +from wrolpi.captions import read_captions, extract_captions +from wrolpi.common import logger, register_modeler, register_after_refresh, limit_concurrent +from wrolpi.db import get_db_curs +from wrolpi.files.indexers import Indexer, register_indexer +from wrolpi.files.models import File +from wrolpi.vars import PYTEST +from .downloader import video_downloader # Import downloaders so they are registered. + +logger = logger.getChild(__name__) + +__all__ = ['video_modeler', 'VideoIndexer'] + + +@register_modeler +def video_modeler(groups: Dict[str, List[File]], session: Session): + # Only process groups that contain a video file. + count = 0 + local_groups = groups.copy() + for stem, group in local_groups.items(): + video_file = next((i for i in group if i.mimetype.split('/')[0] == 'video' and i.path.suffix != '.part'), None) + if not video_file: + # Not a video group. + continue + + session.flush(group) + poster_file = next((i for i in group if i.mimetype.split('/')[0] == 'image'), None) + caption_file = next((i for i in group if i.path.name.endswith('.en.vtt') or i.path.name.endswith('.en.srt')), + None) + info_json_file = next((i for i in group if i.path.name.endswith('.info.json')), None) + + if poster_file: + poster_file.associated = True + poster_file.do_index() + if caption_file: + caption_file.associated = True + caption_file.do_index() + if info_json_file: + info_json_file.associated = True + info_json_file.do_index() + + video: Video = Video.upsert(video_file, session) + video.poster_file = poster_file + video.caption_file = caption_file + video.info_json_file = info_json_file + + video.video_file.do_index() + video.validate(session) + + # Remove this group, it will not be processed by other modelers. + del groups[stem] + + count += 1 + if not count % 100: + # Commit occasionally. + session.commit() + + +@register_after_refresh +@limit_concurrent(1) +def video_cleanup(): + # Claim all Videos in a Channel's directory for that Channel. + logger.info('Claiming Videos for their Channels') + with get_db_curs(commit=True) as curs: + curs.execute(''' + UPDATE video v + SET channel_id = c.id + FROM channel c + WHERE + v.video_path LIKE c.directory || '/%'::VARCHAR + AND v.channel_id IS NULL + ''') + + +EXTRACT_SUBTITLES = False + + +@register_indexer('video') +class VideoIndexer(Indexer, ABC): + """Handles video files like mp4/ogg.""" + + @staticmethod + def get_description(file): + video_path: pathlib.Path = file.path.path if hasattr(file.path, 'path') else file.path + info_json_path = video_path.with_suffix('.info.json') + if info_json_path.is_file(): + with info_json_path.open('rt') as fh: + try: + return json.load(fh)['description'] + except Exception: + if not PYTEST: + logger.warning(f'Video info json file exists, but cannot get description. {file}') + return None + + @classmethod + def create_index(cls, file): + """ + Index the video file and it's associated files. + + a = title. + b = + c = description from info json. + d = captions from info json, or extracted from video file. + """ + a = cls.get_title(file) + c = cls.get_description(file) + + # Detect the associated caption file. + caption_path = file.path.with_suffix('.en.vtt') + d = None + if caption_path.is_file(): + d = read_captions(caption_path) + elif EXTRACT_SUBTITLES: + d = extract_captions(file.path) or '' + + return a, None, c, d diff --git a/modules/videos/api.py b/modules/videos/api.py index 6e41c581..f8fd16e5 100644 --- a/modules/videos/api.py +++ b/modules/videos/api.py @@ -1,16 +1,12 @@ -import asyncio -from functools import wraps from http import HTTPStatus -from typing import List from sanic import Blueprint, response from sanic.request import Request from sanic_ext import validate from sanic_ext.extensions.openapi import openapi -from wrolpi.common import create_websocket_feed, get_sanic_url, \ - wrol_mode_check from wrolpi.common import logger +from wrolpi.common import wrol_mode_check from wrolpi.root_api import add_blueprint, json_response from wrolpi.schema import JSONErrorResponse from . import lib, schema @@ -29,43 +25,6 @@ add_blueprint(bp) logger = logger.getChild(__name__) -refresh_queue, refresh_event = create_websocket_feed('refresh', '/feeds/refresh', content_bp) - - -@content_bp.post('/refresh') -@content_bp.post('/refresh/') -@openapi.description('Search for videos that have previously been downloaded and stored.') -@openapi.response(HTTPStatus.OK, schema.StreamResponse) -@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse) -@wrol_mode_check -async def refresh(_, channel_id: int = None): - refresh_logger = logger.getChild('refresh') - stream_url = get_sanic_url(scheme='ws', path='/api/videos/feeds/refresh') - - # Only one refresh can run at a time - if refresh_event.is_set(): - return response.json({'error': 'Refresh already running', 'stream_url': stream_url}, HTTPStatus.CONFLICT) - - refresh_event.set() - - async def do_refresh(): - try: - refresh_logger.info('refresh started') - - channel_ids = [channel_id] if channel_id else None - await refresh_videos(channel_ids) - - refresh_logger.info('refresh complete') - except Exception: - raise - finally: - refresh_event.clear() - - coro = do_refresh() - asyncio.ensure_future(coro) - refresh_logger.debug('do_refresh scheduled') - return response.json({'code': 'stream-started', 'stream_url': stream_url}) - @content_bp.post('/download/') @openapi.description('Update a channel catalog, download any missing videos') @@ -77,11 +36,6 @@ def download(_, channel_id: int = None): return response.empty() -@wraps(lib.refresh_videos) -async def refresh_videos(channel_ids: List[int] = None): - return lib.refresh_videos(channel_ids=channel_ids) - - @content_bp.post('/favorite') @openapi.definition( description='Toggle the favorite flag on a video', diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py index 1ec6adba..95a6766c 100644 --- a/modules/videos/channel/api.py +++ b/modules/videos/channel/api.py @@ -1,4 +1,5 @@ import asyncio +import pathlib from http import HTTPStatus from sanic import response, Blueprint @@ -8,11 +9,12 @@ from sanic_ext.extensions.openapi import openapi from wrolpi.common import logger, wrol_mode_check, make_media_directory, \ get_media_directory, run_after +from wrolpi.errors import UnknownDirectory, UnknownChannel from wrolpi.root_api import json_response from wrolpi.schema import JSONErrorResponse from wrolpi.vars import PYTEST from . import lib -from .. import schema +from .. import schema, Channel from ..lib import save_channels_config channel_bp = Blueprint('Channel', url_prefix='/api/videos/channels') @@ -32,7 +34,7 @@ async def get_channels(_: Request): @openapi.description('Get a Channel') @openapi.response(HTTPStatus.OK, schema.ChannelResponse) def channel_get(_: Request, channel_id: int = None): - channel = lib.get_channel(channel_id=channel_id) + channel = lib.get_channel(channel_id=channel_id, return_dict=True) channel.pop('info_json') return json_response({'channel': channel}) @@ -55,9 +57,7 @@ def channel_post(_: Request, body: schema.ChannelPostRequest): # Refresh the videos asynchronously if not PYTEST: - from ..api import refresh_videos - coro = refresh_videos([channel.id]) - asyncio.ensure_future(coro) + asyncio.ensure_future(channel.refresh_files()) return response.json({'success': 'Channel created successfully'}, HTTPStatus.CREATED, {'Location': f'/api/videos/channels/{channel.id}'}) @@ -74,6 +74,7 @@ def channel_post(_: Request, body: schema.ChannelPostRequest): @run_after(save_channels_config) @wrol_mode_check def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest): + body.directory = pathlib.Path(body.directory) if body.directory else None channel = lib.update_channel(data=body, channel_id=channel_id) return response.raw('', HTTPStatus.NO_CONTENT, headers={'Location': f'/api/videos/channels/{channel.id}'}) @@ -86,3 +87,25 @@ def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest): def channel_delete(_: Request, channel_id: int): lib.delete_channel(channel_id=channel_id) return response.raw('', HTTPStatus.NO_CONTENT) + + +@channel_bp.post('/refresh/') +@openapi.description('Refresh all files in the Channel directory') +@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) +@wrol_mode_check +def channel_refresh(_: Request, channel_id: int): + channel: Channel = lib.get_channel(channel_id=channel_id, return_dict=False) + if not channel: + raise UnknownChannel() + + asyncio.ensure_future(channel.refresh_files()) + return response.empty() + + +@channel_bp.post('/download/') +@openapi.description('Download the latest catalog for a Channel, download any missing videos.') +@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) +@wrol_mode_check +def channel_download(_: Request, channel_id: int): + lib.download_channel(channel_id) + return response.empty() diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index 2dd9efbe..02a654af 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -1,5 +1,6 @@ +import functools from pathlib import Path -from typing import List, Dict, Union +from typing import List, Dict, Union, Optional from sqlalchemy.orm import Session from sqlalchemy.orm.exc import NoResultFound @@ -7,7 +8,6 @@ from sqlalchemy.orm.exc import NoResultFound from wrolpi.common import run_after, logger, \ get_media_directory from wrolpi.db import get_db_curs, optional_session -from wrolpi.downloader import download_manager from wrolpi.errors import UnknownChannel, UnknownDirectory, APIError, ValidationError, InvalidDownload from .. import schema from ..common import check_for_channel_conflicts @@ -125,6 +125,9 @@ def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_ data.directory.mkdir() else: raise UnknownDirectory() + elif data.directory: + # Keep the old directory because the user can't change a Channel's directory. + data.directory = channel.directory # Verify that the URL/Name/directory aren't taken check_for_channel_conflicts( @@ -165,7 +168,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic name=data.name, url=data.url or None, match_regex=data.match_regex, - directory=data.directory, # noqa + directory=get_media_directory() / data.directory, download_frequency=data.download_frequency, source_id=data.source_id, ) diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index b40c4a01..b2b637dc 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -9,11 +9,10 @@ import mock import pytest from modules.videos.channel.lib import delete_channel -from modules.videos.common import get_no_channel_directory -from modules.videos.models import Channel, Video +from modules.videos.models import Channel from wrolpi.dates import now from wrolpi.db import get_db_session -from wrolpi.downloader import download_manager, Download +from wrolpi.downloader import download_manager, Download, DownloadResult from wrolpi.test.common import assert_dict_contains @@ -26,31 +25,8 @@ def test_get_channels(test_directory, channel_factory, test_client): assert len(response.json['channels']) == 3 -@mock.patch('modules.videos.lib.validate_video') -def test_refresh_no_channel(_, test_session, test_client, test_directory, simple_channel): - # A regular user-defined channel - vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4' - vid1.touch() - vid2 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title2.mp4' - vid2.touch() - - # The special NO CHANNEL directory. - no_channel_path = get_no_channel_directory() - vid3 = no_channel_path / 'some video.mp4' - vid3.touch() - no_channel_subdir = no_channel_path / 'subdir' - no_channel_subdir.mkdir() - vid4 = no_channel_subdir / 'other video.mp4' - vid4.touch() - - test_client.post('/api/videos/refresh') - assert test_session.query(Video).count() == 4 - - def test_get_video(test_client, test_session, simple_channel, video_factory): - """ - Test that you get can information about a video. Test that video file can be gotten. - """ + """Test that you get can information about a video. Test that video file can be gotten.""" now_ = now() video1 = video_factory(channel_id=simple_channel.id) video1.upload_date, video1.title = now_, 'vid1' @@ -66,11 +42,11 @@ def test_get_video(test_client, test_session, simple_channel, video_factory): # Get the video info we inserted _, response = test_client.get('/api/videos/video/1') assert response.status_code == HTTPStatus.OK, response.json - assert_dict_contains(response.json['video'], {'title': 'vid1'}) + assert_dict_contains(response.json['file']['video'], {'title': 'vid1'}) # The next video is included. assert response.json['prev'] is None - assert_dict_contains(response.json['next'], {'title': 'vid2'}) + assert_dict_contains(response.json['next']['video'], {'title': 'vid2'}) def test_channel_no_download_frequency(test_client, test_session, test_directory, simple_channel): @@ -118,7 +94,7 @@ def test_channel_frequency_update(download_channel, test_client, test_session, t assert old_frequency data = dict( - directory=str(download_channel.directory.path), + directory=str(download_channel.directory), name=download_channel.name, url=download_channel.url, download_frequency=100, @@ -192,73 +168,12 @@ def test_channel_download_crud(test_session, simple_channel): assert test_session.query(Download).count() == 0 -def test_video_search(test_client, test_session): - """ - Test that videos can be searched and that their order is by their textsearch rank. - """ - # These captions have repeated letters, so they will be higher in the ranking. - videos = [ - ('1', 'b b b b e d d'), - ('2', '2 b b b d'), - ('3', 'b b'), - ('4', 'b e e'), - ('5', ''), - ] - for title, caption in videos: - test_session.add(Video(title=title, caption=caption, video_path='foo')) - test_session.commit() - - def do_search(search_str, limit=20): - d = json.dumps({'search_str': search_str, 'limit': limit}) - _, resp = test_client.post('/api/videos/search', content=d) - return resp - - def search_is_as_expected(resp, expected): - assert resp.status_code == HTTPStatus.OK - response_ids = [i['id'] for i in resp.json['videos']] - assert response_ids == expected - assert resp.json['totals']['videos'] == len(expected) - - # Repeated runs should return the same result - for _ in range(2): - # Only videos with a b are returned, ordered by the amount of b's - response = do_search('b') - search_is_as_expected(response, [1, 2, 3, 4]) - - # Only two captions have e - response = do_search('e') - search_is_as_expected(response, [4, 1]) - - # Only two captions have d - response = do_search('d') - search_is_as_expected(response, [1, 2]) - - # 5 can be gotten by it's title - response = do_search('5') - search_is_as_expected(response, [5]) - - # only video 1 has e and d - response = do_search('e d') - search_is_as_expected(response, [1]) - - # video 1 and 4 have b and e, but 1 has more - response = do_search('b e') - search_is_as_expected(response, [1, 4]) - - # Check totals are correct even with a limit - response = do_search('b', 2) - assert [i['id'] for i in response.json['videos']] == [1, 2] - assert response.json['totals']['videos'] == 4 - - def test_video_file_name(test_session, simple_video, test_client): - """ - If a Video has no title, the front-end can use the file name as the title. - """ + """If a Video has no title, the front-end can use the file name as the title.""" _, resp = test_client.get(f'/api/videos/video/{simple_video.id}') assert resp.status_code == HTTPStatus.OK - assert resp.json['video']['video_path'] == 'simple_video.mp4' - assert resp.json['video'].get('stem') == 'simple_video' + assert resp.json['file']['video']['video_path'] == 'simple_video.mp4' + assert resp.json['file']['video'].get('stem') == 'simple_video' def test_channel_conflicts(test_client, test_session, test_directory): @@ -339,26 +254,46 @@ def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_direc @pytest.mark.asyncio -async def test_download_channel_no_refresh(test_session, download_channel, video_download_manager): - """A Channel cannot be downloaded until it has been refreshed.""" +async def test_download_channel_no_refresh(test_session, download_channel, video_download_manager, video_factory): + """A Channel cannot be downloaded until it has been refreshed. + + Videos already downloaded are not downloaded again.""" + vid = video_factory(channel_id=download_channel.id, with_video_file=True, with_poster_ext='jpg') + vid.url = 'https://example.com/1' d = download_channel.get_download() test_session.commit() + assert not d.next_download - def check_refreshed(expected: bool): + def assert_refreshed(expected: bool): channel = test_session.query(Channel).one() assert channel.refreshed == expected - check_refreshed(False) + assert_refreshed(False) test_session.commit() - with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: - mock_extract_info.return_value = {'entries': [], 'url': 'foo', 'uploader': 'some uploader', + downloaded_urls = [] + + async def do_download(_, download): + # Track all downloaded URLs. + downloaded_urls.append(download.url) + return DownloadResult(success=True) + + with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \ + mock.patch('modules.videos.downloader.VideoDownloader.do_download', do_download): + entries = [ + dict(id=vid.source_id, view_count=0, webpage_url='https://example.com/1'), # Already downloaded. + dict(id='not downloaded', view_count=0, webpage_url='https://example.com/2'), + ] + mock_extract_info.return_value = {'entries': entries, 'url': 'foo', 'uploader': 'some uploader', 'channel_id': 'the id', 'id': 'the id'} await video_download_manager.do_downloads() await video_download_manager.wait_for_all_downloads() - check_refreshed(True) + # Channel was refreshed before downloading videos. + assert_refreshed(True) assert d.next_download + # Only the missing video was downloaded. + assert downloaded_urls == ['https://example.com/2'] def test_channel_post_directory(test_session, test_client, test_directory): @@ -367,7 +302,7 @@ def test_channel_post_directory(test_session, test_client, test_directory): data = dict(name='foo', directory='foo') request, response = test_client.post('/api/videos/channels', content=json.dumps(data)) assert response.status_code == HTTPStatus.CREATED - directory = test_session.query(Channel).filter_by(id=1).one().directory.path + directory = test_session.query(Channel).filter_by(id=1).one().directory assert (test_directory / 'foo') == directory assert not directory.is_dir() assert directory.is_absolute() @@ -376,7 +311,7 @@ def test_channel_post_directory(test_session, test_client, test_directory): data = dict(name='bar', directory='bar', mkdir=True) request, response = test_client.post('/api/videos/channels', content=json.dumps(data)) assert response.status_code == HTTPStatus.CREATED - directory = test_session.query(Channel).filter_by(id=2).one().directory.path + directory = test_session.query(Channel).filter_by(id=2).one().directory assert (test_directory / 'bar') == directory assert directory.is_dir() assert directory.is_absolute() @@ -455,7 +390,7 @@ def test_channel_crud(test_session, test_client, test_directory): assert response.status_code == HTTPStatus.NOT_FOUND -def test_search_channel_videos(test_client, test_session): +def test_search_videos_channel(test_client, test_session, video_factory): with get_db_session(commit=True) as session: channel1 = Channel(name='Foo') channel2 = Channel(name='Bar') @@ -469,29 +404,29 @@ def test_search_channel_videos(test_client, test_session): d = dict(channel_id=channel1.id) request, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == 0 + assert len(response.json['files']) == 0 with get_db_session(commit=True) as session: - vid1 = Video(title='vid1', channel_id=channel2.id, video_path='foo') - vid2 = Video(title='vid2', channel_id=channel1.id, video_path='foo') + vid1 = video_factory(title='vid1', channel_id=channel2.id) + vid2 = video_factory(title='vid2', channel_id=channel1.id) session.add(vid1) session.add(vid2) # Videos are gotten by their respective channels request, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == 1 - assert response.json['totals']['videos'] == 1 - assert_dict_contains(response.json['videos'][0], dict(id=2, title='vid2', channel_id=channel1.id)) + assert len(response.json['files']) == 1 + assert response.json['totals']['files'] == 1 + assert_dict_contains(response.json['files'][0], dict(path='vid2.mp4', video=dict(channel_id=channel1.id))) d = dict(channel_id=channel2.id) request, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == 1 - assert_dict_contains(response.json['videos'][0], dict(id=1, title='vid1', channel_id=channel2.id)) + assert len(response.json['files']) == 1 + assert_dict_contains(response.json['files'][0], dict(path='vid1.mp4', video=dict(channel_id=channel2.id))) -def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory): +def test_get_channel_videos_pagination(test_client, test_session, simple_channel, video_factory, assert_video_search): for i in range(50): video_factory(channel_id=simple_channel.id) @@ -500,7 +435,7 @@ def test_get_channel_videos_pagination(test_client, test_session, simple_channel test_session.flush() test_session.refresh(channel2) video_factory(channel_id=channel2.id) - test_session.add(Video(title='vid2', channel_id=channel2.id, video_path='foo')) + video_factory(title='vid2', channel_id=channel2.id, with_video_file=True) test_session.commit() # Get first, second, third, and empty pages of videos. @@ -513,10 +448,8 @@ def test_get_channel_videos_pagination(test_client, test_session, simple_channel ] last_ids = [] for offset, video_count in tests: - d = dict(channel_id=simple_channel.id, order_by='id', offset=offset) - _, response = test_client.post(f'/api/videos/search', content=json.dumps(d)) - assert response.status_code == HTTPStatus.OK - assert len(response.json['videos']) == video_count - current_ids = [i['id'] for i in response.json['videos']] + _, response = assert_video_search(channel_id=simple_channel.id, order_by='id', offset=offset, limit=20) + assert len(response.json['files']) == video_count, 'Returned videos does not match' + current_ids = [i['video']['id'] for i in response.json['files']] assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}' last_ids = current_ids diff --git a/modules/videos/channel/test/test_lib.py b/modules/videos/channel/test/test_lib.py index f1af902a..baf69b04 100644 --- a/modules/videos/channel/test/test_lib.py +++ b/modules/videos/channel/test/test_lib.py @@ -1,9 +1,11 @@ +from unittest import mock + import pytest from modules.videos import schema from modules.videos.channel import lib from modules.videos.lib import save_channels_config, import_channels_config -from modules.videos.models import Channel +from modules.videos.models import Channel, Video from wrolpi.errors import UnknownChannel @@ -89,3 +91,39 @@ def test_channels_no_url(test_session, test_directory): save_channels_config() import_channels_config() + + +def test_refresh_videos_finds_channel(test_session, test_client, channel_factory, video_factory): + """A Video will be assigned to a Channel by its info json first, then to its directory.""" + channel1 = channel_factory(source_id='channel1') + channel2 = channel_factory(source_id='channel2') + + # Put the video in the wrong directory. It should be matched to the info json channel first, then to it's directory + channel1.source_id = 'channel source id' + vid1 = video_factory( + with_video_file=(channel2.directory / 'video1.mp4'), + with_info_json={'channel_id': channel1.source_id}, + ) + test_session.commit() + assert not vid1.channel_id + + with mock.patch('wrolpi.common.after_refresh') as mock_after_refresh: + # Channel can be found in `video_cleanup`, prevent that with mock. + mock_after_refresh.side_effect = [] + test_client.post('/api/files/refresh') + + # Channel in info_json was found during validation. + assert vid1.channel_id == channel1.id + + # Refresh using cleanup, info json should be trusted. + test_client.post('/api/files/refresh') + assert vid1.channel_id == channel1.id + + # Remove the info json, the Video's Channel will be its directory. + vid1.channel_id = None + vid1.info_json_path.unlink() + test_session.commit() + test_client.post('/api/files/refresh') + test_session.refresh(vid1) + vid1 = test_session.query(Video).one() + assert vid1.channel_id == channel2.id diff --git a/modules/videos/common.py b/modules/videos/common.py index 78047b1f..07cfa9fd 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -1,12 +1,15 @@ import json +import json import os import pathlib +import re import subprocess import tempfile +from datetime import timedelta from decimal import Decimal from functools import partial from pathlib import Path -from typing import Union, Tuple, List, Set, Iterable, Optional +from typing import Union, List, Set, Iterable, Optional, Tuple import PIL from PIL import Image @@ -14,13 +17,13 @@ from sqlalchemy.orm import Session from wrolpi.cmd import which from wrolpi.common import logger, iterify, get_media_directory, \ - minimize_dict, any_extensions + minimize_dict, any_extensions, match_paths_to_suffixes from wrolpi.db import get_db_session, get_db_curs -from wrolpi.errors import UnknownFile, ChannelNameConflict, ChannelURLConflict, \ +from wrolpi.errors import ChannelNameConflict, ChannelURLConflict, \ ChannelDirectoryConflict, ChannelSourceIdConflict -from wrolpi.media_path import MediaPath +from wrolpi.files.models import File from wrolpi.vars import DEFAULT_FILE_PERMISSIONS -from .models import Channel, Video +from .models import Channel logger = logger.getChild(__name__) @@ -32,6 +35,7 @@ MINIMUM_VIDEO_KEYS = {'id', 'title', 'upload_date', 'duration', 'channel', 'chan 'view_count'} # These are the supported video formats. These are in order of their preference. VIDEO_EXTENSIONS = ('mp4', 'ogg', 'webm', 'flv') +match_video_extensions = partial(any_extensions, extensions=VIDEO_EXTENSIONS) FFMPEG_BIN = which('ffmpeg', '/usr/bin/ffmpeg', warn=True) @@ -49,40 +53,13 @@ def get_videos_directory() -> pathlib.Path: def get_no_channel_directory() -> pathlib.Path: - """Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist.""" + """Get the "NO CHANNEL" directory in the videos directory. Make it if it does not exist.""" # noqa directory = get_videos_directory() / 'NO CHANNEL' if not directory.is_dir(): directory.mkdir(parents=True) return directory -VALID_VIDEO_KINDS = {'video', 'caption', 'poster', 'description', 'info_json'} - - -def get_absolute_video_path(video: Video, kind: str = 'video') -> MediaPath: - if kind not in VALID_VIDEO_KINDS: - raise Exception(f'Unknown video path kind {kind}') - path = getattr(video, f'{kind}_path') - if path: - return path - raise UnknownFile() - - -match_video_extensions = partial(any_extensions, extensions=VIDEO_EXTENSIONS) - - -def generate_video_paths(directory: Union[str, pathlib.Path]) -> Tuple[str, pathlib.Path]: - """Generate a list of video paths in the provided directory.""" - directory = pathlib.Path(directory) - - for child in directory.iterdir(): - if child.is_file() and match_video_extensions(child.name): - child = child.absolute() - yield child - elif child.is_dir(): - yield from generate_video_paths(child) - - @iterify(set) def remove_duplicate_video_paths(paths: Iterable[Path]) -> Set[Path]: """ @@ -190,13 +167,46 @@ def get_matching_directories(path: Union[str, Path]) -> List[str]: return paths -def generate_video_poster(video_path: Path) -> Path: - """Create a poster (aka thumbnail) next to the provided video_path.""" +def duration_to_seconds(duration: str) -> int: + """Convert ffmpeg duration to seconds""" + hours, minutes, seconds = duration.split(':') + seconds, microseconds = seconds.split('.') + return int(timedelta( + hours=int(hours), + minutes=int(minutes), + seconds=int(seconds), + microseconds=int(microseconds), + ).total_seconds()) + + +def seconds_to_duration(seconds: int) -> str: + """Convert integer seconds to ffmpeg duration""" + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return f'{hours:02}:{minutes:02}:{seconds:02}.000' + + +DURATION_REGEX = re.compile(r'Duration: ([\d:.]+),') + + +def get_duration_from_ffmpeg(stderr: str) -> int: + """Extract the duration from FFPEG's stderr, return this duration as integer seconds.""" + duration = DURATION_REGEX.findall(stderr) + duration = duration_to_seconds(duration[0]) if duration else None + return duration + + +def generate_video_poster(video_path: Path) -> Tuple[Path, Optional[int]]: + """Create a poster (aka thumbnail) next to the provided video_path. Also returns the video duration.""" poster_path = video_path.with_suffix('.jpg') cmd = (FFMPEG_BIN, '-n', '-i', str(video_path), '-f', 'mjpeg', '-vframes', '1', '-ss', '00:00:05.000', str(poster_path)) + if poster_path.exists(): + # Poster already exists. + return poster_path, None try: - subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stderr = proc.stderr.decode() logger.info(f'Generated poster at {poster_path}') except subprocess.CalledProcessError as e: logger.warning(f'FFMPEG poster generation failed with stdout: {e.stdout.decode()}') @@ -205,7 +215,8 @@ def generate_video_poster(video_path: Path) -> Path: if not poster_path.exists(): raise Exception(f'Failed to find generated poster! {poster_path}') - return poster_path + duration = get_duration_from_ffmpeg(stderr) + return poster_path, duration def convert_image(existing_path: Path, destination_path: Path, ext: str = 'jpeg'): @@ -237,7 +248,7 @@ def is_valid_poster(poster_path: Path) -> bool: FFPROBE_BIN = which('ffprobe', '/usr/bin/ffprobe', warn=True) -def get_video_duration(video_path: Path) -> Optional[int]: +def get_video_duration(video_path: Path) -> int: """Get the duration of a video in seconds. Do this using ffprobe.""" if not isinstance(video_path, Path): video_path = Path(video_path) @@ -261,25 +272,29 @@ def get_video_duration(video_path: Path) -> Optional[int]: def check_for_video_corruption(video_path: Path) -> bool: - """Uses ffprobe to check for specific ways a video file can be corrupt.""" - if not isinstance(video_path, Path): - video_path = Path(video_path) + """Uses ffprobe to check for specific ways a video file can be corrupt. + + Also attempts to screenshot the video near the end of it's duration, this check if the video is corrupted somewhere + in the middle.""" + video_path = Path(video_path) if not isinstance(video_path, Path) else video_path + if not video_path.is_file(): raise FileNotFoundError(f'{video_path} does not exist!') if not FFPROBE_BIN: raise SystemError('ffprobe is not installed!') - cmd = [FFPROBE_BIN, str(video_path)] + # Just read the video using FFMPEG, this prints out useful information about the video. + cmd = (FFPROBE_BIN, str(video_path)) try: proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except subprocess.CalledProcessError as e: - logger.warning(f'FFPROBE failed to check for corruption with stdout: {e.stdout.decode()}') logger.warning(f'FFPROBE failed to check for corruption with stderr: {e.stderr.decode()}') return True # video is corrupt. messages = ( b'Invalid NAL unit size', b'Error splitting the input into NAL units', + b'Invalid data found when processing input', ) corrupt = False for error in messages: @@ -347,3 +362,19 @@ def minimize_video(video: dict) -> dict: video['info_json'] = minimize_video_info_json(video['info_json']) return video + + +match_video_paths = partial(match_paths_to_suffixes, suffix_groups=( + tuple(f'.{i}' for i in VIDEO_EXTENSIONS), + ('.jpg', '.jpeg', '.webp', '.png'), + ('.description',), + ('.en.vtt', '.en.srt'), + ('.info.json',), +)) + + +@iterify(tuple) +def match_video_files(files: List[File]) -> Tuple[File, File, File, File, File]: + video_paths = match_video_paths([i.path for i in files]) + for path in video_paths: + yield next((i for i in files if i.path == path), None) diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index df2c3e9c..3bdefeac 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -1,6 +1,7 @@ import json import pathlib import shutil +from http import HTTPStatus from itertools import zip_longest from typing import List from uuid import uuid4 @@ -13,6 +14,7 @@ from modules.videos.downloader import VideoDownloader, ChannelDownloader from modules.videos.lib import set_test_channels_config, set_test_downloader_config from modules.videos.models import Channel, Video from wrolpi.downloader import DownloadFrequency, DownloadManager, Download +from wrolpi.files.models import File from wrolpi.vars import PROJECT_DIR @@ -54,9 +56,7 @@ def channel_factory(test_session, test_directory): @pytest.fixture def download_channel(test_session, test_directory, video_download_manager) -> Channel: - """ - Get a test Channel that has a download frequency. - """ + """Get a test Channel that has a download frequency.""" # Add a frequency to the test channel, then give it a download. channel = Channel( directory=test_directory, @@ -75,64 +75,75 @@ def simple_video(test_session, test_directory, simple_channel) -> Video: """A Video with an empty video file whose channel is the Simple Channel.""" video_path = test_directory / 'simple_video.mp4' video_path.touch() - video = Video(video_path=video_path, channel_id=simple_channel.id) + video_file = File(path=video_path, model='video') + video = Video(video_file=video_file, channel_id=simple_channel.id) test_session.add(video) test_session.commit() video = test_session.query(Video).filter_by(id=video.id).one() return video -@pytest.fixture -def video_file(test_directory) -> pathlib.Path: - destination = test_directory / f'{uuid4()}.mp4' - shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', destination) - - yield destination - - destination.unlink() - - @pytest.fixture def video_factory(test_session, test_directory): """Creates Videos for testing.""" - def factory(channel_id: int = None, title: str = None, upload_date=None, with_video_file: bool = False, - with_info_json: dict = None, with_poster_ext: str = None, with_caption_file: bool = False): + def factory(channel_id: int = None, title: str = None, upload_date=None, with_video_file=None, + with_info_json: dict = None, with_poster_ext: str = None, with_caption_file: bool = False, + source_id: str = None): title = title or str(uuid4()) - if channel_id: + + if with_video_file and isinstance(with_video_file, (pathlib.Path, str)): + # Put the video exactly where specified for the test. + path = pathlib.Path(with_video_file) if not isinstance(with_video_file, pathlib.Path) else with_video_file + elif channel_id: + # Put the video in its Channel's directory. channel = test_session.query(Channel).filter_by(id=channel_id).one() path = (channel.directory or test_directory) / f'{title}.mp4' else: - (test_directory / 'videos/NO CHANNEL').mkdir(parents=True, exist_ok=True) + # Put any video not in a Channel and without a specified file in the NO CHANNEL directory. + (test_directory / 'videos/NO CHANNEL').mkdir(exist_ok=True, parents=True) path = test_directory / f'videos/NO CHANNEL/{title}.mp4' + video_file = File(path=path, model='video') + test_session.add(video_file) + if with_video_file: shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path) + else: + path.touch() info_json_path = None - if with_info_json is True: - # User requests a info json file, but does not provide one. - info_json_path = path.with_suffix('.info.json') - info_json_path.write_text(json.dumps({})) - elif with_info_json: + if with_info_json: # Use the provided object as the json. + with_info_json = {'duration': 5} if with_info_json is True else with_info_json info_json_path = path.with_suffix('.info.json') info_json_path.write_text(json.dumps(with_info_json)) + test_session.add(File(path=info_json_path)) poster_path = None if with_poster_ext: poster_path = path.with_suffix(f'.{with_poster_ext}') Image.new('RGB', (25, 25)).save(poster_path) + test_session.add(File(path=poster_path)) caption_path = None if with_caption_file: caption_path = path.with_suffix('.en.vtt') - caption_path.touch() + shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', caption_path) + test_session.add(File(path=caption_path)) - video = Video(video_path=str(path), title=title, channel_id=channel_id, source_id=title, - info_json_path=info_json_path, poster_path=poster_path, caption_path=caption_path, - upload_date=upload_date) + video = Video( + video_path=path, + title=title, + channel_id=channel_id, + source_id=source_id or title, + info_json_path=info_json_path, + poster_path=poster_path, + caption_path=caption_path, + upload_date=upload_date, + ) test_session.add(video) + video_file.do_index() return video return factory @@ -199,3 +210,71 @@ def assert_video_ids(test_session): assert id_ == expected, f'Video ids were not as expected: {expected_video_ids=} != {video_ids=}' return checker + + +@pytest.fixture +def video_with_search_factory(test_session, test_directory): + """A factory that creates a Video record with an associated video File record.""" + + def video_with_search(path: str = None, title: str = None, b_text: str = None, c_text: str = None, + d_text: str = None, mimetype: str = 'video/mp4'): + path = path or (test_directory / str(uuid4())).absolute() + video_file = File(path=path, a_text=title, d_text=d_text, mimetype=mimetype, model='video') + video = Video(title=title, video_file=video_file) + test_session.add(video) + return video + + return video_with_search + + +@pytest.fixture +def assert_video_search(test_client): + """A fixture which performs a video search request against the API. + + If assert_* params are passed, the response is checked.""" + + def search_videos( + assert_total: int = None, + assert_ids: List[int] = None, + assert_paths: List[str] = None, + search_str: str = None, + filters: List[str] = None, + offset: int = None, + limit: int = None, + order_by: str = None, + channel_id: int = None, + ): + content = dict() + if search_str is not None: + content['search_str'] = search_str + if filters is not None: + content['filters'] = filters + if offset is not None: + content['offset'] = offset + if limit is not None: + content['limit'] = limit + if order_by is not None: + content['order_by'] = order_by + if channel_id is not None: + content['channel_id'] = channel_id + + request, response = test_client.post('/api/videos/search', content=json.dumps(content)) + + assert response.json + assert response.status_code == HTTPStatus.OK + + if assert_ids is not None: + assert 'files' in response.json, 'No video files in response' + assert [i['video']['id'] for i in response.json['files']] == assert_ids, 'Video IDs do not match' + + if assert_paths is not None: + assert 'files' in response.json, 'No video files in response' + assert [i['video']['video_path'] for i in response.json['files']] == assert_paths, \ + 'Video paths do not match' + + if assert_total: + assert response.json['totals']['files'] == int(assert_total), 'Total video files does not match' + + return request, response + + return search_videos diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 98c95a78..e8850db5 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -13,19 +13,20 @@ from yt_dlp.extractor import YoutubeTabIE # noqa from yt_dlp.utils import UnsupportedError, DownloadError from wrolpi.cmd import which -from wrolpi.common import logger, extract_domain, get_media_directory +from wrolpi.common import logger, get_media_directory, escape_file_name from wrolpi.dates import now from wrolpi.db import get_db_session, get_db_curs from wrolpi.db import optional_session from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError +from wrolpi.files.models import File from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel from .common import apply_info_json, get_no_channel_directory, get_videos_directory -from .lib import upsert_video, refresh_channel_videos, get_downloader_config +from .lib import get_downloader_config from .models import Video, Channel from .schema import ChannelPostRequest -from .video_url_resolver import video_url_resolver +from .video_url_resolver import normalize_youtube_shorts_url logger = logger.getChild(__name__) ydl_logger = logger.getChild('youtube-dl') @@ -84,16 +85,16 @@ class ChannelDownloader(Downloader, ABC): async def do_download(self, download: Download) -> DownloadResult: """Update a Channel's catalog, then schedule downloads of every missing video.""" info = extract_info(download.url, process=False) - # Resolve the entries generator. + # Resolve the "entries" generator. info['entries'] = list(info['entries']) download.info_json = info if session := Session.object_session(download): # May not have a session during testing. session.commit() - name = info['uploader'] + kind = info['uploader'] channel_source_id = info['channel_id'] - channel = get_or_create_channel(channel_source_id, download.url, name) + channel = get_or_create_channel(channel_source_id, download.url, kind) channel.dict() # get all attributes while we have the session. location = f'/videos/channel/{channel.id}/video' if channel and channel.id else None @@ -103,33 +104,49 @@ class ChannelDownloader(Downloader, ABC): if is_a_playlist: downloads = self.get_playlist_downloads(download) else: - downloads = self.get_channel_downloads(download, info, channel) + downloads = await self.get_channel_downloads(download, info, channel) return DownloadResult(success=True, location=location, downloads=downloads) except Exception: - name = 'playlist' if is_a_playlist else 'channel' - logger.warning(f'Failed to update catalog of {name} {download.url}') + if PYTEST: + raise + kind = 'playlist' if is_a_playlist else 'channel' + logger.warning(f'Failed to update catalog of {kind} {download.url}') return DownloadResult(success=False, location=location, error=str(traceback.format_exc())) @staticmethod - def get_channel_downloads(download: Download, info: dict, channel: Channel) -> List[str]: + async def get_channel_downloads(download: Download, info: dict, channel: Channel) -> List[str]: """Get a list of all videos in a Channel, schedule downloads for any missing videos.""" + channel_id = channel.id + update_channel_catalog(channel, info) - domain = extract_domain(download.url) - missing_videos = find_all_missing_videos(channel.id) - downloads = [] - for _, _, missing_video in missing_videos: - url = missing_video.get('webpage_url') or video_url_resolver(domain, missing_video) - downloads.append(url) + with get_db_session(commit=True) as session: + channel = get_channel(session, channel_id=channel_id, return_dict=False) + if not channel.videos or not channel.refreshed: + logger.warning(f'Refreshing videos in {channel.directory} because {channel} has no video records!') + await channel.refresh_files() + session.commit() # Commit the refresh. + session.refresh(channel) + + # Get all videos in the channel. + downloads = [i.get('webpage_url') or i.get('url') for i in download.info_json['entries']] + # YouTube Shorts are handled specially. + downloads = list(map(normalize_youtube_shorts_url, downloads)) + # Only download those that have not yet been downloaded. + already_downloaded = [i.url for i in video_downloader.already_downloaded(*downloads)] + downloads = [i for i in downloads if i not in already_downloaded] return downloads @staticmethod def get_playlist_downloads(download: Download) -> List[str]: """Get a list of all videos in a playlist, schedule downloads for any missing videos.""" # Get all videos in the playlist - downloads = [i['url'] for i in download.info_json['entries']] + downloads = [i.get('webpage_url') or i.get('url') for i in download.info_json['entries']] + # YouTube Shorts are handled specially. + downloads = [normalize_youtube_shorts_url(i) for i in downloads] # Only download those that have not yet been downloaded. - downloads = [i for i in downloads if not video_downloader.already_downloaded(i)] + already_downloaded = [i.url for i in video_downloader.already_downloaded(*downloads)] + downloads = [i for i in downloads if i not in already_downloaded] return downloads @@ -153,9 +170,13 @@ class VideoDownloader(Downloader, ABC): return f'' @optional_session - def already_downloaded(self, url: str, session: Session = None) -> bool: + def already_downloaded(self, *urls: str, session: Session = None) -> List: # We only consider a video record with a video file as "downloaded". - return bool(session.query(Video).filter(Video.url == url, Video.video_path != None, ).count()) # noqa + videos = list(session.query(Video).filter( + Video.url.in_(urls), + Video.video_path != None, # noqa + )) + return videos @classmethod def valid_url(cls, url) -> Tuple[bool, Optional[dict]]: @@ -200,7 +221,7 @@ class VideoDownloader(Downloader, ABC): # Use the default directory if this video has no channel. out_dir = get_no_channel_directory() if channel: - out_dir = channel.directory.path + out_dir = channel.directory out_dir.mkdir(exist_ok=True, parents=True) logs = None # noqa @@ -244,10 +265,14 @@ class VideoDownloader(Downloader, ABC): ) with get_db_session(commit=True) as session: - # If the video is from a channel, it will already be in the database. - source_id = entry['id'] - existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none() - video = upsert_video(session, video_path, channel, id_=existing_video.id if existing_video else None) + video_file = File.upsert(video_path, session) + video: Video = Video.upsert(video_file, session) + video.source_id = entry['id'] + if channel: + video.channel_id = channel.id + video.video_file.do_index() + video.validate(session) + session.flush([video]) video_id = video.id except UnrecoverableDownloadError: raise @@ -335,7 +360,7 @@ def get_or_create_channel(source_id: str = None, url: str = None, name: str = No raise ValueError(f'Cannot create channel without a name') # Channel does not exist. Create one in the video directory. - channel_directory = get_videos_directory() / name + channel_directory = get_videos_directory() / escape_file_name(name) if not channel_directory.is_dir(): channel_directory.mkdir(parents=True) data = ChannelPostRequest( @@ -409,7 +434,7 @@ def update_channel_catalog(channel: Channel, info: dict): # Write the Channel's info to a JSON file. if channel.directory: - info_json_path = channel.directory.path / f'{channel.name}.info.json' + info_json_path = channel.directory / f'{channel.name}.info.json' with info_json_path.open('wt') as fh: json.dump(info, fh, indent=2) @@ -440,7 +465,7 @@ def _find_all_missing_videos(channel_id: id) -> List[Tuple]: return missing_videos -def find_all_missing_videos(channel_id: int = None) -> Tuple[dict, dict]: +async def find_all_missing_videos(channel_id: int = None): """ Find all videos that don't have a video file, but are found in the DB (taken from the channel's info_json). @@ -453,7 +478,7 @@ def find_all_missing_videos(channel_id: int = None) -> Tuple[dict, dict]: # Check that the channel has some videos. We can't be sure what is missing if we don't know what we have. if channel.refreshed is False: # Refresh this channel's videos. - refresh_channel_videos(channel) + await channel.refresh_files() match_regex = re.compile(channel.match_regex) if channel.match_regex else None diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 9efe41cb..7cbebc31 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -1,24 +1,23 @@ +import functools import html import pathlib import re from collections import defaultdict -from typing import Tuple, Optional, List, Union -from uuid import uuid1 +from typing import Tuple, Optional, Union from sqlalchemy import or_ from sqlalchemy.orm import Session from yt_dlp import YoutubeDL from wrolpi import before_startup -from wrolpi.common import chunks, ConfigFile, get_media_directory, sanitize_link +from wrolpi.common import ConfigFile, get_media_directory, sanitize_link from wrolpi.dates import from_timestamp, Seconds from wrolpi.db import get_db_curs, get_db_session, optional_session -from wrolpi.media_path import MediaPath +from wrolpi.files.models import File from wrolpi.vars import PYTEST -from .captions import get_captions -from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \ - is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \ - get_no_channel_directory, check_for_video_corruption +from .common import is_valid_poster, convert_image, \ + generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \ + get_no_channel_directory, get_video_duration from .models import Channel, Video logger = logger.getChild(__name__) @@ -26,156 +25,66 @@ logger = logger.getChild(__name__) DEFAULT_DOWNLOAD_FREQUENCY = Seconds.week -def refresh_channel_videos(channel: Channel): - """ - Find all video files in a channel's directory. Add any videos not in the DB to the DB. - """ - # Set the idempotency key, we can remove any videos not touched during this search. - with get_db_curs(commit=True) as curs: - curs.execute('UPDATE video SET idempotency=NULL WHERE channel_id=%s', (channel.id,)) - - idempotency = str(uuid1()) - directory = channel.directory.path - - # A set of absolute paths that exist in the file system - possible_new_paths = generate_video_paths(directory) - possible_new_paths = remove_duplicate_video_paths(possible_new_paths) - - # Update all videos that match the current video paths - new_paths = [str(i) for i in possible_new_paths] - with get_db_curs(commit=True) as curs: - query = 'UPDATE video SET idempotency = %s WHERE channel_id = %s AND video_path = ANY(%s) RETURNING video_path' - curs.execute(query, (idempotency, channel.id, new_paths)) - existing_paths = {i for (i,) in curs.fetchall()} - - # Get the paths for any video not yet in the DB - # (paths in DB are relative, but we need to pass an absolute path) - new_videos = {i for i in possible_new_paths if str(i) not in existing_paths} - - for chunk in chunks(new_videos, 20): - with get_db_session(commit=True) as session: - for video_path in chunk: - video_path = pathlib.Path(video_path) - upsert_video(session, video_path, channel, idempotency=idempotency) - logger.debug(f'{channel.name}: Added {video_path.name}') - - with get_db_curs(commit=True) as curs: - stmt = 'DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL AND video_path IS NOT NULL RETURNING id' - curs.execute(stmt, (channel.id,)) - deleted_count = len(curs.fetchall()) - - if deleted_count: - deleted_status = f'Deleted {deleted_count} video records from channel {channel.name}' - logger.info(deleted_status) - - logger.info(f'{channel.name}: {len(new_videos)} new videos, {len(existing_paths)} already existed. ') - - with get_db_session(commit=True) as session: - channel = session.query(Channel).filter_by(id=channel.id).one() - channel.refreshed = True - - apply_info_json(channel.id) +class VideoInfoJSON(object): + channel_source_id = None + title = None + duration = None + view_count = None + url = None -def refresh_no_channel_videos(): - """ - Refresh the Videos in the NO CHANNEL directory. - """ - from modules.videos.downloader import get_no_channel_directory - directory = get_no_channel_directory() - if not directory.is_dir(): - return - - logger.info('Refreshing NO CHANNEL videos') - - idempotency = str(uuid1()) - - # Get all WROLPi compatible videos, remove any duplicates (different formats). - possible_new_paths = generate_video_paths(directory) - possible_new_paths = remove_duplicate_video_paths(possible_new_paths) - - new_paths = [str(i) for i in possible_new_paths] - with get_db_curs(commit=True) as curs: - query = 'UPDATE video SET idempotency = %s WHERE video_path = ANY(%s) RETURNING video_path' - curs.execute(query, (idempotency, new_paths)) - existing_paths = {i for (i,) in curs.fetchall()} - - new_videos = {i for i in possible_new_paths if str(i) not in existing_paths} - - for chunk in chunks(new_videos, 20): - with get_db_session(commit=True) as session: - for video_path in chunk: - upsert_video(session, pathlib.Path(video_path), idempotency=idempotency) - logger.debug(f'Added NO CHANNEL video {video_path}') - - with get_db_curs(commit=True) as curs: - curs.execute('DELETE FROM video WHERE channel_id IS NULL AND idempotency IS NULL RETURNING id') - deleted_count = len(curs.fetchall()) - - if deleted_count: - deleted_status = f'Deleted {deleted_count} video records in NO CHANNEL.' - logger.info(deleted_status) - - -def process_video_info_json(video: Video): +def process_video_info_json(video: Video) -> VideoInfoJSON: """ Parse the Video's info json file, return the relevant data. """ - title = duration = view_count = url = None + video_info_json = VideoInfoJSON() if info_json := video.get_info_json(): title = info_json.get('fulltitle') or info_json.get('title') - title = html.unescape(title) if title else None + video_info_json.title = html.unescape(title) if title else None - duration = info_json.get('duration') - view_count = info_json.get('view_count') - url = info_json.get('webpage_url') or info_json.get('url') + video_info_json.duration = info_json.get('duration') + video_info_json.view_count = info_json.get('view_count') + video_info_json.url = info_json.get('webpage_url') or info_json.get('url') + video_info_json.channel_source_id = info_json.get('channel_id') - return title, duration, view_count, url + return video_info_json -def validate_videos(): +@optional_session +@functools.lru_cache +def get_channel_id_by_source_id(session: Session, source_id: str) -> Optional[int]: + """Return the id of the Channel which matches the provided `source_id`. + + These are cached results.""" + channel = session.query(Channel).filter_by(source_id=source_id).one_or_none() + return channel.id if channel else None + + +def validate_video(video: Video, channel_generate_poster: bool, session: Session): """ - Validate all Videos not yet validated. A Video is validated when we have attempted to find its: title, duration, - view_count, url, caption, size. A Video is also valid when it has a JPEG poster, if any. If no poster can be - found, it will be generated from the video file. + Validate a single video. - This function marks the Video as validated, even if no data can be found so a Video will not be validated multiple - times. - """ - with get_db_curs() as curs: - curs.execute('SELECT id FROM video WHERE video_path IS NOT NULL AND validated IS FALSE') - video_ids = [i['id'] for i in curs.fetchall()] - curs.execute('SELECT id, generate_posters FROM channel') - channel_generate_posters = {i['id']: i['generate_posters'] for i in curs.fetchall()} + A Video is validated when we have attempted to find its: title, duration, view_count, url, caption, size, and + channel. - logger.info(f'Validating {len(video_ids)} videos.') - for chunk in chunks(video_ids, 20): - with get_db_session(commit=True) as session: - videos = session.query(Video).filter(Video.id.in_(chunk)).all() - for video in videos: - video.validate() - - -def validate_video(video: Video, channel_generate_poster: bool): - """ - Validate a single video. A Video is validated when we have attempted to find its: title, duration, - view_count, url, caption, size. A Video is also valid when it has a JPEG poster, if any. If no poster can be + A Video is also valid when it has a JPEG poster, if any. If no poster can be found, it will be generated from the video file. """ - if not video.title or not video.duration or not video.view_count or not video.url: + info_json_path = video.info_json_file.path if video.info_json_file else video.info_json_path + if info_json_path and (not video.title or not video.duration or not video.view_count or not video.url): # These properties can be found in the info json. - title, duration, view_count, url = process_video_info_json(video) - video.title = title - video.duration = duration - video.url = url + video_info_json = process_video_info_json(video) + video.title = video_info_json.title + video.duration = video_info_json.duration + video.url = video_info_json.url # View count will probably be overwritten by more recent data when this Video's Channel is # updated. - video.view_count = video.view_count or view_count + video.view_count = video.view_count or video_info_json.view_count - video_path = video.video_path.path if isinstance(video.video_path, MediaPath) else video.video_path + if video_info_json.channel_source_id: + video.channel_id = get_channel_id_by_source_id(source_id=video_info_json.channel_source_id) - # Check for corruption, but don't throw away the video even if it is corrupt. Errors will be logged in this call. - check_for_video_corruption(video_path) + video_path = video.video_path if not video.title or not video.upload_date or not video.source_id: # Video is missing things that can be extracted from the video file name. @@ -184,41 +93,41 @@ def validate_video(video: Video, channel_generate_poster: bool): video.title = video.title or html.unescape(title) video.upload_date = video.upload_date or upload_date video.source_id = video.source_id or source_id - if not video.duration: - # Video duration was not in the info json, use ffprobe. - video.duration = get_video_duration(video_path) - if not video.caption and video.caption_path: - video.caption = get_captions(video.caption_path.path) if not video.size or not video.modification_datetime: stat = video_path.stat() video.size = video.size or stat.st_size video.modification_datetime = video.modification_datetime or from_timestamp(stat.st_mtime) - if not video.poster_path: - # Video poster is not found, lets check near the video file. - for ext in ('.jpg', '.jpeg', '.webp', '.png'): - if (poster_path := video_path.with_suffix(ext)).is_file(): - video.poster_path = poster_path - break if channel_generate_poster: # Try to convert/generate, but keep the old poster if those fail. - video.poster_path = convert_or_generate_poster(video) or video.poster_path + new_poster_path, duration = convert_or_generate_poster(video) or video.poster_path + if new_poster_path: + # Poster was created/updated. + poster_file = File.upsert(new_poster_path, session) + session.flush([poster_file]) + video.poster_file = poster_file + video.poster_path = new_poster_path + if duration: + video.duration = duration + + if not video.duration: + # Duration was not retrieved during poster generation. + video.duration = get_video_duration(video_path) -def convert_or_generate_poster(video: Video) -> Optional[pathlib.Path]: +def convert_or_generate_poster(video: Video) -> Tuple[Optional[pathlib.Path], Optional[int]]: """ If a Video has a poster, but the poster is invalid, convert it. If a Video has no poster, generate one from the video file. Returns None if the poster was not converted, and not generated. """ - video_path = video.video_path.path + video_path = video.video_path # Modification/generation of poster is enabled for this channel. - if video.poster_path: + if video.poster_path or video.poster_file: # Check that the poster is a more universally supported JPEG. - old: pathlib.Path = video.poster_path.path if \ - isinstance(video.poster_path, MediaPath) else video.poster_path + old: pathlib.Path = video.poster_path if video.poster_path else video.poster_file.path new = old.with_suffix('.jpg') if old != new and new.exists(): @@ -231,63 +140,22 @@ def convert_or_generate_poster(video: Video) -> Optional[pathlib.Path]: try: convert_image(old, new) old.unlink(missing_ok=True) - logger.info(f'Converted invalid poster {old} to {new}') - return new + logger.info(f'Converted invalid poster {repr(str(old))} to {repr(str(new))}') + return new, None except Exception as e: logger.error(f'Failed to convert invalid poster {old} to {new}', exc_info=e) - return + return None, None - if not video.poster_path: + if not video.poster_path or not video.poster_file: # Video poster was not discovered, or converted. Let's generate it. try: - poster_path = generate_video_poster(video_path) + poster_path, duration = generate_video_poster(video_path) logger.debug(f'Generated poster for {video}') - return poster_path + return poster_path, duration except Exception as e: logger.error(f'Failed to generate poster for {video}', exc_info=e) - -def refresh_videos(channel_ids: List[int] = None): - """ - Find any videos in the channel directories and add them to the DB. Delete DB records of any videos not in the - file system. - - Yields status updates to be passed to the UI. - - :return: - """ - logger.info('Refreshing video files') - with get_db_session() as session: - if channel_ids: - channels = session.query(Channel).filter(Channel.id.in_(channel_ids)) - else: - channels = session.query(Channel).all() - - channels = list(channels) - - if not channels and channel_ids: - raise Exception(f'No channels match id(s): {channel_ids}') - elif not channels: - raise Exception(f'No channels in DB. Have you created any?') - - for channel in channels: - try: - refresh_channel_videos(channel) - except Exception as e: - logger.fatal(f'Failed to refresh videos for channel {channel.name}!', exc_info=e) - pass - - if not channel_ids: - # Refresh NO CHANNEL videos when not refreshing a specific channel. - refresh_no_channel_videos() - - # Fill in any missing data for all videos. - if not PYTEST: - import_channels_config() - - validate_videos() - - logger.info('Refresh of video files complete') + return None, None class ChannelsConfig(ConfigFile): @@ -470,9 +338,10 @@ def get_channels_config_from_db(session: Session) -> dict: favorites = defaultdict(lambda: {}) for video in favorite_videos: if video.channel: - favorites[str(video.channel.directory.relative)][video.video_path.path.name] = dict(favorite=video.favorite) + directory = video.channel.directory.relative_to(get_media_directory()) + favorites[str(directory)][video.video_path.name] = dict(favorite=video.favorite) else: - favorites['NO CHANNEL'][video.video_path.path.name] = dict(favorite=video.favorite) + favorites['NO CHANNEL'][video.video_path.name] = dict(favorite=video.favorite) favorites = dict(favorites) return dict(channels=channels, favorites=favorites) @@ -510,7 +379,6 @@ def import_channels_config(): """Import channel settings to the DB. Existing channels will be updated.""" logger.info('Importing videos config') try: - from .downloader import ChannelDownloader config = get_channels_config() channels, favorites = config.channels, config.favorites @@ -569,7 +437,7 @@ def import_channels_config(): if not channel: logger.warning(f'Cannot find channel {directory=} for favorites!') continue - channel_dir = channel.directory.path + channel_dir = channel.directory else: channel_dir = get_no_channel_directory() @@ -698,74 +566,3 @@ def parse_video_file_name(video_path: pathlib.Path) -> \ # Return the stem as a last resort title = pathlib.Path(video_path).stem.strip() return None, None, None, title - - -def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = None, idempotency: str = None, - id_: str = None) -> Video: - """ - Insert a video into the DB. Also, find any meta-files near the video file and store them on the video row. - - If id_ is provided, update that entry. - """ - if not video_path.is_absolute(): - raise ValueError(f'Video path is not absolute: {video_path}') - - # This function can update or insert a Video. - video = None - if id_: - video = session.query(Video).filter_by(id=id_).one() - if not video: - # The video may already be in the database, match the source_id to any existing video. - _, _, source_id, _ = parse_video_file_name(video_path) - if source_id: - video = session.query(Video).filter_by(source_id=source_id).one_or_none() - if not video: - video = Video() - session.add(video) - - # Set the file values, all other things can be found using these files. - poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path) - video.caption_path = caption_path - video.description_path = description_path - video.info_json_path = info_json_path - video.poster_path = poster_path - video.video_path = video_path - if channel and not str(video_path).startswith(str(channel.directory.path)): - raise ValueError(f'Video path is not within its channel {video_path=} not in {channel.directory=}') - - if channel: - video.channel_id = channel.id - video.idempotency = idempotency - - session.flush() - session.refresh(video) - - video.validate() - - session.flush() - - return video - - -def find_meta_files(path: pathlib.Path) -> Tuple[pathlib.Path, pathlib.Path, pathlib.Path, pathlib.Path]: - """ - Find all files that share a file's full path, except their extensions. It is assumed that file with the - same name, but different extension is related to that file. A None will be yielded if the meta file doesn't exist. - - Example: - >>> foo = pathlib.Path('foo.bar') - >>> find_meta_files(foo) - (pathlib.Path('foo.jpg'), pathlib.Path('foo.description'), - pathlib.Path('foo.en.vtt'), pathlib.Path('foo.info.json')) - """ - suffix = path.suffix - name, suffix, _ = str(path.name).rpartition(suffix) - meta_file_exts = (('.jpg', '.webp', '.png'), ('.description',), ('.en.vtt', '.en.srt'), ('.info.json',)) - for meta_exts in meta_file_exts: - for meta_ext in meta_exts: - meta_path = path.with_suffix(meta_ext) - if meta_path.exists(): - yield meta_path - break - else: - yield None diff --git a/modules/videos/models.py b/modules/videos/models.py index 74a30a82..38ad86ab 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -1,18 +1,21 @@ import json +import pathlib from datetime import datetime from pathlib import Path -from typing import Optional, Generator +from typing import Optional, Generator, Dict, List -from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, Computed -from sqlalchemy.orm import relationship, Session, deferred +from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey +from sqlalchemy.orm import relationship, Session from sqlalchemy.orm.collections import InstrumentedList -from wrolpi.common import Base, tsvector, ModelHelper, logger, get_media_directory +from wrolpi.common import Base, ModelHelper, logger, get_media_directory from wrolpi.dates import now, TZDateTime from wrolpi.db import get_db_curs from wrolpi.downloader import Download, download_manager -from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory -from wrolpi.media_path import MediaPathType, MediaPath +from wrolpi.errors import UnknownVideo +from wrolpi.files.lib import refresh_directory_files_recursively, glob_shared_stem +from wrolpi.files.models import File +from wrolpi.media_path import MediaPathType logger = logger.getChild(__name__) @@ -22,18 +25,8 @@ class Video(ModelHelper, Base): id = Column(Integer, primary_key=True) channel_id = Column(Integer, ForeignKey('channel.id')) channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos') - idempotency = Column(String) validated = Column(Boolean, default=False) - # File paths - caption_path = Column(MediaPathType) - description_path = Column(MediaPathType) - ext = Column(String) - info_json_path = Column(MediaPathType) - poster_path = Column(MediaPathType) - video_path = Column(MediaPathType) - - caption = deferred(Column(String)) # slow to fetch censored = Column(Boolean, default=False) duration = Column(Integer) favorite = Column(TZDateTime) @@ -46,18 +39,42 @@ class Video(ModelHelper, Base): view_count = Column(Integer) viewed = Column(TZDateTime) - textsearch = deferred( - Column(tsvector, Computed('''to_tsvector('english'::regconfig, - ((COALESCE(title, ''::text) || ' '::text) || - COALESCE(caption, ''::text)))'''))) + # Associated Files + caption_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE')) + caption_file: File = relationship('File', primaryjoin='Video.caption_path==File.path') + info_json_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE')) + info_json_file: File = relationship('File', primaryjoin='Video.info_json_path==File.path') + poster_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE')) + poster_file: File = relationship('File', primaryjoin='Video.poster_path==File.path') + video_path: pathlib.Path = Column(MediaPathType, ForeignKey('file.path', ondelete='CASCADE')) + video_file: File = relationship('File', primaryjoin='Video.video_path==File.path') def __repr__(self): - video_path = self.video_path - if self.video_path and isinstance(self.video_path, MediaPath): - video_path = self.video_path.path.relative_to(get_media_directory()) - return f'