diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index b8df54b5..666b729b 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -78,9 +78,7 @@ async def test_relationships(test_session, test_directory): def test_archive_refresh_deleted_archive(test_client, test_session, archive_directory, archive_factory): - """ - Archives/Domains should be deleted when archive files are deleted. - """ + """Archives/Domains should be deleted when archive files are deleted.""" archive1 = archive_factory('example.com', 'https://example.com/1') archive2 = archive_factory('example.com', 'https://example.com/1') archive3 = archive_factory('example.com') diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index cbee07b5..2cc409a7 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -240,10 +240,7 @@ def test_get_or_create_channel(test_session): (dict(name='qux'), c4), ] for kwargs, expected in tests: - try: - channel = get_or_create_channel(**kwargs) - except Exception as e: - raise Exception(f'get_or_create_channel failed with {kwargs=}') from e + channel = get_or_create_channel(**kwargs) assert expected.id == channel.id, f'Expected {expected} for {kwargs} but got {channel}' # A new channel is created. It will not be automatically downloaded. diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py index f2aa1773..8d9c9b14 100644 --- a/modules/videos/video/api.py +++ b/modules/videos/video/api.py @@ -7,7 +7,6 @@ from sanic_ext.extensions.openapi import openapi from wrolpi.common import logger, wrol_mode_check, run_after, get_media_directory, \ get_relative_to_media_directory -from wrolpi.db import get_db_session from wrolpi.errors import InvalidOrderBy, ValidationError from wrolpi.root_api import json_response from wrolpi.schema import JSONErrorResponse @@ -79,7 +78,7 @@ def video_delete(_: Request, video_ids: str): try: video_ids = [int(i) for i in str(video_ids).split(',')] except Exception: - return ValidationError('Unable to parse video ids') + raise ValidationError('Unable to parse video ids') lib.delete_videos(*video_ids) return response.raw('', HTTPStatus.NO_CONTENT) diff --git a/modules/videos/video/test/test_api.py b/modules/videos/video/test/test_api.py index 337a1ead..13d7be29 100644 --- a/modules/videos/video/test/test_api.py +++ b/modules/videos/video/test/test_api.py @@ -2,6 +2,8 @@ from datetime import timedelta from http import HTTPStatus from json import dumps +import pytest + from modules.videos.models import Video from modules.videos.video.lib import get_video_for_app from wrolpi.dates import now @@ -73,7 +75,8 @@ def test_get_video_for_app(test_session, simple_channel, simple_video): assert vid['video']['id'] == simple_video.id -def test_video_delete(test_session, test_directory, channel_factory, video_factory): +@pytest.mark.asyncio +def test_video_delete(test_client, test_session, test_directory, channel_factory, video_factory): """Video.delete() removes the video's files, but leave the DB record.""" channel1, channel2 = channel_factory(), channel_factory() vid1 = video_factory(channel_id=channel1.id, with_video_file=True, with_caption_file=True) @@ -89,7 +92,8 @@ def test_video_delete(test_session, test_directory, channel_factory, video_facto assert channel1.skip_download_videos is None assert channel2.skip_download_videos is None - vid1.delete() + request, response = test_client.delete(f'/api/videos/video/{vid1.id}') + assert response.status_code == HTTPStatus.NO_CONTENT # Video was added to skip list. assert len(channel1.skip_download_videos) == 1 @@ -98,12 +102,21 @@ def test_video_delete(test_session, test_directory, channel_factory, video_facto assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False assert vid2_video_path.is_file() and vid2_info_json_path.is_file() - vid2.delete() + request, response = test_client.delete(f'/api/videos/video/{vid2.id}') + assert response.status_code == HTTPStatus.NO_CONTENT assert test_session.query(Video).count() == 2 assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False + # 3 does not exist. + request, response = test_client.delete(f'/api/videos/video/3') + assert response.status_code == HTTPStatus.NOT_FOUND + + # Can't parse the ids. + request, response = test_client.delete(f'/api/videos/video/3,') + assert response.status_code == HTTPStatus.BAD_REQUEST + def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixture, test_download_manager): """Many methods are blocked when WROL Mode is enabled.""" @@ -144,3 +157,28 @@ def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixtu wrol_mode_fixture(False) assert not test_download_manager.stopped.is_set() + + +def test_video_directory_search(test_client, make_files_structure): + """Test that video directories can be searched.""" + make_files_structure([ + 'foo/', + 'fool/', + 'not a directory', + ]) + + def assert_directories(search_str, expected): + body = dict(search_str=search_str) + request, response = test_client.post('/api/videos/directories', content=dumps(body)) + assert response.status_code == HTTPStatus.OK + assert response.json['directories'] == expected + + # All directories are returned. + assert_directories(None, ['foo', 'fool', 'videos']) + assert_directories('', ['foo', 'fool', 'videos']) + # Matches both directories. + assert_directories('fo', ['foo', 'fool']) + # Matches the one directory exactly. + assert_directories('foo', ['foo']) + # Does not exist. + assert_directories('food', []) diff --git a/test/corrupted.mp4 b/test/corrupted.mp4 new file mode 100644 index 00000000..0a0de715 Binary files /dev/null and b/test/corrupted.mp4 differ diff --git a/test/example3.en.srt b/test/example3.en.srt new file mode 100644 index 00000000..e4136bdd --- /dev/null +++ b/test/example3.en.srt @@ -0,0 +1,49 @@ +1 +00:00:00,000 --> 00:00:05,269 + + +2 +00:00:05,269 --> 00:00:05,279 +okay welcome to this session this is + +3 +00:00:05,279 --> 00:00:07,760 +okay welcome to this session this is +called the kinetic bunny need to meet + +4 +00:00:07,760 --> 00:00:07,770 +called the kinetic bunny need to meet + +5 +00:00:07,770 --> 00:00:11,450 +called the kinetic bunny need to meet +the virtual bonnie emilio stop this is + +6 +00:00:11,450 --> 00:00:11,460 +the virtual bonnie emilio stop this is + +7 +00:00:11,460 --> 00:00:15,410 +the virtual bonnie emilio stop this is +jana car who just to give a little bit + +8 +00:00:15,410 --> 00:00:15,420 +jana car who just to give a little bit + +9 +00:00:15,420 --> 00:00:18,859 +jana car who just to give a little bit +of background information we come from a + +10 +00:00:18,859 --> 00:00:18,869 +of background information we come from a + +11 +00:00:18,869 --> 00:00:21,109 +of background information we come from a +company called le code from finland and + diff --git a/wrolpi/captions.py b/wrolpi/captions.py index 97df544c..395b14a2 100755 --- a/wrolpi/captions.py +++ b/wrolpi/captions.py @@ -17,14 +17,17 @@ __all__ = ['read_captions', 'extract_captions'] def get_caption_text(caption_path: Union[str, Path]) -> Generator: """Return all text from each caption of a caption file.""" if str(caption_path).endswith('vtt'): + # VTT for caption in webvtt.read(caption_path): text = str(caption.text).strip() yield text else: + # Finally, try SRT with open(caption_path, 'rt') as fh: contents = fh.read() for subtitle in srt.parse(contents): - yield subtitle.content + if any(subtitle.content): + yield subtitle.content def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator: diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 065f167f..5d0b5163 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -182,6 +182,15 @@ def video_file(test_directory) -> pathlib.Path: yield destination +@pytest.fixture +def corrupted_video_file(test_directory) -> pathlib.Path: + """Return a copy of the corrupted video file in the `test_directory`.""" + destination = test_directory / f'{uuid4()}.mp4' + shutil.copy(PROJECT_DIR / 'test/corrupted.mp4', destination) + + yield destination + + @pytest.fixture def image_file(test_directory) -> pathlib.Path: """Create a small image file in the `test_directory`.""" @@ -193,7 +202,7 @@ def image_file(test_directory) -> pathlib.Path: @pytest.fixture def vtt_file1(test_directory) -> pathlib.Path: """Return a copy of the example1 VTT file in the `test_directory`.""" - destination = test_directory / f'{uuid4()}.jpeg' + destination = test_directory / f'{uuid4()}.en.vtt' shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', destination) yield destination @@ -201,11 +210,27 @@ def vtt_file1(test_directory) -> pathlib.Path: @pytest.fixture def vtt_file2(test_directory) -> pathlib.Path: """Return a copy of the example2 VTT file in the `test_directory`.""" - destination = test_directory / f'{uuid4()}.jpeg' + destination = test_directory / f'{uuid4()}.en.vtt' shutil.copy(PROJECT_DIR / 'test/example2.en.vtt', destination) yield destination +@pytest.fixture +def srt_file3(test_directory) -> pathlib.Path: + """Return a copy of the example3 SRT file in the `test_directory`.""" + destination = test_directory / f'{uuid4()}.en.srt' + shutil.copy(PROJECT_DIR / 'test/example3.en.srt', destination) + yield destination + + +@pytest.fixture +def bad_vtt_file(test_directory) -> pathlib.Path: + """Return a copy of the bad_caption VTT file in the `test_directory`.""" + destination = test_directory / f'{uuid4()}.en.vtt' + shutil.copy(PROJECT_DIR / 'test/bad_caption.en.vtt', destination) + yield destination + + @pytest.fixture def make_files_structure(test_directory) -> Callable[[Union[List, Dict]], List[pathlib.Path]]: def create_files(paths: List) -> List[pathlib.Path]: diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 1ff0f2fe..2d491a43 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -323,7 +323,8 @@ class DownloadManager: name = f'{pid}.{num}' worker_logger = logger.getChild(f'download_worker.{name}') - worker_logger.info(f'Starting up. DownloadManager is disabled={self.disabled.is_set()}.') + disabled = 'is disabled' if self.disabled.is_set() else 'is NOT disabled' + worker_logger.info(f'Starting up. DownloadManager {disabled}.') last_heartbeat = now() while True: diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index 225988b8..78f91f31 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -29,8 +29,6 @@ def test_list_files_api(test_client, make_files_structure, test_directory): assert not response.json.get('errors') results = sorted(response.json['files'], key=lambda i: i['key']) for f1, f2 in zip_longest(results, expected_files): - if f1 is None or f2 is None: - assert f1 == f2 for key in f2.keys(): assert f1[key] == f2[key], f'{f1} != {f2}' diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 6c70ea48..9465d7ca 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -151,7 +151,7 @@ def test_delete_file(make_files_structure, test_directory): ('123foo bar', ['123foo', 'bar']), ] ) -def split_file_name_into_words(path, expected): +def test_split_file_name_into_words(path, expected): assert lib.split_file_name_into_words(Path(path)) == expected diff --git a/wrolpi/media_path.py b/wrolpi/media_path.py index 3add4629..8c492677 100644 --- a/wrolpi/media_path.py +++ b/wrolpi/media_path.py @@ -3,99 +3,27 @@ from typing import Union from sqlalchemy import types -from wrolpi.common import get_media_directory - PATH_TYPE = Union[str, pathlib.Path] -class MediaPath: - """Enforce that a given path is within the Media Directory. - - Provide access to methods and properties on the underlying pathlib.Path. - """ - impl = types.String - - def __init__(self, path: PATH_TYPE): - path = self._validate_path(path) - self._path = path.absolute() - - def __repr__(self): - return f'' - - def __str__(self): - return str(self.relative) - - def __getitem__(self, item): - return self._path - - @classmethod - def _validate_path(cls, path: PATH_TYPE): - if not path: - raise ValueError(f'MediaPath cannot not be empty!') - - path = pathlib.Path(path) if not isinstance(path, pathlib.Path) else path - if not path.is_absolute(): - # Assume the path is relative to the media directory. - path = get_media_directory() / path - elif not str(path).startswith(str(get_media_directory())): - # Absolute paths must be within the media directory. - raise ValueError(f'Path {path} must be in {get_media_directory()}') - - return path - - def __json__(self): - # Always return a relative path to frontend. - return str(self.relative) - - @property - def path(self) -> pathlib.Path: - return self._path - - @path.setter - def path(self, path: PATH_TYPE): - path = self._validate_path(path) - self._path = path.absolute() - - def __eq__(self, other): - return self._path == other - - def __truediv__(self, other): - return self._path / other - - def startswith(self, other): - return str(self._path).startswith(other) - - def endswith(self, other): - return str(self._path).endswith(other) - - @property - def name(self): - return self.path.name - - @property - def relative(self): - """Get the path relative to the media directory.""" - return self._path.relative_to(get_media_directory()) - - def __hash__(self): - return hash(str(self._path)) if self._path else hash(None) - - class MediaPathType(types.TypeDecorator): # noqa impl = types.String def process_bind_param(self, value, dialect): """Convert paths into what the DB expects. (pathlib.Path -> str)""" + if value is None: + return value + if value == '': raise ValueError('MediaPath cannot be empty') if isinstance(value, pathlib.Path): - return str(value.absolute()) - elif isinstance(value, str) or value is None: - return value - elif isinstance(value, MediaPath): - return str(value.path) - raise ValueError(f'Invalid MediaPath ({type(value)}): {value}') + value = str(value.absolute()) + + if not isinstance(value, str): + raise ValueError(f'Invalid MediaPath type ({type(value)}): {value}') + + return value def process_result_value(self, value, dialect): if value: diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 07fa3d20..98b615e7 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -327,8 +327,6 @@ class CustomJSONEncoder(json.JSONEncoder): return obj.__json__() elif isinstance(obj, datetime): return obj.timestamp() - elif isinstance(obj, date): - return datetime(obj.year, obj.month, obj.day).timestamp() elif isinstance(obj, Decimal): return str(obj) elif isinstance(obj, Base): diff --git a/wrolpi/test/test_captions.py b/wrolpi/test/test_captions.py index 664e19a7..f13919ba 100644 --- a/wrolpi/test/test_captions.py +++ b/wrolpi/test/test_captions.py @@ -1,50 +1,56 @@ -import shutil -from pathlib import Path - from wrolpi import captions -from wrolpi.test.common import TestAPI -from wrolpi.vars import PROJECT_DIR -class TestCaption(TestAPI): - - def setUp(self) -> None: - super().setUp() - self.vtt_path1 = Path(self.tmp_dir.name) / 'example1.en.vtt' - self.vtt_path1.touch() - self.vtt_path1.write_text((PROJECT_DIR / 'test/example1.en.vtt').read_text()) - self.vtt_path2 = Path(self.tmp_dir.name) / 'example2.en.vtt' - self.vtt_path2.touch() - self.vtt_path2.write_text((PROJECT_DIR / 'test/example2.en.vtt').read_text()) - - def test_example_en_vtt(self): - expected = ['okay welcome to this session this is', - 'okay welcome to this session this is', - 'okay welcome to this session this is\ncalled the kinetic bunny need to meet', - 'called the kinetic bunny need to meet', - 'called the kinetic bunny need to meet\nthe virtual bonnie emilio stop this is', - 'the virtual bonnie emilio stop this is', - 'the virtual bonnie emilio stop this is\njana car who just to give a little bit', - 'jana car who just to give a little bit', - 'jana car who just to give a little bit\nof background information we come from a', - 'of background information we come from a', - 'of background information we come from a\ncompany called le code from finland and'] - self.assertEqual(list(captions.get_caption_text(self.vtt_path1)), expected) - - expected = ['okay welcome to this session this is', - 'called the kinetic bunny need to meet', - 'the virtual bonnie emilio stop this is', - 'jana car who just to give a little bit', - 'of background information we come from a', - 'company called le code from finland and'] - self.assertEqual(list(captions.get_unique_caption_lines(self.vtt_path1)), expected) - - -def test_bad_caption(test_directory): +def test_bad_caption(bad_vtt_file): """A caption file that cannot be parsed is ignored.""" - bad_caption_file = PROJECT_DIR / 'test/bad_caption.en.vtt' - test_file = test_directory / 'bad_caption.en.vtt' - shutil.copy(bad_caption_file, test_file) - - result = captions.read_captions(test_file) + result = captions.read_captions(bad_vtt_file) assert result is None + + +def test_get_caption_text(vtt_file1, srt_file3): + expected = ['okay welcome to this session this is', + 'okay welcome to this session this is', + 'okay welcome to this session this is\ncalled the kinetic bunny need to meet', + 'called the kinetic bunny need to meet', + 'called the kinetic bunny need to meet\nthe virtual bonnie emilio stop this is', + 'the virtual bonnie emilio stop this is', + 'the virtual bonnie emilio stop this is\njana car who just to give a little bit', + 'jana car who just to give a little bit', + 'jana car who just to give a little bit\nof background information we come from a', + 'of background information we come from a', + 'of background information we come from a\ncompany called le code from finland and'] + assert list(captions.get_caption_text(vtt_file1)) == expected + + expected = ['okay welcome to this session this is', + 'okay welcome to this session this is\ncalled the kinetic bunny need to meet', + 'called the kinetic bunny need to meet', + 'called the kinetic bunny need to meet\n' + 'the virtual bonnie emilio stop this is', + 'the virtual bonnie emilio stop this is', + 'the virtual bonnie emilio stop this is\n' + 'jana car who just to give a little bit', + 'jana car who just to give a little bit', + 'jana car who just to give a little bit\n' + 'of background information we come from a', + 'of background information we come from a', + 'of background information we come from a\n' + 'company called le code from finland and'] + assert list(captions.get_caption_text(srt_file3)) == expected + + +def test_get_unique_caption_lines(vtt_file1, srt_file3): + expected = ['okay welcome to this session this is', + 'called the kinetic bunny need to meet', + 'the virtual bonnie emilio stop this is', + 'jana car who just to give a little bit', + 'of background information we come from a', + 'company called le code from finland and'] + assert list(captions.get_unique_caption_lines(vtt_file1)) == expected + + expected = ['okay welcome to this session this is', + 'called the kinetic bunny need to meet', + 'the virtual bonnie emilio stop this is', + 'jana car who just to give a little bit', + 'of background information we come from a', + 'company called le code from finland and'] + assert list(captions.get_unique_caption_lines(srt_file3)) == expected diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 6ea18d4f..73fd1428 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -333,7 +333,7 @@ def test_truncate_object_bytes(): assert truncate_object_bytes([], 200) == [] assert truncate_object_bytes(None, 100) is None - assert truncate_object_bytes('', 100) is '' + assert truncate_object_bytes('', 100) == '' assert truncate_object_bytes('foo' * 100, 99) == 'foofoofoofoofoofoofoofoofoofoofoofoofoof' assert truncate_object_bytes('foo' * 100, 80) == 'foofoofoofoofoofoofoofoofo' diff --git a/wrolpi/test/test_media_path.py b/wrolpi/test/test_media_path.py index 6c4f0626..7249fe16 100644 --- a/wrolpi/test/test_media_path.py +++ b/wrolpi/test/test_media_path.py @@ -1,50 +1,36 @@ -import json -import pathlib -import tempfile -import unittest -from unittest import mock +import pytest +from sqlalchemy import Column +from sqlalchemy.exc import StatementError -from wrolpi.common import set_test_media_directory -from wrolpi.media_path import MediaPath -from wrolpi.root_api import CustomJSONEncoder +from wrolpi.common import ModelHelper, Base +from wrolpi.media_path import MediaPathType -class TestMediaPath(unittest.TestCase): +class TestTable(Base, ModelHelper): + __tablename__ = 'testtable' + path = Column(MediaPathType, primary_key=True) - def setUp(self): - self.test_dir = tempfile.TemporaryDirectory() - set_test_media_directory(pathlib.Path(self.test_dir.name)) - path = pathlib.Path(self.test_dir.name).absolute() - self.patch = mock.patch('wrolpi.media_path.get_media_directory', lambda: path) - self.patch.start() - def tearDown(self): - set_test_media_directory(None) - self.test_dir.cleanup() - self.patch.stop() +def test_media_path_type(test_session, test_directory): + """MediaPathType checks that paths are valid. They must be in the media directory, absolute, and not empty.""" + # A NULL media path is ok. + foo = TestTable() + test_session.commit() - def test_errors(self): - self.assertRaises(ValueError, MediaPath, '') - self.assertRaises(ValueError, MediaPath, '/') + # The correct media path is in the media directory, and absolute. + foo.path = test_directory / 'foo' + test_session.add(foo) + test_session.commit() - def test_json(self): - p = MediaPath('foo') - result = json.dumps(p, cls=CustomJSONEncoder) - self.assertEqual(result, '"foo"') + # The path cannot be empty. + foo.path = '' + with pytest.raises(StatementError) as error: + test_session.commit() + assert 'cannot be empty' in str(error) + test_session.rollback() - def test_paths(self): - # Paths may be relative to media directory. - d = MediaPath('foo') - self.assertEqual(d._path, pathlib.Path(f'{self.test_dir.name}/foo')) - - # Absolute paths must be in media directory. - d = MediaPath(f'{self.test_dir.name}/foo') - self.assertEqual(d._path, pathlib.Path(f'{self.test_dir.name}/foo')) - - # Error is raised when the path is not in the media path. - self.assertRaises(ValueError, MediaPath, '/tmp/foo') - - def test_repr(self): - d = MediaPath('foo') - assert self.test_dir.name not in repr(d) - assert self.test_dir.name not in str(d) + # Paths must be strings or Path. + foo.path = 1 + with pytest.raises(StatementError) as error: + test_session.commit() + assert 'Invalid MediaPath type' in str(error)