mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
* Squashed commit of the following:
commit c527475fcb4d4906457be5c10f0e9c6dee7fedc1
Author: Roland <roland@learningselfreliance.com>
Date: Tue Dec 14 22:16:36 2021 -0700
Only saving a favorite video to the config if it has a source_id.
commit ae8b712ebeaef467b8d5a5b87ca0f4f276373a0d
Author: Roland <roland@learningselfreliance.com>
Date: Tue Dec 14 22:14:26 2021 -0700
Ading description and changing Save button to Download.
commit 03a72be6dfdfa383915950a80e3f1ee9febf6705
Author: Roland <roland@learningselfreliance.com>
Date: Tue Dec 14 22:09:44 2021 -0700
Removing uploader for now.
commit 5d0231284bf84e459687f6ab903b06bf4e3762bd
Author: Roland <roland@learningselfreliance.com>
Date: Tue Dec 14 22:07:17 2021 -0700
Removing unique data from example info.
commit 9cc721a32d4715d9d6433e51db3b35e039c78d68
Author: Roland <roland@learningselfreliance.com>
Date: Tue Dec 14 22:02:33 2021 -0700
Updating downloader tests.
commit 8e193b03349ae4613f105418cd3e8176c7101304
Author: Roland <roland@learningselfreliance.com>
Date: Tue Dec 14 21:53:31 2021 -0700
Toasting error when WROL Mode is enabled. Disabling new downloads during WROL Mode.
commit 48c4311c31db8f2d2c8dd0a65f979a5f0a49a911
Author: Roland <roland@learningselfreliance.com>
Date: Tue Dec 14 21:45:06 2021 -0700
Raising error if download exits without a 0. Clearing kill event when starting a new download.
commit 54ba3a890d59facc837410d34bf7afb40db3b685
Author: Roland <roland@learningselfreliance.com>
Date: Tue Dec 14 21:16:46 2021 -0700
Downloads can be restarted and killed.
commit bd60751262247259c3e9ceb5116e017d6314c219
Author: Roland <roland@learningselfreliance.com>
Date: Mon Dec 13 16:28:18 2021 -0700
Downloads can be started again. Downloads are sorted on Downloads page.
commit ea7f39231b92982ae0b263b76c5bd1902cc8169d
Author: Roland <roland@learningselfreliance.com>
Date: Sat Dec 11 21:24:16 2021 -0700
Building Downloads tab.
commit d3321872ac3743ff6df781b07ef0fb537f1d4c98
Author: Roland <roland@learningselfreliance.com>
Date: Sat Dec 11 20:41:46 2021 -0700
Not scheduling channels withtout a URL/frequency.
commit 84f268a0d4f58c85f86b6377900492e0766f6619
Author: Roland <roland@learningselfreliance.com>
Date: Sat Dec 11 19:56:08 2021 -0700
Channel downloads will be spread out when each channel is created.
commit 26fa6beea38ef3f8d6b7bf218f18222a51a87a60
Author: Roland <roland@learningselfreliance.com>
Date: Sat Dec 11 13:08:11 2021 -0700
Recurring downloads are renewed on startup.
commit fc2ab850533f1c3ad073f4692c7dea2be43e85d6
Author: Roland <roland@learningselfreliance.com>
Date: Fri Dec 10 17:03:57 2021 -0700
Fetching download again after do_download.
commit fdadad327fc09c17a49eb63b11e632fb72cc3195
Author: Roland <roland@learningselfreliance.com>
Date: Fri Dec 10 16:20:51 2021 -0700
Fixing error with test. Mock was not being reset.
commit 070dee238e53377acd22097c63cf3794e68ed423
Author: Roland <roland@learningselfreliance.com>
Date: Fri Dec 10 15:36:39 2021 -0700
Fixed bug in date handling (big surprise).
commit e53a8992268d68620c679455adaec3a590f20602
Author: Roland <roland@learningselfreliance.com>
Date: Wed Dec 1 16:48:36 2021 -0700
Using chunks for regular channel refresh.
commit 3bd60c58d1bef0226325f321978c0434b36b3be5
Author: Roland <roland@learningselfreliance.com>
Date: Wed Dec 1 15:22:02 2021 -0700
Reformatting.
commit 4fdc325f25a32c311ca6543fe321624b2bd564b2
Author: Roland <roland@learningselfreliance.com>
Date: Wed Dec 1 15:13:34 2021 -0700
Creating function that refreshes NO CHANNEL videos.
commit d67feafb6c2994f5a4a1295920a70db369cc6816
Author: Roland <roland@learningselfreliance.com>
Date: Wed Dec 1 14:35:54 2021 -0700
Removing unused functions.
commit 6b3918e4e78fbac64920f85c4de0df873ec366a1
Author: Roland <roland@learningselfreliance.com>
Date: Wed Dec 1 14:31:48 2021 -0700
Removing reporter and events. This will be replaced in the future with some sort of SSE.
commit 54fa1b0f5e93ca25c80b780dcdc3876b700273b1
Author: Roland <roland@learningselfreliance.com>
Date: Sun Nov 28 17:55:43 2021 -0700
Better error handling for archive button.
commit c31d124e511b27f626ed2df71486d47302b396f3
Author: Roland <roland@learningselfreliance.com>
Date: Sun Nov 28 17:29:43 2021 -0700
Displaying View Count in a human readable format.
commit 0ee2bd65588fd93b475cad6144e76f1b45191b39
Author: Roland <roland@learningselfreliance.com>
Date: Sat Nov 27 09:29:41 2021 -0700
Fixing tests.
commit e64aff32bab95c311ea3747e1a21b58b99750d5e
Author: Roland <roland@learningselfreliance.com>
Date: Sat Nov 27 08:59:51 2021 -0700
Handling when a Video has no Channel in UI.
commit b9549ecd5aac03c002adca7b7667f0a7189b662a
Author: Roland <roland@learningselfreliance.com>
Date: Fri Nov 26 08:35:36 2021 -0700
Downloaders can raise UnrecoverableDownloadError to signal a downlaod should not be retried.
commit da88798e2584d3e84900a70e34166c8c208c64fb
Author: Roland <roland@learningselfreliance.com>
Date: Wed Nov 24 20:40:29 2021 -0700
Downloaders can refuse a download, which will then be deleted.
commit b865d856121db5ec8a3660c5d3c13b9846fde688
Author: Roland <roland@learningselfreliance.com>
Date: Wed Nov 24 19:35:43 2021 -0700
Removing relative path handling.
commit 6d7757870b02ba030d78b3ef7e43f28b906eea61
Author: Roland <roland@learningselfreliance.com>
Date: Wed Nov 24 19:28:27 2021 -0700
Using __json__ for Video and Channel models.
commit 649e4d6a61608272759d1e7ee02b490bb5c20b3e
Author: Roland <roland@learningselfreliance.com>
Date: Wed Nov 24 18:23:38 2021 -0700
Fixing video links in UI.
commit 9d081af12742070b8519bd752812130cb5dbe393
Author: Roland <roland@learningselfreliance.com>
Date: Wed Nov 24 14:42:38 2021 -0700
Fixing docs.
commit 1202cbe0f9ae744d7801b53a5071cded8ea39924
Author: Roland <roland@learningselfreliance.com>
Date: Wed Nov 24 14:40:36 2021 -0700
Fixing wrolpi tests.
commit 3f6af8cc1123f2f63a3fc0f21ec4d994c642cf13
Author: Roland <roland@learningselfreliance.com>
Date: Wed Nov 24 14:13:35 2021 -0700
Fixing tests to use MediaPath.
commit faec04f2a5e4c267a502c03884940a9c5bcfe387
Author: Roland <roland@learningselfreliance.com>
Date: Fri Nov 19 10:21:20 2021 -0700
Testing when a video has no channel.
commit cbf22aad7a0b72e61685e0236e843554b57c4af1
Author: Roland <roland@learningselfreliance.com>
Date: Thu Nov 18 16:24:03 2021 -0700
Using MediaPath for Video and Channel models. Testing VideoDownloader.do_download.
commit 7916ed61e43439e6cd60f23ce3fdf6c1a75bf02d
Author: Roland <roland@learningselfreliance.com>
Date: Thu Nov 18 12:43:44 2021 -0700
Channels/Videos downloaders can get gotten.
commit 808ef796cec01db6e7009065f9e26d25d9500ce2
Author: Roland <roland@learningselfreliance.com>
Date: Wed Nov 17 21:45:19 2021 -0700
Upgrading to Sanic 21.9.1.
commit ea32813be28e78393ac456ee782a7245c098188f
Author: Roland <roland@learningselfreliance.com>
Date: Wed Nov 17 20:29:01 2021 -0700
Creating Downloader and DownloaderManager classes which handle downloading URLs.
commit e859a77a77277ddf8b6e221ec9c9383c8ab91d66
Author: Roland <roland@learningselfreliance.com>
Date: Wed Nov 17 12:41:39 2021 -0700
Creating UI segement which allows the download/upload of media.
* Displaying all recurring downloads.
* Not downloading channels with empty url.
* Showing positive row for recurring downloads.
* Removing default last_successful_download.
* Only displaying Last Successful Download when there is one.
* Refreshing videos before finding missing videos.
* Using short video path name for log.
* Removing download history feature in favor of new Downloads page.
* Only showing info date if it exists.
* Making get_or_create_channel more reliable.
* Allowing null for channel download frequency.
* Logging when searching for channel.
* Storing channel's source_id in config.
* Importing using link instead of name.
* Getting channel id when importing channels.
* Removing redunant parentheses from repr.
* Preserving database channel values if there are not values in the config.
* Only highlighting pending downloads.
* Deleting old downloads periodically.
* Sorting recurring by next_download/frequency.
* Creating fixtures for channels, media directory and db session.
* Removing unused function. Wrapping config for tests.
* Deleting channel download when deleting a channel.
* Testing that a channel's download is deleted when the channel is deleted.
* Updating youtube-dl
* Switching from youtube-dl in favor of yt-dlp.
* Sorting once-downlaods by last_successful_download then id.
* Adding special handling to video_url_resolver for full urls.
* Consolidating downloader methods and functions. Reversing once-download sort order so the most recently downloaded are at the top.
* Getting session if one is not provided.
* Adding test for create_downloads.
* Not passing format option when getting filename.
* Linting
Signed-off-by: Roland <roland@learningselfreliance.com>
219 lines
6.6 KiB
Python
219 lines
6.6 KiB
Python
import json
|
|
import pathlib
|
|
import tempfile
|
|
import unittest
|
|
from contextlib import contextmanager
|
|
from functools import partialmethod
|
|
from http import HTTPStatus
|
|
from itertools import zip_longest
|
|
from shutil import copyfile
|
|
from typing import List, Optional
|
|
|
|
import mock
|
|
import websockets
|
|
import yaml
|
|
from sanic_openapi.api import Response
|
|
|
|
from wrolpi.common import EXAMPLE_CONFIG_PATH, get_config, set_test_media_directory, get_media_directory
|
|
from wrolpi.conftest import ROUTES_ATTACHED, test_db, test_client # noqa
|
|
from wrolpi.db import postgres_engine
|
|
from wrolpi.vars import PROJECT_DIR
|
|
|
|
TEST_CONFIG_PATH = tempfile.NamedTemporaryFile(mode='rt', delete=False)
|
|
|
|
|
|
def wrap_test_db(func):
|
|
"""
|
|
Wrap a test so that when calling wrolpi.common.get_db, it returns a testing database cloned from the api
|
|
template.
|
|
"""
|
|
|
|
def wrapped(*a, **kw):
|
|
test_engine, session = test_db()
|
|
|
|
def fake_get_db_session():
|
|
"""Get the testing db"""
|
|
return test_engine, session
|
|
|
|
try:
|
|
with mock.patch('wrolpi.db._get_db_session', fake_get_db_session):
|
|
# Run the test.
|
|
result = func(*a, **kw)
|
|
return result
|
|
finally:
|
|
session.rollback()
|
|
session.close()
|
|
test_engine.dispose()
|
|
conn = postgres_engine.connect()
|
|
conn.execute(f'DROP DATABASE IF EXISTS {test_engine.engine.url.database}')
|
|
|
|
return wrapped
|
|
|
|
|
|
class PytestCase:
|
|
"""
|
|
Replicate unittest.TestCase methods to bridge the gap between it and Pytest.
|
|
"""
|
|
|
|
@staticmethod
|
|
def assertGreater(a, b, msg: str = None): # noqa
|
|
assert a > b, msg
|
|
|
|
@staticmethod
|
|
def assertLess(a, b, msg: str = None): # noqa
|
|
assert a < b, msg
|
|
|
|
@staticmethod
|
|
def assertEqual(a, b, msg: str = None): # noqa
|
|
assert a == b, msg or f'{a} != {b}'
|
|
|
|
@staticmethod
|
|
def assertRaises(exception, func, *args, **kwargs): # noqa
|
|
try:
|
|
func(*args, **kwargs)
|
|
except exception:
|
|
pass
|
|
|
|
@classmethod
|
|
def assertDictContains(cls, d1: dict, d2: dict): # noqa
|
|
if hasattr(d1, '__dict__'):
|
|
d1 = d1.__dict__
|
|
if hasattr(d2, '__dict__'):
|
|
d2 = d2.__dict__
|
|
|
|
for k2 in d2.keys():
|
|
assert d1, f'dict 1 is empty: {d1}'
|
|
assert d2, f'dict 1 is empty: {d2}'
|
|
assert k2 in d1, f'dict 1 does not contain {k2}'
|
|
if isinstance(d1[k2], dict):
|
|
cls.assertDictContains(d1[k2], d2[k2])
|
|
else:
|
|
assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1'
|
|
|
|
def assertError(self, response, http_status: int, code=None): # noqa
|
|
self.assertEqual(response.status_code, http_status)
|
|
if code:
|
|
self.assertEqual(response.json['code'], code)
|
|
|
|
@staticmethod
|
|
def assertTruth(value, expected): # noqa
|
|
"""
|
|
Check that a value is Truthy or Falsy.
|
|
"""
|
|
if expected is True:
|
|
assert value, f'Value {value} should have been truthy'
|
|
else:
|
|
assert not value, f'Value {value} should have been falsey'
|
|
|
|
assertTruthy = partialmethod(assertTruth, expected=True)
|
|
assertFalsey = partialmethod(assertTruth, expected=False)
|
|
|
|
def assertItemsTruthyOrFalsey(self, items_list: List, expected_list: List):
|
|
for d1, d2 in zip_longest(items_list, expected_list):
|
|
for d2_key in d2:
|
|
if d1 is None:
|
|
raise ValueError('d1 is None')
|
|
if d2 is None:
|
|
raise ValueError('d2 is None')
|
|
self.assertTruth(d1[d2_key], d2[d2_key])
|
|
|
|
@staticmethod
|
|
def assertLength(a, b): # noqa
|
|
assert len(a) == len(b), f'{len(a)=} != {len(b)=}'
|
|
|
|
|
|
class ExtendedTestCase(PytestCase, unittest.TestCase):
|
|
"""
|
|
Add any specialized test methods to this class.
|
|
"""
|
|
pass
|
|
|
|
|
|
class TestAPI(ExtendedTestCase):
|
|
|
|
def setUp(self) -> None:
|
|
self.config_path_patch = mock.patch('wrolpi.vars.CONFIG_PATH', TEST_CONFIG_PATH.name)
|
|
self.config_path_patch.start()
|
|
self.tmp_dir = tempfile.TemporaryDirectory()
|
|
self.tmp_path = pathlib.Path(self.tmp_dir.name)
|
|
set_test_media_directory(self.tmp_dir.name)
|
|
# Copy the example config to test against
|
|
copyfile(str(EXAMPLE_CONFIG_PATH), TEST_CONFIG_PATH.name)
|
|
# Setup the testing video root directory
|
|
config = get_config()
|
|
config['media_directory'] = str(PROJECT_DIR / 'test')
|
|
with open(TEST_CONFIG_PATH.name, 'wt') as fh:
|
|
fh.write(yaml.dump(config))
|
|
|
|
def tearDown(self) -> None:
|
|
self.config_path_patch.stop()
|
|
set_test_media_directory(None)
|
|
|
|
def assertHTTPStatus(self, response: Response, status: int):
|
|
self.assertEqual(response.status_code, status)
|
|
|
|
assertOK = partialmethod(assertHTTPStatus, status=HTTPStatus.OK)
|
|
assertCONFLICT = partialmethod(assertHTTPStatus, status=HTTPStatus.CONFLICT)
|
|
assertNO_CONTENT = partialmethod(assertHTTPStatus, status=HTTPStatus.NO_CONTENT)
|
|
|
|
|
|
@contextmanager
|
|
def wrap_media_directory(path: Optional[pathlib.Path] = None):
|
|
cleanup = False
|
|
if not path:
|
|
path = tempfile.TemporaryDirectory()
|
|
cleanup = True
|
|
|
|
set_test_media_directory(path.name)
|
|
try:
|
|
yield
|
|
finally:
|
|
set_test_media_directory(None)
|
|
if cleanup:
|
|
path.cleanup()
|
|
|
|
|
|
@contextmanager
|
|
def build_test_directories(paths: List[str], tmp_dir: pathlib.Path = None) -> pathlib.Path:
|
|
"""
|
|
Create directories based on the provided structure.
|
|
|
|
Example:
|
|
>>> build_test_directories([
|
|
'channel1/vid1.mp4',
|
|
'channel2/vid1.mp4',
|
|
'channel2/vid2.mp4',
|
|
'channel2/vid2.en.vtt'
|
|
])
|
|
|
|
Creates directories like so:
|
|
channel1/vid1.mp4
|
|
channel2/vid1.mp4
|
|
channel2/vid2.mp4
|
|
channel2/vid2.en.vtt
|
|
"""
|
|
tmp_dir = tmp_dir or get_media_directory()
|
|
|
|
directories = filter(lambda i: i.endswith('/'), paths)
|
|
for directory in directories:
|
|
(tmp_dir / directory).mkdir(parents=True)
|
|
|
|
files = filter(lambda i: not i.endswith('/'), paths)
|
|
for file in files:
|
|
file = tmp_dir / file
|
|
parents = file.parents
|
|
parents[0].mkdir(parents=True, exist_ok=True)
|
|
(tmp_dir / file).touch()
|
|
|
|
yield tmp_dir.absolute()
|
|
|
|
|
|
async def get_all_ws_messages(ws) -> List[dict]:
|
|
messages = []
|
|
while True:
|
|
try:
|
|
message = await ws.recv()
|
|
except websockets.exceptions.ConnectionClosedOK:
|
|
break
|
|
messages.append(json.loads(message))
|
|
return messages
|