mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
d95249218b
Adds two optional DownloadManager settings: - download_daily_limit_per_domain: max non-recurring downloads per domain per day - download_daily_limit_global: max non-recurring downloads across all domains per day Recurring Channel/RSS catalog updates are never gated and never counted; only the videos/archives they spawn and one-off downloads count. Both successful and failed attempts consume a slot (counted once per download per day). Limits reset at local midnight (configured timezone). Over-limit downloads stay "new" and resume the next day. Implementation: - New Download.last_download_attempt column (+ index, migration) set at dispatch time so the count survives restarts and is correct across workers. - dispatch_downloads gates non-recurring downloads: per-domain skips that domain, global stops dispatching counted downloads for the cycle. daily_download_counts derives the day's totals from the DB. - Domain grouping uses normalized URLs via a new Downloader.normalize_domain method (base strips www/m/mobile + port); VideoDownloader/ChannelDownloader override it to collapse youtu.be, /shorts/, and tracking params to youtube.com. ArchiveDownloader is the seam for future archive URL normalization. - Settings wired through WROLPiConfig, the /settings API (GET/PATCH, non-negative validation, 0 = unlimited), and the React Settings page. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
940 lines
42 KiB
Python
940 lines
42 KiB
Python
import json
|
|
from http import HTTPStatus
|
|
from itertools import zip_longest
|
|
|
|
import pytest
|
|
from mock import mock
|
|
from sanic import Request
|
|
|
|
from wrolpi.api_utils import json_error_handler
|
|
from wrolpi.common import get_wrolpi_config
|
|
from wrolpi.downloader import Download, get_download_manager_config
|
|
from wrolpi.errors import ValidationError, SearchEmpty
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_index(test_session, async_client):
|
|
"""
|
|
Index should have some details in an HTML response
|
|
"""
|
|
request, response = await async_client.get('/')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert b'html' in response.body
|
|
|
|
request, response = await async_client.get('/api')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert b'html' in response.body
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_valid_regex(test_session, async_client):
|
|
"""
|
|
The endpoint should only return valid if the regex is valid.
|
|
"""
|
|
data = {'regex': 'foo'}
|
|
request, response = await async_client.post('/api/valid_regex', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert json.loads(response.body) == {'valid': True, 'regex': 'foo'}
|
|
|
|
data = {'regex': '.*(title match).*'}
|
|
request, response = await async_client.post('/api/valid_regex', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert json.loads(response.body) == {'valid': True, 'regex': '.*(title match).*'}
|
|
|
|
data = {'regex': '.*(missing parenthesis.*'}
|
|
request, response = await async_client.post('/api/valid_regex', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert json.loads(response.body) == {'valid': False, 'regex': '.*(missing parenthesis.*'}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_settings(test_session, async_client):
|
|
get_wrolpi_config().ignored_directories = list()
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.status_code == HTTPStatus.OK
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_daily_download_limit_settings(test_session, async_client):
|
|
"""The daily download limits round-trip through the settings API and reject negative values."""
|
|
# Defaults are unlimited (None).
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.json['download_daily_limit_global'] is None
|
|
assert response.json['download_daily_limit_per_domain'] is None
|
|
|
|
data = {'download_daily_limit_global': 100, 'download_daily_limit_per_domain': 50}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.json['download_daily_limit_global'] == 100
|
|
assert response.json['download_daily_limit_per_domain'] == 50
|
|
assert get_wrolpi_config().download_daily_limit_global == 100
|
|
assert get_wrolpi_config().download_daily_limit_per_domain == 50
|
|
|
|
# 0 means unlimited and is accepted.
|
|
data = {'download_daily_limit_per_domain': 0}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
# Negative values are rejected.
|
|
data = {'download_daily_limit_global': -1}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_log_level(async_client):
|
|
"""Log level must be between 0 and 40."""
|
|
data = {'log_level': 0}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
data = {'log_level': -1}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
data = {'log_level': 40}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
data = {'log_level': 41}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
data = {'log_level': None}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
data = {}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_downloads(test_session, async_client):
|
|
request, response = await async_client.get('/api/download')
|
|
assert_dict_contains(response.json, {'once_downloads': [], 'recurring_downloads': []})
|
|
|
|
d1 = Download(url='https://example.com/1', status='complete')
|
|
d2 = Download(url='https://example.com/2', status='pending')
|
|
d3 = Download(url='https://example.com/3', status='deferred')
|
|
d4 = Download(url='https://example.com/4', status='deferred')
|
|
d5 = Download(url='https://example.com/5', status='new')
|
|
d6 = Download(url='https://example.com/6', status='deferred', frequency=60)
|
|
d7 = Download(url='https://example.com/7', status='pending', frequency=60)
|
|
test_session.add_all([d1, d2, d3, d4, d5, d6, d7])
|
|
test_session.commit()
|
|
|
|
request, response = await async_client.get('/api/download')
|
|
expected = [
|
|
{'url': 'https://example.com/2', 'status': 'pending'},
|
|
{'url': 'https://example.com/5', 'status': 'new'},
|
|
{'url': 'https://example.com/3', 'status': 'deferred'},
|
|
{'url': 'https://example.com/4', 'status': 'deferred'},
|
|
{'url': 'https://example.com/1', 'status': 'complete'},
|
|
]
|
|
for download, expected in zip_longest(response.json['once_downloads'], expected):
|
|
assert_dict_contains(download, expected)
|
|
expected = [
|
|
{'url': 'https://example.com/7', 'status': 'pending'},
|
|
{'url': 'https://example.com/6', 'status': 'deferred'},
|
|
]
|
|
for download, expected in zip_longest(response.json['recurring_downloads'], expected):
|
|
assert_dict_contains(download, expected)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_downloads_sorter_recurring(async_client, test_session):
|
|
downloads = [
|
|
dict(status='complete', frequency=1),
|
|
dict(status='complete', frequency=2, next_download='2020-01-01T00:00:01+00:00'),
|
|
dict(status='complete', frequency=2, next_download='2020-01-01T00:00:02+00:00'),
|
|
dict(status='pending', frequency=1),
|
|
dict(status='pending', frequency=4),
|
|
dict(status='failed', frequency=1),
|
|
dict(status='failed', frequency=1),
|
|
]
|
|
for idx, download in enumerate(downloads):
|
|
test_session.add(Download(url=f'https://example.com/{idx}', **download))
|
|
test_session.commit()
|
|
|
|
expected = [
|
|
dict(status='pending', frequency=1),
|
|
dict(status='pending', frequency=4),
|
|
dict(status='failed', frequency=1),
|
|
dict(status='failed'),
|
|
dict(status='complete', frequency=2, next_download='2020-01-01T00:00:01+00:00'),
|
|
dict(status='complete', frequency=2, next_download='2020-01-01T00:00:02+00:00'),
|
|
dict(status='complete', frequency=1),
|
|
]
|
|
request, response = await async_client.get('/api/download')
|
|
recurring_downloads = response.json['recurring_downloads']
|
|
for d1, d2 in zip_longest(expected, recurring_downloads):
|
|
assert_dict_contains(d2, d1)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_downloads_sorter(async_client, test_session):
|
|
downloads = [
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:01+00:00'),
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:03+00:00'),
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:02+00:00'),
|
|
# last_successful_download are equal, so id is used next.
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:01+00:00', url='1'),
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:01+00:00', url='2'),
|
|
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:04+00:00'),
|
|
dict(status='failed'),
|
|
dict(status='failed', last_successful_download='2020-01-01T00:00:01+00:00'),
|
|
]
|
|
for idx, download in enumerate(downloads):
|
|
test_session.add(Download(url=download.pop('url', f'https://example.com/{idx}'), **download))
|
|
test_session.commit()
|
|
|
|
expected = [
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:04+00:00'),
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:01+00:00', url='1'),
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:01+00:00', url='2'),
|
|
dict(status='failed'),
|
|
dict(status='failed', last_successful_download='2020-01-01T00:00:01+00:00'),
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:03+00:00'),
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:02+00:00'),
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:01+00:00'),
|
|
]
|
|
request, response = await async_client.get('/api/download')
|
|
once_downloads = response.json['once_downloads']
|
|
for d1, d2 in zip_longest(expected, once_downloads):
|
|
assert_dict_contains(d2, d1)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_echo(async_client):
|
|
"""
|
|
Echo should send back a JSON object with our request details.
|
|
"""
|
|
request, response = await async_client.get('/api/echo?hello=world')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert_dict_contains(response.json['form'], {})
|
|
assert response.json['method'] == 'GET', 'Method was not GET'
|
|
assert response.json['json'] is None, 'JSON was not empty'
|
|
assert response.json['args'] == {'hello': ['world']}
|
|
|
|
data = {'foo': 'bar'}
|
|
request, response = await async_client.post('/api/echo', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert_dict_contains(response.json['form'], {})
|
|
assert response.json['method'] == 'POST', 'Method was not POST'
|
|
assert response.json['json'] == data, 'JSON was not data'
|
|
assert response.json['args'] == {}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_hotspot_settings(test_session, async_client, test_wrolpi_config):
|
|
"""
|
|
The User can configure hotspot settings (ssid, password, device) via /settings.
|
|
Hotspot control (on/off) is now done through Controller endpoints.
|
|
"""
|
|
config = get_wrolpi_config()
|
|
assert config.hotspot_on_startup is True
|
|
|
|
# Hotspot password can be changed.
|
|
content = {'hotspot_password': 'new password', 'hotspot_ssid': 'new ssid'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert config.hotspot_password == 'new password'
|
|
assert config.hotspot_ssid == 'new ssid'
|
|
|
|
# Hotspot password must be at least 8 characters.
|
|
content = {'hotspot_password': '1234567', 'hotspot_ssid': 'new ssid'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json['code'] == 'INVALID_CONFIG'
|
|
assert '8 characters' in response.json['error']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clear_downloads(test_session, async_client, test_wrolpi_config, test_download_manager_config):
|
|
d1 = Download(url='https://example.com/1', status='complete')
|
|
d2 = Download(url='https://example.com/2', status='pending')
|
|
d3 = Download(url='https://example.com/3', status='deferred')
|
|
d4 = Download(url='https://example.com/4', status='new')
|
|
d5 = Download(url='https://example.com/5', status='failed')
|
|
d6 = Download(url='https://example.com/6', status='failed', frequency=60)
|
|
d7 = Download(url='https://example.com/7', status='complete', frequency=60)
|
|
test_session.add_all([d1, d2, d3, d4, d5, d6, d7])
|
|
test_session.commit()
|
|
|
|
def check_downloads(response_, once_downloads_, recurring_downloads_, status_code=None):
|
|
if status_code:
|
|
assert response_.status_code == status_code
|
|
|
|
for download, once_download in zip_longest(response_.json['once_downloads'], once_downloads_):
|
|
assert download
|
|
assert once_download
|
|
assert download['url'] == once_download['url']
|
|
assert download['status'] == once_download['status']
|
|
for download, recurring_download in zip_longest(response_.json['recurring_downloads'], recurring_downloads_):
|
|
assert download
|
|
assert recurring_download
|
|
assert download['url'] == recurring_download['url']
|
|
assert download['status'] == recurring_download['status']
|
|
|
|
# All created downloads are present.
|
|
request, response = await async_client.get('/api/download')
|
|
once_downloads = [
|
|
{'url': 'https://example.com/2', 'status': 'pending'},
|
|
{'url': 'https://example.com/5', 'status': 'failed'},
|
|
{'url': 'https://example.com/4', 'status': 'new'},
|
|
{'url': 'https://example.com/3', 'status': 'deferred'},
|
|
{'url': 'https://example.com/1', 'status': 'complete'},
|
|
]
|
|
recurring_downloads = [
|
|
{'url': 'https://example.com/6', 'status': 'failed'},
|
|
{'url': 'https://example.com/7', 'status': 'complete'},
|
|
]
|
|
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
|
|
|
|
# Once "complete" download is removed.
|
|
request, response = await async_client.post('/api/download/clear_completed')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
request, response = await async_client.get('/api/download')
|
|
once_downloads = [
|
|
{'url': 'https://example.com/2', 'status': 'pending'},
|
|
{'url': 'https://example.com/5', 'status': 'failed'},
|
|
{'url': 'https://example.com/4', 'status': 'new'},
|
|
{'url': 'https://example.com/3', 'status': 'deferred'},
|
|
]
|
|
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
|
|
|
|
# Failed "once" download is removed, recurring failed is not removed.
|
|
request, response = await async_client.post('/api/download/clear_failed')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
request, response = await async_client.get('/api/download')
|
|
once_downloads = [
|
|
{'url': 'https://example.com/2', 'status': 'pending'},
|
|
{'url': 'https://example.com/4', 'status': 'new'},
|
|
{'url': 'https://example.com/3', 'status': 'deferred'},
|
|
]
|
|
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
|
|
|
|
# Failed once-downloads will not be downloaded again.
|
|
assert get_download_manager_config().skip_urls == ['https://example.com/5', ]
|
|
|
|
# Downloads can be retried.
|
|
request, response = await async_client.post('/api/download/retry_once')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
request, response = await async_client.get('/api/download')
|
|
once_downloads = [
|
|
{'url': 'https://example.com/2', 'status': 'new'},
|
|
{'url': 'https://example.com/3', 'status': 'new'},
|
|
{'url': 'https://example.com/4', 'status': 'new'},
|
|
]
|
|
recurring_downloads_after_retry = [
|
|
{'url': 'https://example.com/6', 'status': 'new'},
|
|
{'url': 'https://example.com/7', 'status': 'complete'},
|
|
]
|
|
check_downloads(response, once_downloads, recurring_downloads_after_retry, status_code=HTTPStatus.OK)
|
|
|
|
# "Delete All" button deletes all once-downloads.
|
|
request, response = await async_client.post('/api/download/delete_once')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
request, response = await async_client.get('/api/download')
|
|
check_downloads(response, [], recurring_downloads_after_retry, status_code=HTTPStatus.OK)
|
|
|
|
# Failed once-downloads will not be downloaded again.
|
|
assert get_download_manager_config().skip_urls == ['https://example.com/5', ]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_downloads(test_session, async_client, test_wrolpi_config, assert_downloads):
|
|
"""Downloads that are pending or deferred can be retried, and their attempts reset."""
|
|
d1 = Download(url='https://example.com/1', status='complete', attempts=1)
|
|
d2 = Download(url='https://example.com/2', status='pending', attempts=1)
|
|
d3 = Download(url='https://example.com/3', status='deferred', attempts=1)
|
|
d4 = Download(url='https://example.com/4', status='new', attempts=1)
|
|
d5 = Download(url='https://example.com/5', status='failed', attempts=1)
|
|
d6 = Download(url='https://example.com/6', status='failed', frequency=60, attempts=1)
|
|
d7 = Download(url='https://example.com/7', status='complete', frequency=60, attempts=1)
|
|
test_session.add_all([d1, d2, d3, d4, d5, d6, d7])
|
|
test_session.commit()
|
|
|
|
# Retry all once-downloads.
|
|
request, response = await async_client.post('/api/download/retry_once')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
assert_downloads([
|
|
dict(url='https://example.com/1', status='complete', attempts=1), # complete is not retried.
|
|
dict(url='https://example.com/2', status='new', attempts=0), # pending is retried.
|
|
dict(url='https://example.com/3', status='new', attempts=0), # deferred is retried.
|
|
dict(url='https://example.com/4', status='new', attempts=1), # new does not need to be retried.
|
|
dict(url='https://example.com/5', status='new', attempts=0), # failed is retried.
|
|
dict(url='https://example.com/6', status='new', frequency=60, attempts=0), # recurring failed is retried.
|
|
dict(url='https://example.com/7', status='complete', frequency=60, attempts=1), # complete recurring is unchanged.
|
|
])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_status(async_client, test_session):
|
|
"""Get the server status information."""
|
|
request, response = await async_client.get('/api/status')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert isinstance(response.json.get('cpu_stats'), dict), 'cpu_stats should be a dict'
|
|
assert isinstance(response.json.get('load_stats'), dict), 'load_stats should be a dict'
|
|
assert isinstance(response.json.get('drives_stats'), list), 'drive_stats should be a dict'
|
|
assert isinstance(response.json.get('downloads'), dict), 'downloads should be a dict'
|
|
assert isinstance(response.json.get('version'), str), 'version should be a str'
|
|
assert isinstance(response.json.get('memory_stats'), dict), 'memory_stats should be a dict'
|
|
assert isinstance(response.json.get('flags'), dict), 'flags should be a dict'
|
|
assert isinstance(response.json.get('sanic_workers'), dict), 'Sanic worker status should be a dict'
|
|
assert isinstance(response.json.get('local_time'), str), 'local_time should be a str'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_status_local_time_with_timezone(async_client, test_session, test_wrolpi_config):
|
|
"""Status endpoint includes local_time using configured timezone."""
|
|
from wrolpi.common import get_wrolpi_config
|
|
config = get_wrolpi_config()
|
|
config.timezone = 'America/Denver'
|
|
|
|
request, response = await async_client.get('/api/status')
|
|
assert response.status_code == HTTPStatus.OK
|
|
local_time = response.json['local_time']
|
|
# ISO 8601 with timezone offset for Denver (UTC-7 or UTC-6).
|
|
assert '+' in local_time or '-' in local_time[10:], 'local_time should include timezone offset'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_crud(test_session, async_client, test_download_manager_config, tag_factory, test_directory):
|
|
"""Test creating once-downloads and recurring downloads."""
|
|
tag1, tag2 = await tag_factory(), await tag_factory()
|
|
|
|
async def dispatch_downloads(*a, **kw):
|
|
pass
|
|
|
|
with mock.patch('wrolpi.downloader.DownloadManager.dispatch_downloads', dispatch_downloads):
|
|
# Create a single recurring Download.
|
|
body = dict(
|
|
urls=['https://example.com/1', ],
|
|
frequency=1_000,
|
|
downloader='archive',
|
|
destination='archives/example.com',
|
|
tag_names=None, # tag_names can be a None.
|
|
)
|
|
request, response = await async_client.post('/api/download', content=json.dumps(body))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
assert {i.url for i in test_session.query(Download)} == {'https://example.com/1', }
|
|
# Assert that the config has been saved
|
|
config = get_download_manager_config()
|
|
assert len(config.downloads) == 1
|
|
assert config.downloads[0]['url'] == 'https://example.com/1'
|
|
assert config.downloads[0]['frequency'] == 1_000
|
|
assert config.downloads[0]['downloader'] == 'archive'
|
|
|
|
# Create two once-downloads.
|
|
body = dict(
|
|
urls=['https://example.com/2', 'https://example.com/3'],
|
|
downloader='archive',
|
|
# tag_names=None, # tag_names can be missing.
|
|
)
|
|
request, response = await async_client.post('/api/download', json=body)
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
assert {i.url for i in test_session.query(Download)} == {f'https://example.com/{i}' for i in range(1, 4)}
|
|
# Assert that the config has been saved
|
|
config = get_download_manager_config()
|
|
assert len(config.downloads) == 3
|
|
assert {d['url'] for d in config.downloads} == {f'https://example.com/{i}' for i in range(1, 4)}
|
|
|
|
download1, download2, download3 = test_session.query(Download).order_by(Download.url).all()
|
|
assert not download1.settings
|
|
# destination is absolute
|
|
assert download1.destination == test_directory / 'archives/example.com'
|
|
expected_download = dict(download1.__json__()).copy()
|
|
|
|
# Update a download.
|
|
body = dict(
|
|
urls=['https://example.com/1'],
|
|
downloader='archive',
|
|
frequency=1_000,
|
|
tag_names=[tag1.name, tag2.name],
|
|
)
|
|
expected_download.update(
|
|
{'url': 'https://example.com/1', 'tag_names': [tag1.name, tag2.name], 'destination': None})
|
|
request, response = await async_client.put(f'/api/download/{download1.id}', json=body)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
test_session.flush([download2])
|
|
download2 = test_session.query(Download).filter_by(id=expected_download['id']).first()
|
|
assert download2.__json__() == expected_download
|
|
assert download2.tag_names == [tag1.name, tag2.name]
|
|
# Assert that the config has been saved
|
|
config = get_download_manager_config()
|
|
assert len(config.downloads) == 3
|
|
download_config = next(d for d in config.downloads if d['url'] == 'https://example.com/1')
|
|
assert download_config['tag_names'] == [tag1.name, tag2.name]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_update_download_with_settings_collection_id(test_session, async_client, test_download_manager_config,
|
|
test_directory):
|
|
"""Updating a download should accept collection_id and other legacy fields in settings.
|
|
|
|
The frontend sends collection_id inside settings when editing a download, even though it's also at the top level.
|
|
This should not cause a validation error."""
|
|
|
|
async def dispatch_downloads(*a, **kw):
|
|
pass
|
|
|
|
with mock.patch('wrolpi.downloader.DownloadManager.dispatch_downloads', dispatch_downloads):
|
|
# Create a recurring download first.
|
|
body = dict(
|
|
urls=['https://www.youtube.com/watch?v=r9pF8CdUbD0'],
|
|
frequency=86400,
|
|
downloader='video',
|
|
)
|
|
request, response = await async_client.post('/api/download', content=json.dumps(body))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
download = test_session.query(Download).one()
|
|
|
|
# Update the download with collection_id in settings (mimics frontend behavior).
|
|
# This is the exact payload from the bug report.
|
|
body = dict(
|
|
destination='',
|
|
downloader='video',
|
|
settings=dict(
|
|
video_format='mp4',
|
|
video_resolutions=['360p'],
|
|
collection_id=195, # This was causing ValidationError
|
|
parent_download_url='https://youtube.com/@cheimerdinger/videos',
|
|
channel_tag_name=None,
|
|
channel_url='https://youtube.com/@cheimerdinger', # Also stored in settings
|
|
destination='videos/test', # Legacy field
|
|
tag_names=['test'], # Legacy field
|
|
),
|
|
tag_names=[],
|
|
urls=['https://www.youtube.com/watch?v=r9pF8CdUbD0'],
|
|
)
|
|
request, response = await async_client.put(f'/api/download/{download.id}', json=body)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, \
|
|
f'Expected NO_CONTENT but got {response.status_code}: {response.json}'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_downloaders(async_client):
|
|
"""A list of Downloaders the user can use can be gotten."""
|
|
request, response = await async_client.get('/api/downloaders')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert 'downloaders' in response.json, 'Downloaders not returned'
|
|
assert isinstance(response.json['downloaders'], list) and len(response.json['downloaders']), \
|
|
'No downloaders returned'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_post_download(async_client):
|
|
"""Cannot have empty download URLs"""
|
|
content = dict(
|
|
urls=[''], # Cannot have empty URLs
|
|
frequency=1_000,
|
|
downloader='archive',
|
|
)
|
|
request, response = await async_client.post('/api/download', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json['code'] == 'INVALID_DOWNLOAD'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_restart_download(test_session, async_client, test_download_manager, test_downloader, await_switches):
|
|
"""A Download can be restarted."""
|
|
# Create a download, fail it, it should be restarted.
|
|
download = test_download_manager.create_download(test_session, 'https://example.com', test_downloader.name)
|
|
download.fail()
|
|
test_session.commit()
|
|
await await_switches()
|
|
assert test_session.query(Download).one().is_failed
|
|
|
|
test_downloader.set_test_failure()
|
|
|
|
# Download is now "new" again.
|
|
request, response = await async_client.post(f'/api/download/{download.id}/restart')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
download = test_session.query(Download).one()
|
|
assert download.is_new, download.status
|
|
|
|
# Wait for the background download to fail. It should be deferred.
|
|
await test_download_manager.wait_for_all_downloads()
|
|
download = test_session.query(Download).one()
|
|
assert download.is_deferred, download.status
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_global_statistics(test_session, async_client):
|
|
request, response = await async_client.get('/api/statistics')
|
|
assert response.json['global_statistics']['db_size'] > 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_vin_number_decoder(async_client):
|
|
"""A VIN number can be decoded."""
|
|
# Invalid VIN number.
|
|
content = dict(vin_number='5N1NJ01CXST00000')
|
|
request, response = await async_client.post('/api/vin_number_decoder', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == dict()
|
|
|
|
# Test VIN from vininfo.
|
|
content = dict(vin_number='5N1NJ01CXST000001')
|
|
request, response = await async_client.post('/api/vin_number_decoder', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['vin'] == {'body': 'Sedan, 4-Door, Standard Body Truck',
|
|
'country': 'United States',
|
|
'engine': 'VH45DE',
|
|
'manufacturer': 'Nissan',
|
|
'model': 'Maxima',
|
|
'plant': 'Tochigi, Oppama',
|
|
'region': 'North America',
|
|
'serial': None,
|
|
'transmission': None,
|
|
'years': '2025,1995'}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_suggestions(test_session, async_client, channel_factory, archive_factory, video_factory):
|
|
# WARNING results are cached, this test uses unique queries to avoid conflicts.
|
|
channel_factory(name='Foo')
|
|
channel_factory(name='Fool')
|
|
channel_factory(name='Bar')
|
|
archive_factory(domain='foo.com')
|
|
archive_factory(domain='bar.com')
|
|
video_factory(channel_id=2) # Channel "Fool" will be first in results because it has the most videos.
|
|
test_session.commit()
|
|
|
|
async def assert_results(body: dict, expected_channels=None, expected_domains=None):
|
|
expected_channels = expected_channels or []
|
|
expected_domains = expected_domains or []
|
|
|
|
request, response = await async_client.post('/api/search_suggestions', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
if expected_channels:
|
|
for channel, expected_channel in zip_longest(response.json['channels'], expected_channels):
|
|
assert_dict_contains(channel, expected_channel)
|
|
assert response.json['domains'] == expected_domains
|
|
|
|
await assert_results(
|
|
dict(search_str='foo'),
|
|
[
|
|
{'directory': 'videos/Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []},
|
|
{'directory': 'videos/Foo', 'id': 1, 'name': 'Foo', 'url': 'https://example.com/Foo', 'downloads': []},
|
|
],
|
|
[{'directory': 'archive/foo.com', 'domain': 'foo.com', 'id': 4}],
|
|
)
|
|
|
|
# Channel name "Fool" is matched because spaces are stripped in addition to only
|
|
await assert_results(
|
|
dict(search_str='foo l'),
|
|
[
|
|
{'directory': 'videos/Fool', 'id': 2, 'name': 'Fool', 'url': 'https://example.com/Fool', 'downloads': []}],
|
|
[],
|
|
)
|
|
|
|
await assert_results(
|
|
dict(search_str='bar'),
|
|
[
|
|
{'directory': 'videos/Bar', 'id': 3, 'name': 'Bar', 'url': 'https://example.com/Bar', 'downloads': []}
|
|
],
|
|
[{'directory': 'archive/bar.com', 'domain': 'bar.com', 'id': 5}],
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_file_estimates(test_session, async_client, archive_factory, tag_factory, video_factory):
|
|
"""Can quickly get count of possible files when searching."""
|
|
body_ = dict(search_str='can search with no files')
|
|
request_, response_ = await async_client.post('/api/search_file_estimates', json=body_)
|
|
assert response_.status_code == HTTPStatus.OK
|
|
|
|
tag1, tag2, tag3 = await tag_factory(), await tag_factory(), await tag_factory()
|
|
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
|
|
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
|
|
video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name])
|
|
test_session.commit()
|
|
|
|
async def assert_results(body: dict, expected_file_groups=None):
|
|
expected_file_groups = expected_file_groups or 0
|
|
|
|
request, response = await async_client.post('/api/search_file_estimates', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['file_groups'] == expected_file_groups or 0
|
|
|
|
await assert_results(dict(search_str='foo'), 1)
|
|
|
|
# Channel name "Fool" is matched because spaces are stripped in addition to only
|
|
await assert_results(dict(search_str='foo l'), 0)
|
|
|
|
await assert_results(dict(search_str='bar'), 1)
|
|
|
|
# "foo" archive and video both contain bunny.
|
|
await assert_results(dict(search_str='bunny'), 2)
|
|
|
|
# Filtering with mimetypes removes archive result.
|
|
await assert_results(dict(search_str='bunny', mimetypes=['video']), 1)
|
|
|
|
# tag1 has been used twice
|
|
await assert_results(dict(tag_names=[tag1.name, ]), 2)
|
|
|
|
# archive2 was tagged with tag2, but only video contains bunny.
|
|
await assert_results(dict(search_str='bunny', tag_names=[tag2.name, ]), 1)
|
|
|
|
# tag2 has been used twice
|
|
await assert_results(dict(tag_names=[tag2.name, ]), 2)
|
|
|
|
# tag2 has been used twice, but filter by video mimetype.
|
|
await assert_results(dict(tag_names=[tag2.name], mimetypes=['video']), 1)
|
|
|
|
# Can use all filters simultaneously.
|
|
await assert_results(dict(search_str='bunny', tag_names=[tag2.name], mimetypes=['video']), 1)
|
|
|
|
# Only the video has been tagged with both tag1 and tag2.
|
|
await assert_results(dict(tag_names=[tag1.name, tag2.name]), 1)
|
|
|
|
# tag3 was never used
|
|
await assert_results(dict(tag_names=[tag3.name]))
|
|
|
|
await assert_results(dict(search_str='Does not exist.'))
|
|
|
|
# No PDFs.
|
|
await assert_results(dict(mimetypes=['application/pdf']))
|
|
|
|
# Can filter by published datetime.
|
|
await assert_results(dict(months=[1, ]), 2)
|
|
await assert_results(dict(months=[2, ]), 0)
|
|
await assert_results(dict(from_year=2000, to_year=2000), 2)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_file_estimates_any_tag(test_session, async_client, archive_factory, tag_factory,
|
|
video_factory):
|
|
"""File estimates can be filtered by any_tag."""
|
|
tag1 = await tag_factory()
|
|
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
|
|
archive_factory(domain='bar.com', contents='contents of bar bunny')
|
|
test_session.commit()
|
|
|
|
async def assert_results(body: dict, expected_file_groups=None):
|
|
expected_file_groups = expected_file_groups or 0
|
|
|
|
request, response = await async_client.post('/api/search_file_estimates', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['file_groups'] == expected_file_groups or 0
|
|
|
|
await assert_results(dict(search_str='bunny'), 2)
|
|
await assert_results(dict(search_str='bunny', any_tag=True), 1)
|
|
await assert_results(dict(search_str='bar'), 1)
|
|
await assert_results(dict(search_str='bar', any_tag=True), 0)
|
|
|
|
body_ = dict(search_str='bunny', tag_names=[tag1.name, ], any_tag=True)
|
|
request_, response_ = await async_client.post('/api/search_file_estimates', json=body_)
|
|
assert response_.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_external_error(async_client):
|
|
"""A non-SanicException error is handled as an Internal Server Error."""
|
|
|
|
@async_client.sanic_app.get('/error')
|
|
async def error(_: Request):
|
|
raise RuntimeError('Oh no!')
|
|
|
|
request, response = await async_client.get('/error')
|
|
assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
|
|
assert response.json == {'code': 'RuntimeError', 'error': 'Oh no!', 'message': None}
|
|
|
|
|
|
def test_recursive_errors():
|
|
"""Errors are reported recursively."""
|
|
|
|
def one():
|
|
raise ValueError('Some error outside WROLPi')
|
|
|
|
def two():
|
|
try:
|
|
one()
|
|
except Exception as e1:
|
|
raise SearchEmpty('A more specific error') from e1
|
|
|
|
def three():
|
|
try:
|
|
two()
|
|
except Exception as e2:
|
|
raise ValidationError() from e2
|
|
|
|
try:
|
|
three()
|
|
except Exception as e:
|
|
response = json_error_handler(None, e)
|
|
assert json.loads(response.body.decode()) == {
|
|
'code': 'VALIDATION_ERROR',
|
|
'error': 'Bad Request', # Fallback when no message is passed on exception creation.
|
|
'message': 'Could not validate the contents of the request',
|
|
'cause': {
|
|
'code': 'SEARCH_EMPTY',
|
|
'error': 'A more specific error',
|
|
'message': 'Search is empty, search_str must have content.',
|
|
'cause': {
|
|
'code': 'ValueError', # Code falls back to Exception type.
|
|
'error': 'Some error outside WROLPi',
|
|
'message': None,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_settings_special_directories(async_client, test_wrolpi_config):
|
|
"""Maintainer can change special directories."""
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['archive_destination'] == 'archive/%(domain_tag)s/%(domain)s'
|
|
assert response.json['map_destination'] == 'map'
|
|
assert response.json['videos_destination'] == 'videos/%(channel_tag)s/%(channel_name)s'
|
|
assert response.json['zims_destination'] == 'zims'
|
|
|
|
data = {'archive_destination': 'archives'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
assert get_wrolpi_config().archive_destination == 'archives'
|
|
|
|
data = {'archive_destination': '/absolute/not/allowed'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
data = {'videos_destination': '/absolute/not/allowed'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
data = {'map_destination': '/absolute/not/allowed'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
data = {'zims_destination': '/absolute/not/allowed'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
# Empty directory restores default.
|
|
data = {'archive_destination': ''}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert get_wrolpi_config().archive_destination == 'archive/%(domain_tag)s/%(domain)s'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_settings_playlists_destination_validation(async_client, test_wrolpi_config):
|
|
"""The playlists destination is strictly validated: the sync deletes files there, so traversal
|
|
and overlap with other content destinations are rejected."""
|
|
# A valid change works.
|
|
data = {'playlists_destination': 'my_playlists'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert get_wrolpi_config().playlists_destination == 'my_playlists'
|
|
|
|
for invalid in (
|
|
'/absolute/not/allowed',
|
|
'../outside-media',
|
|
'a/../../b',
|
|
'videos', # equals/overlaps videos_destination's base
|
|
'videos/playlists', # inside videos_destination
|
|
'zims',
|
|
'tags',
|
|
):
|
|
data = {'playlists_destination': invalid}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST, \
|
|
f'{invalid!r} should have been rejected'
|
|
|
|
# Empty restores the default.
|
|
data = {'playlists_destination': ''}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert get_wrolpi_config().playlists_destination == 'playlists'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_settings_save_ffprobe_json(async_client, test_wrolpi_config):
|
|
"""Maintainer can toggle save_ffprobe_json setting."""
|
|
config = get_wrolpi_config()
|
|
|
|
# Default is True.
|
|
assert config.save_ffprobe_json is True
|
|
|
|
# GET returns the setting.
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['save_ffprobe_json'] is True
|
|
|
|
# Setting can be disabled.
|
|
data = {'save_ffprobe_json': False}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert config.save_ffprobe_json is False
|
|
|
|
# Setting can be re-enabled.
|
|
data = {'save_ffprobe_json': True}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert config.save_ffprobe_json is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_settings_timezone(async_client, test_wrolpi_config):
|
|
"""Maintainer can set and clear the timezone setting."""
|
|
config = get_wrolpi_config()
|
|
|
|
# Default is None.
|
|
assert config.timezone is None
|
|
|
|
# GET returns the setting.
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['timezone'] is None
|
|
|
|
# Setting a valid timezone.
|
|
data = {'timezone': 'America/Denver'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert config.timezone == 'America/Denver'
|
|
|
|
# GET reflects the change.
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['timezone'] == 'America/Denver'
|
|
|
|
# Empty string clears the timezone.
|
|
data = {'timezone': ''}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert config.timezone is None
|
|
|
|
# Invalid timezone is rejected.
|
|
data = {'timezone': 'Not/A/Timezone'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_other_estimates(async_client, test_session, channel_factory, tag_factory):
|
|
# Can search without Tags or Channels.
|
|
body = dict(tag_names=[])
|
|
request, response = await async_client.post('/api/search_other_estimates', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['others']['channel_count'] == 0, 'Only one Channel is tagged.'
|
|
|
|
tag = await tag_factory()
|
|
channel1 = channel_factory(tag_name=tag.name)
|
|
channel2 = channel_factory()
|
|
test_session.commit()
|
|
|
|
# One Channel is tagged.
|
|
body = dict(tag_names=[tag.name, ])
|
|
request, response = await async_client.post('/api/search_other_estimates', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['others']['channel_count'] == 1, 'Only one Channel is tagged.'
|