mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
Deleting, restarting, clearing and batch-editing Downloads, and creating,
renaming and deleting Tags, read then wrote in the request's deferred
transaction. SQLite fails that lock upgrade instantly ("database is
locked") when another connection wrote in between, so these failed with a
500 while a download or refresh was writing. They now write inside
request.ctx.db.write(), which begins as a writer and lets busy_timeout
absorb the contention.
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1127 lines
52 KiB
Python
1127 lines
52 KiB
Python
import json
|
|
from http import HTTPStatus
|
|
from itertools import zip_longest
|
|
|
|
import pytest
|
|
from mock import mock
|
|
from sanic import Request
|
|
|
|
from wrolpi.api_utils import json_error_handler
|
|
from wrolpi.common import get_wrolpi_config
|
|
from wrolpi.downloader import Download, get_download_manager_config
|
|
from wrolpi.errors import ValidationError, SearchEmpty
|
|
from wrolpi.conftest import production_like_sessions, write_lock_held_briefly
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_index(test_session, async_client):
|
|
"""
|
|
Index should have some details in an HTML response
|
|
"""
|
|
request, response = await async_client.get('/')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert b'html' in response.body
|
|
|
|
request, response = await async_client.get('/api')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert b'html' in response.body
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_valid_regex(test_session, async_client):
|
|
"""
|
|
The endpoint should only return valid if the regex is valid.
|
|
"""
|
|
data = {'regex': 'foo'}
|
|
request, response = await async_client.post('/api/valid_regex', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert json.loads(response.body) == {'valid': True, 'regex': 'foo'}
|
|
|
|
data = {'regex': '.*(title match).*'}
|
|
request, response = await async_client.post('/api/valid_regex', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert json.loads(response.body) == {'valid': True, 'regex': '.*(title match).*'}
|
|
|
|
data = {'regex': '.*(missing parenthesis.*'}
|
|
request, response = await async_client.post('/api/valid_regex', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert json.loads(response.body) == {'valid': False, 'regex': '.*(missing parenthesis.*'}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_settings(test_session, async_client):
|
|
get_wrolpi_config().ignored_directories = list()
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.status_code == HTTPStatus.OK
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_daily_download_limit_settings(test_session, async_client):
|
|
"""The daily download limits round-trip through the settings API and reject negative values."""
|
|
# Defaults are unlimited (None).
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.json['download_daily_limit_global'] is None
|
|
assert response.json['download_daily_limit_per_domain'] is None
|
|
|
|
data = {'download_daily_limit_global': 100, 'download_daily_limit_per_domain': 50}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.json['download_daily_limit_global'] == 100
|
|
assert response.json['download_daily_limit_per_domain'] == 50
|
|
assert get_wrolpi_config().download_daily_limit_global == 100
|
|
assert get_wrolpi_config().download_daily_limit_per_domain == 50
|
|
|
|
# 0 means unlimited and is accepted.
|
|
data = {'download_daily_limit_per_domain': 0}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
# Negative values are rejected.
|
|
data = {'download_daily_limit_global': -1}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_changing_zims_destination_restarts_kiwix(test_session, async_client):
|
|
"""Changing zims_destination should activate the restart_kiwix switch so Kiwix serves the new
|
|
directory; an unrelated settings change should not."""
|
|
from modules.zim import lib as zim_lib
|
|
|
|
with mock.patch.object(zim_lib.restart_kiwix_handler, 'activate_switch') as mock_activate:
|
|
# An unrelated change does not restart Kiwix.
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps({'log_level': 20}))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
mock_activate.assert_not_called()
|
|
|
|
# Changing the Zim directory does.
|
|
data = {'zims_destination': 'my_zims'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert get_wrolpi_config().zims_destination == 'my_zims'
|
|
mock_activate.assert_called_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_zims_destination_rejects_traversal(test_session, async_client):
|
|
"""zims_destination must be relative and must not contain traversal ('..') or be absolute."""
|
|
for bad in ('../../escape', 'a/../../b', '/abs/path'):
|
|
request, response = await async_client.patch('/api/settings',
|
|
content=json.dumps({'zims_destination': bad}))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST, f'{bad!r} should be rejected'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_log_level(async_client):
|
|
"""Log level must be between 0 and 40."""
|
|
data = {'log_level': 0}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
data = {'log_level': -1}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
data = {'log_level': 40}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
data = {'log_level': 41}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
data = {'log_level': None}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
data = {}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_downloads(test_session, async_client):
|
|
request, response = await async_client.get('/api/download')
|
|
assert_dict_contains(response.json, {'once_downloads': [], 'recurring_downloads': []})
|
|
|
|
d1 = Download(url='https://example.com/1', status='complete')
|
|
d2 = Download(url='https://example.com/2', status='pending')
|
|
d3 = Download(url='https://example.com/3', status='deferred')
|
|
d4 = Download(url='https://example.com/4', status='deferred')
|
|
d5 = Download(url='https://example.com/5', status='new')
|
|
d6 = Download(url='https://example.com/6', status='deferred', frequency=60)
|
|
d7 = Download(url='https://example.com/7', status='pending', frequency=60)
|
|
test_session.add_all([d1, d2, d3, d4, d5, d6, d7])
|
|
test_session.commit()
|
|
|
|
request, response = await async_client.get('/api/download')
|
|
expected = [
|
|
{'url': 'https://example.com/2', 'status': 'pending'},
|
|
{'url': 'https://example.com/5', 'status': 'new'},
|
|
{'url': 'https://example.com/3', 'status': 'deferred'},
|
|
{'url': 'https://example.com/4', 'status': 'deferred'},
|
|
{'url': 'https://example.com/1', 'status': 'complete'},
|
|
]
|
|
for download, expected in zip_longest(response.json['once_downloads'], expected):
|
|
assert_dict_contains(download, expected)
|
|
expected = [
|
|
{'url': 'https://example.com/7', 'status': 'pending'},
|
|
{'url': 'https://example.com/6', 'status': 'deferred'},
|
|
]
|
|
for download, expected in zip_longest(response.json['recurring_downloads'], expected):
|
|
assert_dict_contains(download, expected)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_downloads_sorter_recurring(async_client, test_session):
|
|
downloads = [
|
|
dict(status='complete', frequency=1),
|
|
dict(status='complete', frequency=2, next_download='2020-01-01T00:00:01+00:00'),
|
|
dict(status='complete', frequency=2, next_download='2020-01-01T00:00:02+00:00'),
|
|
dict(status='pending', frequency=1),
|
|
dict(status='pending', frequency=4),
|
|
dict(status='failed', frequency=1),
|
|
dict(status='failed', frequency=1),
|
|
]
|
|
for idx, download in enumerate(downloads):
|
|
test_session.add(Download(url=f'https://example.com/{idx}', **download))
|
|
test_session.commit()
|
|
|
|
expected = [
|
|
dict(status='pending', frequency=1),
|
|
dict(status='pending', frequency=4),
|
|
dict(status='failed', frequency=1),
|
|
dict(status='failed'),
|
|
dict(status='complete', frequency=2, next_download='2020-01-01T00:00:01+00:00'),
|
|
dict(status='complete', frequency=2, next_download='2020-01-01T00:00:02+00:00'),
|
|
dict(status='complete', frequency=1),
|
|
]
|
|
request, response = await async_client.get('/api/download')
|
|
recurring_downloads = response.json['recurring_downloads']
|
|
for d1, d2 in zip_longest(expected, recurring_downloads):
|
|
assert_dict_contains(d2, d1)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_downloads_sorter(async_client, test_session):
|
|
downloads = [
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:01+00:00'),
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:03+00:00'),
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:02+00:00'),
|
|
# last_successful_download are equal, so id is used next.
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:01+00:00', url='1'),
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:01+00:00', url='2'),
|
|
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:04+00:00'),
|
|
dict(status='failed'),
|
|
dict(status='failed', last_successful_download='2020-01-01T00:00:01+00:00'),
|
|
]
|
|
for idx, download in enumerate(downloads):
|
|
test_session.add(Download(url=download.pop('url', f'https://example.com/{idx}'), **download))
|
|
test_session.commit()
|
|
|
|
expected = [
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:04+00:00'),
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:01+00:00', url='1'),
|
|
dict(status='pending', last_successful_download='2020-01-01T00:00:01+00:00', url='2'),
|
|
dict(status='failed'),
|
|
dict(status='failed', last_successful_download='2020-01-01T00:00:01+00:00'),
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:03+00:00'),
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:02+00:00'),
|
|
dict(status='complete', last_successful_download='2020-01-01T00:00:01+00:00'),
|
|
]
|
|
request, response = await async_client.get('/api/download')
|
|
once_downloads = response.json['once_downloads']
|
|
for d1, d2 in zip_longest(expected, once_downloads):
|
|
assert_dict_contains(d2, d1)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_echo(async_client):
|
|
"""
|
|
Echo should send back a JSON object with our request details.
|
|
"""
|
|
request, response = await async_client.get('/api/echo?hello=world')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert_dict_contains(response.json['form'], {})
|
|
assert response.json['method'] == 'GET', 'Method was not GET'
|
|
assert response.json['json'] is None, 'JSON was not empty'
|
|
assert response.json['args'] == {'hello': ['world']}
|
|
|
|
data = {'foo': 'bar'}
|
|
request, response = await async_client.post('/api/echo', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert_dict_contains(response.json['form'], {})
|
|
assert response.json['method'] == 'POST', 'Method was not POST'
|
|
assert response.json['json'] == data, 'JSON was not data'
|
|
assert response.json['args'] == {}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_hotspot_settings(test_session, async_client, test_wrolpi_config):
|
|
"""
|
|
The User can configure hotspot settings (ssid, password, device) via /settings.
|
|
Hotspot control (on/off) is now done through Controller endpoints.
|
|
"""
|
|
config = get_wrolpi_config()
|
|
assert config.hotspot_on_startup is True
|
|
|
|
# Hotspot password can be changed.
|
|
content = {'hotspot_password': 'new password', 'hotspot_ssid': 'new ssid'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert config.hotspot_password == 'new password'
|
|
assert config.hotspot_ssid == 'new ssid'
|
|
|
|
# Hotspot password must be at least 8 characters.
|
|
content = {'hotspot_password': '1234567', 'hotspot_ssid': 'new ssid'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json['code'] == 'INVALID_CONFIG'
|
|
assert '8 characters' in response.json['error']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clear_downloads(test_session, async_client, test_wrolpi_config, test_download_manager_config):
|
|
d1 = Download(url='https://example.com/1', status='complete')
|
|
d2 = Download(url='https://example.com/2', status='pending')
|
|
d3 = Download(url='https://example.com/3', status='deferred')
|
|
d4 = Download(url='https://example.com/4', status='new')
|
|
d5 = Download(url='https://example.com/5', status='failed')
|
|
d6 = Download(url='https://example.com/6', status='failed', frequency=60)
|
|
d7 = Download(url='https://example.com/7', status='complete', frequency=60)
|
|
test_session.add_all([d1, d2, d3, d4, d5, d6, d7])
|
|
test_session.commit()
|
|
|
|
def check_downloads(response_, once_downloads_, recurring_downloads_, status_code=None):
|
|
if status_code:
|
|
assert response_.status_code == status_code
|
|
|
|
for download, once_download in zip_longest(response_.json['once_downloads'], once_downloads_):
|
|
assert download
|
|
assert once_download
|
|
assert download['url'] == once_download['url']
|
|
assert download['status'] == once_download['status']
|
|
for download, recurring_download in zip_longest(response_.json['recurring_downloads'], recurring_downloads_):
|
|
assert download
|
|
assert recurring_download
|
|
assert download['url'] == recurring_download['url']
|
|
assert download['status'] == recurring_download['status']
|
|
|
|
# All created downloads are present.
|
|
request, response = await async_client.get('/api/download')
|
|
once_downloads = [
|
|
{'url': 'https://example.com/2', 'status': 'pending'},
|
|
{'url': 'https://example.com/5', 'status': 'failed'},
|
|
{'url': 'https://example.com/4', 'status': 'new'},
|
|
{'url': 'https://example.com/3', 'status': 'deferred'},
|
|
{'url': 'https://example.com/1', 'status': 'complete'},
|
|
]
|
|
recurring_downloads = [
|
|
{'url': 'https://example.com/6', 'status': 'failed'},
|
|
{'url': 'https://example.com/7', 'status': 'complete'},
|
|
]
|
|
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
|
|
|
|
# Once "complete" download is removed.
|
|
request, response = await async_client.post('/api/download/clear_completed')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
request, response = await async_client.get('/api/download')
|
|
once_downloads = [
|
|
{'url': 'https://example.com/2', 'status': 'pending'},
|
|
{'url': 'https://example.com/5', 'status': 'failed'},
|
|
{'url': 'https://example.com/4', 'status': 'new'},
|
|
{'url': 'https://example.com/3', 'status': 'deferred'},
|
|
]
|
|
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
|
|
|
|
# Failed "once" download is removed, recurring failed is not removed.
|
|
request, response = await async_client.post('/api/download/clear_failed')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
request, response = await async_client.get('/api/download')
|
|
once_downloads = [
|
|
{'url': 'https://example.com/2', 'status': 'pending'},
|
|
{'url': 'https://example.com/4', 'status': 'new'},
|
|
{'url': 'https://example.com/3', 'status': 'deferred'},
|
|
]
|
|
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
|
|
|
|
# Failed once-downloads will not be downloaded again.
|
|
assert get_download_manager_config().skip_urls == ['https://example.com/5', ]
|
|
|
|
# Downloads can be retried.
|
|
request, response = await async_client.post('/api/download/retry_once')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
request, response = await async_client.get('/api/download')
|
|
once_downloads = [
|
|
{'url': 'https://example.com/2', 'status': 'new'},
|
|
{'url': 'https://example.com/3', 'status': 'new'},
|
|
{'url': 'https://example.com/4', 'status': 'new'},
|
|
]
|
|
recurring_downloads_after_retry = [
|
|
{'url': 'https://example.com/6', 'status': 'new'},
|
|
{'url': 'https://example.com/7', 'status': 'complete'},
|
|
]
|
|
check_downloads(response, once_downloads, recurring_downloads_after_retry, status_code=HTTPStatus.OK)
|
|
|
|
# "Delete All" button deletes all once-downloads.
|
|
request, response = await async_client.post('/api/download/delete_once')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
request, response = await async_client.get('/api/download')
|
|
check_downloads(response, [], recurring_downloads_after_retry, status_code=HTTPStatus.OK)
|
|
|
|
# Failed once-downloads will not be downloaded again.
|
|
assert get_download_manager_config().skip_urls == ['https://example.com/5', ]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_downloads(test_session, async_client, test_wrolpi_config, assert_downloads):
|
|
"""Downloads that are pending or deferred can be retried, and their attempts reset."""
|
|
d1 = Download(url='https://example.com/1', status='complete', attempts=1)
|
|
d2 = Download(url='https://example.com/2', status='pending', attempts=1)
|
|
d3 = Download(url='https://example.com/3', status='deferred', attempts=1)
|
|
d4 = Download(url='https://example.com/4', status='new', attempts=1)
|
|
d5 = Download(url='https://example.com/5', status='failed', attempts=1)
|
|
d6 = Download(url='https://example.com/6', status='failed', frequency=60, attempts=1)
|
|
d7 = Download(url='https://example.com/7', status='complete', frequency=60, attempts=1)
|
|
test_session.add_all([d1, d2, d3, d4, d5, d6, d7])
|
|
test_session.commit()
|
|
|
|
# Retry all once-downloads.
|
|
request, response = await async_client.post('/api/download/retry_once')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
assert_downloads([
|
|
dict(url='https://example.com/1', status='complete', attempts=1), # complete is not retried.
|
|
dict(url='https://example.com/2', status='new', attempts=0), # pending is retried.
|
|
dict(url='https://example.com/3', status='new', attempts=0), # deferred is retried.
|
|
dict(url='https://example.com/4', status='new', attempts=1), # new does not need to be retried.
|
|
dict(url='https://example.com/5', status='new', attempts=0), # failed is retried.
|
|
dict(url='https://example.com/6', status='new', frequency=60, attempts=0), # recurring failed is retried.
|
|
dict(url='https://example.com/7', status='complete', frequency=60, attempts=1), # complete recurring is unchanged.
|
|
])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_status(async_client, test_session):
|
|
"""Get the server status information."""
|
|
request, response = await async_client.get('/api/status')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert isinstance(response.json.get('cpu_stats'), dict), 'cpu_stats should be a dict'
|
|
assert isinstance(response.json.get('load_stats'), dict), 'load_stats should be a dict'
|
|
assert isinstance(response.json.get('drives_stats'), list), 'drive_stats should be a dict'
|
|
assert isinstance(response.json.get('downloads'), dict), 'downloads should be a dict'
|
|
assert isinstance(response.json.get('version'), str), 'version should be a str'
|
|
assert isinstance(response.json.get('memory_stats'), dict), 'memory_stats should be a dict'
|
|
assert isinstance(response.json.get('flags'), dict), 'flags should be a dict'
|
|
assert isinstance(response.json.get('sanic_workers'), dict), 'Sanic worker status should be a dict'
|
|
assert isinstance(response.json.get('local_time'), str), 'local_time should be a str'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_status_reports_perpetual_process(async_client, monkeypatch):
|
|
"""/api/status lists the perpetual process next to the server workers, and nothing else the manager tracks."""
|
|
import multiprocessing
|
|
from unittest import mock
|
|
from sanic.worker.process import Worker, WorkerProcess
|
|
from wrolpi.api_utils import api_app
|
|
from wrolpi.perpetual import PERPETUAL_PROCESS_NAME
|
|
|
|
def sanic_name(name):
|
|
worker = Worker(name, name, lambda: None, {}, multiprocessing.get_context(), {}, 1)
|
|
return next(iter(worker.processes)).name
|
|
|
|
server = sanic_name(f'{WorkerProcess.SERVER_LABEL}-0')
|
|
perpetual_name = sanic_name(PERPETUAL_PROCESS_NAME)
|
|
multiplexer = mock.Mock()
|
|
multiplexer.workers = {
|
|
'Sanic-Main': {'pid': 1},
|
|
server: {'pid': 2, 'state': 'ACKED'},
|
|
perpetual_name: {'pid': 3, 'state': 'STARTED'},
|
|
sanic_name('Inspector'): {'pid': 4, 'state': 'STARTED'},
|
|
}
|
|
monkeypatch.setattr(api_app, 'multiplexer', multiplexer, raising=False)
|
|
|
|
request, response = await async_client.get('/api/status')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['sanic_workers'] == {
|
|
server: {'pid': 2, 'state': 'ACKED'},
|
|
perpetual_name: {'pid': 3, 'state': 'STARTED'},
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_status_local_time_with_timezone(async_client, test_session, test_wrolpi_config):
|
|
"""Status endpoint includes local_time using configured timezone."""
|
|
from wrolpi.common import get_wrolpi_config
|
|
config = get_wrolpi_config()
|
|
config.timezone = 'America/Denver'
|
|
|
|
request, response = await async_client.get('/api/status')
|
|
assert response.status_code == HTTPStatus.OK
|
|
local_time = response.json['local_time']
|
|
# ISO 8601 with timezone offset for Denver (UTC-7 or UTC-6).
|
|
assert '+' in local_time or '-' in local_time[10:], 'local_time should include timezone offset'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_crud(test_session, async_client, test_download_manager_config, tag_factory, test_directory):
|
|
"""Test creating once-downloads and recurring downloads."""
|
|
tag1, tag2 = await tag_factory(), await tag_factory()
|
|
|
|
async def dispatch_downloads(*a, **kw):
|
|
pass
|
|
|
|
with mock.patch('wrolpi.downloader.DownloadManager.dispatch_downloads', dispatch_downloads):
|
|
# Create a single recurring Download.
|
|
body = dict(
|
|
urls=['https://example.com/1', ],
|
|
frequency=1_000,
|
|
downloader='archive',
|
|
destination='archives/example.com',
|
|
tag_names=None, # tag_names can be a None.
|
|
)
|
|
request, response = await async_client.post('/api/download', content=json.dumps(body))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
assert {i.url for i in test_session.query(Download)} == {'https://example.com/1', }
|
|
# Assert that the config has been saved
|
|
config = get_download_manager_config()
|
|
assert len(config.downloads) == 1
|
|
assert config.downloads[0]['url'] == 'https://example.com/1'
|
|
assert config.downloads[0]['frequency'] == 1_000
|
|
assert config.downloads[0]['downloader'] == 'archive'
|
|
|
|
# Create two once-downloads.
|
|
body = dict(
|
|
urls=['https://example.com/2', 'https://example.com/3'],
|
|
downloader='archive',
|
|
# tag_names=None, # tag_names can be missing.
|
|
)
|
|
request, response = await async_client.post('/api/download', json=body)
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
assert {i.url for i in test_session.query(Download)} == {f'https://example.com/{i}' for i in range(1, 4)}
|
|
# Assert that the config has been saved
|
|
config = get_download_manager_config()
|
|
assert len(config.downloads) == 3
|
|
assert {d['url'] for d in config.downloads} == {f'https://example.com/{i}' for i in range(1, 4)}
|
|
|
|
download1, download2, download3 = test_session.query(Download).order_by(Download.url).all()
|
|
assert not download1.settings
|
|
# destination is absolute
|
|
assert download1.destination == test_directory / 'archives/example.com'
|
|
expected_download = dict(download1.__json__()).copy()
|
|
|
|
# Update a download.
|
|
body = dict(
|
|
urls=['https://example.com/1'],
|
|
downloader='archive',
|
|
frequency=1_000,
|
|
tag_names=[tag1.name, tag2.name],
|
|
)
|
|
expected_download.update(
|
|
{'url': 'https://example.com/1', 'tag_names': [tag1.name, tag2.name], 'destination': None})
|
|
request, response = await async_client.put(f'/api/download/{download1.id}', json=body)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
test_session.flush([download2])
|
|
download2 = test_session.query(Download).filter_by(id=expected_download['id']).first()
|
|
assert download2.__json__() == expected_download
|
|
assert download2.tag_names == [tag1.name, tag2.name]
|
|
# Assert that the config has been saved
|
|
config = get_download_manager_config()
|
|
assert len(config.downloads) == 3
|
|
download_config = next(d for d in config.downloads if d['url'] == 'https://example.com/1')
|
|
assert download_config['tag_names'] == [tag1.name, tag2.name]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_update_download_with_settings_collection_id(test_session, async_client, test_download_manager_config,
|
|
test_directory):
|
|
"""Updating a download should accept collection_id and other legacy fields in settings.
|
|
|
|
The frontend sends collection_id inside settings when editing a download, even though it's also at the top level.
|
|
This should not cause a validation error."""
|
|
|
|
async def dispatch_downloads(*a, **kw):
|
|
pass
|
|
|
|
with mock.patch('wrolpi.downloader.DownloadManager.dispatch_downloads', dispatch_downloads):
|
|
# Create a recurring download first.
|
|
body = dict(
|
|
urls=['https://www.youtube.com/watch?v=r9pF8CdUbD0'],
|
|
frequency=86400,
|
|
downloader='video',
|
|
)
|
|
request, response = await async_client.post('/api/download', content=json.dumps(body))
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
|
|
download = test_session.query(Download).one()
|
|
|
|
# Update the download with collection_id in settings (mimics frontend behavior).
|
|
# This is the exact payload from the bug report.
|
|
body = dict(
|
|
destination='',
|
|
downloader='video',
|
|
settings=dict(
|
|
video_format='mp4',
|
|
video_resolutions=['360p'],
|
|
collection_id=195, # This was causing ValidationError
|
|
parent_download_url='https://youtube.com/@cheimerdinger/videos',
|
|
channel_tag_name=None,
|
|
channel_url='https://youtube.com/@cheimerdinger', # Also stored in settings
|
|
destination='videos/test', # Legacy field
|
|
tag_names=['test'], # Legacy field
|
|
),
|
|
tag_names=[],
|
|
urls=['https://www.youtube.com/watch?v=r9pF8CdUbD0'],
|
|
)
|
|
request, response = await async_client.put(f'/api/download/{download.id}', json=body)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, \
|
|
f'Expected NO_CONTENT but got {response.status_code}: {response.json}'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_update_download_frequency_once(test_session, async_client, test_download_manager_config,
|
|
test_download_manager):
|
|
"""Editing a recurring Download to a frequency of 0 ("Once") stores a once-download, not a zero interval."""
|
|
|
|
async def dispatch_downloads(*a, **kw):
|
|
pass
|
|
|
|
with mock.patch('wrolpi.downloader.DownloadManager.dispatch_downloads', dispatch_downloads):
|
|
body = dict(urls=['https://example.com/feed.xml'], downloader='rss', sub_downloader='archive',
|
|
frequency=86400)
|
|
request, response = await async_client.post('/api/download', json=body)
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
download = test_session.query(Download).one()
|
|
assert download.frequency == 86400
|
|
|
|
body = dict(urls=['https://example.com/feed.xml'], downloader='rss', sub_downloader='archive', frequency=0)
|
|
request, response = await async_client.put(f'/api/download/{download.id}', json=body)
|
|
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
|
|
|
test_session.expire_all()
|
|
download = test_session.query(Download).one()
|
|
assert download.frequency is None
|
|
assert test_download_manager.get_recurring_downloads(test_session) == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_downloaders(async_client):
|
|
"""A list of Downloaders the user can use can be gotten."""
|
|
request, response = await async_client.get('/api/downloaders')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert 'downloaders' in response.json, 'Downloaders not returned'
|
|
assert isinstance(response.json['downloaders'], list) and len(response.json['downloaders']), \
|
|
'No downloaders returned'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_post_download(async_client):
|
|
"""Cannot have empty download URLs"""
|
|
content = dict(
|
|
urls=[''], # Cannot have empty URLs
|
|
frequency=1_000,
|
|
downloader='archive',
|
|
)
|
|
request, response = await async_client.post('/api/download', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json['code'] == 'INVALID_DOWNLOAD'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_restart_download(test_session, async_client, test_download_manager, test_downloader, await_switches):
|
|
"""A Download can be restarted."""
|
|
# Create a download, fail it, it should be restarted.
|
|
download = test_download_manager.create_download(test_session, 'https://example.com', test_downloader.name)
|
|
download.fail()
|
|
test_session.commit()
|
|
await await_switches()
|
|
assert test_session.query(Download).one().is_failed
|
|
|
|
test_downloader.set_test_failure()
|
|
|
|
# Download is now "new" again.
|
|
request, response = await async_client.post(f'/api/download/{download.id}/restart')
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
download = test_session.query(Download).one()
|
|
assert download.is_new, download.status
|
|
|
|
# Wait for the background download to fail. It should be deferred.
|
|
await test_download_manager.wait_for_all_downloads()
|
|
download = test_session.query(Download).one()
|
|
assert download.is_deferred, download.status
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_global_statistics(test_session, async_client):
|
|
request, response = await async_client.get('/api/statistics')
|
|
assert response.json['global_statistics']['db_size'] > 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_post_vin_number_decoder(async_client):
|
|
"""A VIN number can be decoded."""
|
|
# Invalid VIN number.
|
|
content = dict(vin_number='5N1NJ01CXST00000')
|
|
request, response = await async_client.post('/api/vin_number_decoder', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
assert response.json == dict()
|
|
|
|
# Test VIN from vininfo.
|
|
content = dict(vin_number='5N1NJ01CXST000001')
|
|
request, response = await async_client.post('/api/vin_number_decoder', content=json.dumps(content))
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['vin'] == {'body': 'Sedan, 4-Door, Standard Body Truck',
|
|
'country': 'United States',
|
|
'engine': 'VH45DE',
|
|
'manufacturer': 'Nissan',
|
|
'model': 'Maxima',
|
|
'plant': 'Tochigi, Oppama',
|
|
'region': 'North America',
|
|
'serial': None,
|
|
'transmission': None,
|
|
'years': '2025,1995'}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_suggestions(test_session, async_client, channel_factory, archive_factory, video_factory):
|
|
# Use unique names so this test does not collide with other workers' fixtures or any
|
|
# leftover shared-process caches. Do not hardcode DB ids — they shift when other
|
|
# fixtures create Channels/Collections first.
|
|
channel_foo = channel_factory(name='SugFoo')
|
|
channel_fool = channel_factory(name='SugFool')
|
|
channel_bar = channel_factory(name='SugBar')
|
|
archive_factory(domain='sugfoo.com')
|
|
archive_factory(domain='sugbar.com')
|
|
# SugFool ranks first for "sugfoo" when ordering by video count.
|
|
video_factory(channel_id=channel_fool.id)
|
|
test_session.commit()
|
|
|
|
async def assert_results(body: dict, expected_channels=None, expected_domains=None):
|
|
expected_channels = expected_channels or []
|
|
expected_domains = expected_domains or []
|
|
|
|
request, response = await async_client.post('/api/search_suggestions', json=body)
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
assert len(response.json['channels']) == len(expected_channels), response.json
|
|
for channel, expected_channel in zip_longest(response.json['channels'], expected_channels):
|
|
assert_dict_contains(channel, expected_channel)
|
|
assert len(response.json['domains']) == len(expected_domains), (
|
|
f"domains mismatch for {body!r}: got {response.json['domains']!r}, "
|
|
f"channels={response.json.get('channels')!r}"
|
|
)
|
|
for domain, expected_domain in zip_longest(response.json['domains'], expected_domains):
|
|
assert_dict_contains(domain, expected_domain)
|
|
|
|
await assert_results(
|
|
dict(search_str='sugfoo'),
|
|
[
|
|
{'directory': 'videos/SugFool', 'id': channel_fool.id, 'name': 'SugFool',
|
|
'url': 'https://example.com/SugFool', 'downloads': []},
|
|
{'directory': 'videos/SugFoo', 'id': channel_foo.id, 'name': 'SugFoo',
|
|
'url': 'https://example.com/SugFoo', 'downloads': []},
|
|
],
|
|
[{'directory': 'archive/sugfoo.com', 'domain': 'sugfoo.com'}],
|
|
)
|
|
|
|
# Channel name "SugFool" is matched because spaces are stripped.
|
|
await assert_results(
|
|
dict(search_str='sugfoo l'),
|
|
[
|
|
{'directory': 'videos/SugFool', 'id': channel_fool.id, 'name': 'SugFool',
|
|
'url': 'https://example.com/SugFool', 'downloads': []},
|
|
],
|
|
[],
|
|
)
|
|
|
|
await assert_results(
|
|
dict(search_str='sugbar'),
|
|
[
|
|
{'directory': 'videos/SugBar', 'id': channel_bar.id, 'name': 'SugBar',
|
|
'url': 'https://example.com/SugBar', 'downloads': []},
|
|
],
|
|
[{'directory': 'archive/sugbar.com', 'domain': 'sugbar.com'}],
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_file_estimates(test_session, async_client, archive_factory, tag_factory, video_factory):
|
|
"""Can quickly get count of possible files when searching."""
|
|
body_ = dict(search_str='can search with no files')
|
|
request_, response_ = await async_client.post('/api/search_file_estimates', json=body_)
|
|
assert response_.status_code == HTTPStatus.OK
|
|
|
|
tag1, tag2, tag3 = await tag_factory(), await tag_factory(), await tag_factory()
|
|
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
|
|
archive_factory(domain='bar.com', contents='contents of bar', tag_names=[tag2.name, ])
|
|
video_factory(with_caption_file=True, tag_names=[tag1.name, tag2.name])
|
|
test_session.commit()
|
|
|
|
async def assert_results(body: dict, expected_file_groups=None):
|
|
expected_file_groups = expected_file_groups or 0
|
|
|
|
request, response = await async_client.post('/api/search_file_estimates', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
# The deep count includes contents/captions matches; the fast (ABC) count is in `file_groups`.
|
|
assert response.json['file_groups_deep'] == expected_file_groups or 0
|
|
|
|
await assert_results(dict(search_str='foo'), 1)
|
|
|
|
# Channel name "Fool" is matched because spaces are stripped in addition to only
|
|
await assert_results(dict(search_str='foo l'), 0)
|
|
|
|
await assert_results(dict(search_str='bar'), 1)
|
|
|
|
# "foo" archive and video both contain bunny.
|
|
await assert_results(dict(search_str='bunny'), 2)
|
|
|
|
# Filtering with mimetypes removes archive result.
|
|
await assert_results(dict(search_str='bunny', mimetypes=['video']), 1)
|
|
|
|
# tag1 has been used twice
|
|
await assert_results(dict(tag_names=[tag1.name, ]), 2)
|
|
|
|
# archive2 was tagged with tag2, but only video contains bunny.
|
|
await assert_results(dict(search_str='bunny', tag_names=[tag2.name, ]), 1)
|
|
|
|
# tag2 has been used twice
|
|
await assert_results(dict(tag_names=[tag2.name, ]), 2)
|
|
|
|
# tag2 has been used twice, but filter by video mimetype.
|
|
await assert_results(dict(tag_names=[tag2.name], mimetypes=['video']), 1)
|
|
|
|
# Can use all filters simultaneously.
|
|
await assert_results(dict(search_str='bunny', tag_names=[tag2.name], mimetypes=['video']), 1)
|
|
|
|
# Only the video has been tagged with both tag1 and tag2.
|
|
await assert_results(dict(tag_names=[tag1.name, tag2.name]), 1)
|
|
|
|
# tag3 was never used
|
|
await assert_results(dict(tag_names=[tag3.name]))
|
|
|
|
await assert_results(dict(search_str='Does not exist.'))
|
|
|
|
# No PDFs.
|
|
await assert_results(dict(mimetypes=['application/pdf']))
|
|
|
|
# Can filter by published datetime.
|
|
await assert_results(dict(months=[1, ]), 2)
|
|
await assert_results(dict(months=[2, ]), 0)
|
|
await assert_results(dict(from_year=2000, to_year=2000), 2)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_file_estimates_any_tag(test_session, async_client, archive_factory, tag_factory,
|
|
video_factory):
|
|
"""File estimates can be filtered by any_tag."""
|
|
tag1 = await tag_factory()
|
|
archive_factory(domain='foo.com', contents='contents of foo with bunny', tag_names=[tag1.name, ])
|
|
archive_factory(domain='bar.com', contents='contents of bar bunny')
|
|
test_session.commit()
|
|
|
|
async def assert_results(body: dict, expected_file_groups=None):
|
|
expected_file_groups = expected_file_groups or 0
|
|
|
|
request, response = await async_client.post('/api/search_file_estimates', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
# The deep count includes contents/captions matches; the fast (ABC) count is in `file_groups`.
|
|
assert response.json['file_groups_deep'] == expected_file_groups or 0
|
|
|
|
await assert_results(dict(search_str='bunny'), 2)
|
|
await assert_results(dict(search_str='bunny', any_tag=True), 1)
|
|
await assert_results(dict(search_str='bar'), 1)
|
|
await assert_results(dict(search_str='bar', any_tag=True), 0)
|
|
|
|
body_ = dict(search_str='bunny', tag_names=[tag1.name, ], any_tag=True)
|
|
request_, response_ = await async_client.post('/api/search_file_estimates', json=body_)
|
|
assert response_.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_external_error(async_client):
|
|
"""A non-SanicException error is handled as an Internal Server Error."""
|
|
|
|
@async_client.sanic_app.get('/error')
|
|
async def error(_: Request):
|
|
raise RuntimeError('Oh no!')
|
|
|
|
request, response = await async_client.get('/error')
|
|
assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
|
|
assert response.json == {'code': 'RuntimeError', 'error': 'Oh no!', 'message': None}
|
|
|
|
|
|
def test_recursive_errors():
|
|
"""Errors are reported recursively."""
|
|
|
|
def one():
|
|
raise ValueError('Some error outside WROLPi')
|
|
|
|
def two():
|
|
try:
|
|
one()
|
|
except Exception as e1:
|
|
raise SearchEmpty('A more specific error') from e1
|
|
|
|
def three():
|
|
try:
|
|
two()
|
|
except Exception as e2:
|
|
raise ValidationError() from e2
|
|
|
|
try:
|
|
three()
|
|
except Exception as e:
|
|
response = json_error_handler(None, e)
|
|
assert json.loads(response.body.decode()) == {
|
|
'code': 'VALIDATION_ERROR',
|
|
'error': 'Bad Request', # Fallback when no message is passed on exception creation.
|
|
'message': 'Could not validate the contents of the request',
|
|
'cause': {
|
|
'code': 'SEARCH_EMPTY',
|
|
'error': 'A more specific error',
|
|
'message': 'Search is empty, search_str must have content.',
|
|
'cause': {
|
|
'code': 'ValueError', # Code falls back to Exception type.
|
|
'error': 'Some error outside WROLPi',
|
|
'message': None,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_settings_special_directories(async_client, test_wrolpi_config):
|
|
"""Maintainer can change special directories."""
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['archive_destination'] == 'archive/%(domain_tag)s/%(domain)s'
|
|
assert response.json['map_destination'] == 'map'
|
|
assert response.json['videos_destination'] == 'videos/%(channel_tag)s/%(channel_name)s'
|
|
assert response.json['zims_destination'] == 'zims'
|
|
|
|
data = {'archive_destination': 'archives'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
|
|
assert get_wrolpi_config().archive_destination == 'archives'
|
|
|
|
data = {'archive_destination': '/absolute/not/allowed'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
data = {'videos_destination': '/absolute/not/allowed'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
data = {'map_destination': '/absolute/not/allowed'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
data = {'zims_destination': '/absolute/not/allowed'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
# Empty directory restores default.
|
|
data = {'archive_destination': ''}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert get_wrolpi_config().archive_destination == 'archive/%(domain_tag)s/%(domain)s'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_settings_playlists_destination_validation(async_client, test_wrolpi_config):
|
|
"""The playlists destination is strictly validated: the sync deletes files there, so traversal
|
|
and overlap with other content destinations are rejected."""
|
|
# A valid change works.
|
|
data = {'playlists_destination': 'my_playlists'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert get_wrolpi_config().playlists_destination == 'my_playlists'
|
|
|
|
for invalid in (
|
|
'/absolute/not/allowed',
|
|
'../outside-media',
|
|
'a/../../b',
|
|
'videos', # equals/overlaps videos_destination's base
|
|
'videos/playlists', # inside videos_destination
|
|
'zims',
|
|
'tags',
|
|
):
|
|
data = {'playlists_destination': invalid}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST, \
|
|
f'{invalid!r} should have been rejected'
|
|
|
|
# Empty restores the default.
|
|
data = {'playlists_destination': ''}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert get_wrolpi_config().playlists_destination == 'playlists'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_settings_save_ffprobe_json(async_client, test_wrolpi_config):
|
|
"""Maintainer can toggle save_ffprobe_json setting."""
|
|
config = get_wrolpi_config()
|
|
|
|
# Default is True.
|
|
assert config.save_ffprobe_json is True
|
|
|
|
# GET returns the setting.
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['save_ffprobe_json'] is True
|
|
|
|
# Setting can be disabled.
|
|
data = {'save_ffprobe_json': False}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert config.save_ffprobe_json is False
|
|
|
|
# Setting can be re-enabled.
|
|
data = {'save_ffprobe_json': True}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert config.save_ffprobe_json is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_settings_timezone(async_client, test_wrolpi_config):
|
|
"""Maintainer can set and clear the timezone setting."""
|
|
config = get_wrolpi_config()
|
|
|
|
# Default is None.
|
|
assert config.timezone is None
|
|
|
|
# GET returns the setting.
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['timezone'] is None
|
|
|
|
# Setting a valid timezone.
|
|
data = {'timezone': 'America/Denver'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert config.timezone == 'America/Denver'
|
|
|
|
# GET reflects the change.
|
|
request, response = await async_client.get('/api/settings')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['timezone'] == 'America/Denver'
|
|
|
|
# Empty string clears the timezone.
|
|
data = {'timezone': ''}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.NO_CONTENT
|
|
assert config.timezone is None
|
|
|
|
# Invalid timezone is rejected.
|
|
data = {'timezone': 'Not/A/Timezone'}
|
|
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_search_other_estimates(async_client, test_session, channel_factory, tag_factory):
|
|
# Can search without Tags or Channels.
|
|
body = dict(tag_names=[])
|
|
request, response = await async_client.post('/api/search_other_estimates', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['others']['channel_count'] == 0, 'Only one Channel is tagged.'
|
|
|
|
tag = await tag_factory()
|
|
channel1 = channel_factory(tag_name=tag.name)
|
|
channel2 = channel_factory()
|
|
test_session.commit()
|
|
|
|
# One Channel is tagged.
|
|
body = dict(tag_names=[tag.name, ])
|
|
request, response = await async_client.post('/api/search_other_estimates', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['others']['channel_count'] == 1, 'Only one Channel is tagged.'
|
|
|
|
# The estimate counts the Channels the search lists: those with any of the Tags.
|
|
tag2 = await tag_factory()
|
|
channel_factory(tag_name=tag2.name)
|
|
test_session.commit()
|
|
body = dict(tag_names=[tag.name, tag2.name])
|
|
request, response = await async_client.post('/api/search_other_estimates', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
estimate = response.json['others']['channel_count']
|
|
request, response = await async_client.post('/api/videos/channels/search', json=body)
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert estimate == len(response.json['channels']) == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_mutating_request_survives_concurrent_writer(async_client, test_session, test_downloader,
|
|
test_download_manager):
|
|
"""Creating a Download must not fail because something else is writing.
|
|
|
|
Regression test for the production failure: a user created a video download while a refresh was
|
|
running and got `database is locked` on `INSERT INTO download`. The request session begins
|
|
*deferred*, the handler reads (tag/duplicate checks) before it INSERTs, and SQLite refuses that
|
|
mid-transaction lock upgrade instantly -- busy_timeout is skipped for upgrades. Mutating
|
|
requests must begin as writers so busy_timeout absorbs the contention instead.
|
|
"""
|
|
db_file = test_session.get_bind().url.database
|
|
body = json.dumps(dict(urls=['https://example.com/during-refresh'], downloader=test_downloader.name))
|
|
# Release anything the fixtures left open on the shared session's connection; only the lock the
|
|
# test takes below should contend with the request.
|
|
test_session.commit()
|
|
|
|
with production_like_sessions(test_session) as maker:
|
|
with write_lock_held_briefly(db_file):
|
|
request, response = await async_client.post('/api/download', content=body)
|
|
|
|
assert response.status_code == HTTPStatus.CREATED, response.body
|
|
|
|
# A separate session sees the committed Download, as the download worker would.
|
|
session = maker()
|
|
try:
|
|
assert [i.url for i in session.query(Download).all()] == ['https://example.com/during-refresh']
|
|
finally:
|
|
session.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize('method,path,body,status', [
|
|
# Deleting a Download saves the skip list before it deletes, so the lock is held longer (see `hold` below).
|
|
('delete', '/api/download/{failed}', None, HTTPStatus.NO_CONTENT),
|
|
('post', '/api/download/{failed}/restart', None, HTTPStatus.NO_CONTENT),
|
|
('post', '/api/download/clear_completed', None, HTTPStatus.NO_CONTENT),
|
|
('post', '/api/download/clear_failed', None, HTTPStatus.NO_CONTENT),
|
|
('post', '/api/download/delete_once', None, HTTPStatus.NO_CONTENT),
|
|
('post', '/api/download/batch/delete', {'download_ids': ['{failed}']}, HTTPStatus.OK),
|
|
('post', '/api/download/batch/retry', {'download_ids': ['{failed}']}, HTTPStatus.OK),
|
|
('post', '/api/download/batch/clear', {'download_ids': ['{complete}']}, HTTPStatus.OK),
|
|
('post', '/api/tag', {'name': 'New', 'color': '#000000'}, HTTPStatus.CREATED),
|
|
('post', '/api/tag/{tag}', {'name': 'Renamed', 'color': '#ffffff'}, HTTPStatus.OK),
|
|
('delete', '/api/tag/{tag}', None, HTTPStatus.NO_CONTENT),
|
|
])
|
|
async def test_download_and_tag_writes_wait_for_a_concurrent_writer(async_client, test_session, test_download_manager,
|
|
tag_factory, method, path, body, status):
|
|
"""These requests read before they write, so each must begin as a writer: SQLite fails a deferred
|
|
transaction's lock upgrade instantly ("database is locked") if another connection wrote meanwhile
|
|
(see `test_mutating_request_survives_concurrent_writer`)."""
|
|
failed = Download(url='https://example.com/failed', status='failed')
|
|
complete = Download(url='https://example.com/complete', status='complete')
|
|
test_session.add_all([failed, complete])
|
|
tag = await tag_factory('Old')
|
|
test_session.commit()
|
|
ids = dict(failed=failed.id, complete=complete.id, tag=tag.id)
|
|
path = path.format(**ids)
|
|
if body and 'download_ids' in body:
|
|
body = {'download_ids': [int(i.format(**ids)) for i in body['download_ids']]}
|
|
db_file = test_session.get_bind().url.database
|
|
|
|
with production_like_sessions(test_session):
|
|
# Hold the lock until the request's first write, even when it does slow work first.
|
|
hold = 3 if method == 'delete' and path.startswith('/api/download/') else 1
|
|
with write_lock_held_briefly(db_file, seconds=hold):
|
|
kwargs = {'content': json.dumps(body)} if body else {}
|
|
_, response = await getattr(async_client, method)(path, **kwargs)
|
|
|
|
assert response.status_code == status, response.body
|