Configs can be saved in background using switches.

This commit is contained in:
Roland 2024-09-15 15:33:07 -06:00
parent 44d4921c84
commit d54735c4bb
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
53 changed files with 967 additions and 574 deletions

View file

@ -82,20 +82,20 @@ def test_archives_search(test_session, archive_directory, archive_factory, test_
@pytest.mark.asyncio
async def test_search_archive_tags(test_session, test_async_client, archive_factory, tag_factory):
async def test_search_archive_tags(test_session, async_client, archive_factory, tag_factory):
"""Tagged Archives can be searched."""
tag = await tag_factory()
archive_factory(domain='example.com', tag_names=[tag.name, ])
test_session.commit()
content = {'search_str': '', 'domain': 'example.com', 'tag_names': [tag.name, ]}
request, response = await test_async_client.post('/api/archive/search', content=json.dumps(content))
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups']
assert response.json['totals']['file_groups'] == 1
content = {'search_str': '', 'domain': 'example.com', 'tag_names': ['does not exist', ]}
request, response = await test_async_client.post('/api/archive/search', content=json.dumps(content))
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == []
assert response.json['totals']['file_groups'] == 0

View file

@ -80,7 +80,7 @@ async def test_relationships(test_session, example_singlefile):
@pytest.mark.asyncio
async def test_archive_title(test_async_client, test_session, archive_factory, singlefile_contents_factory):
async def test_archive_title(async_client, test_session, archive_factory, singlefile_contents_factory):
"""An Archive's title can be fetched in multiple ways. This tests from most to least reliable."""
# Create some test files, delete all records for a fresh refresh.
archive_factory(
@ -459,7 +459,7 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b
@pytest.mark.asyncio
async def test_refresh_archives(test_session, test_directory, test_async_client, make_files_structure):
async def test_refresh_archives(test_session, test_directory, async_client, make_files_structure):
"""Archives can be found and put in the database. A single Archive will have multiple files."""
# The start of a typical singlefile html file.
singlefile_contents = '''<!DOCTYPE html> <html lang="en"><!--
@ -484,7 +484,7 @@ async def test_refresh_archives(test_session, test_directory, test_async_client,
})
# The single archive is found.
await test_async_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
assert test_session.query(Archive).count() == 1
# Cause a re-index of the archive.
@ -493,7 +493,7 @@ async def test_refresh_archives(test_session, test_directory, test_async_client,
test_session.commit()
# Running the refresh does not result in a new archive.
await test_async_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
assert test_session.query(Archive).count() == 1
# Archives file format was changed, lets check the new formats are found.
@ -501,13 +501,13 @@ async def test_refresh_archives(test_session, test_directory, test_async_client,
'archive/example.com/2021-10-05-16-20-11_The Title.html': '<html></html>',
'archive/example.com/2021-10-05-16-20-11_The Title.readability.json': '{"url": "https://example.com"}',
})
await test_async_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
# The old formatted archive above is renamed.
assert (test_directory / 'archive/example.com/2021-10-05-16-20-10_NA.html').is_file()
assert test_session.query(Archive).count() == 2
content = json.dumps({'search_str': 'text', 'model': 'archive'})
request, response = await test_async_client.post('/api/files/search', content=content)
request, response = await async_client.post('/api/files/search', content=content)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'], 'No files matched "text"'
assert response.json['file_groups'][0]['model'] == 'archive', 'Returned file was not an archive'
@ -821,7 +821,7 @@ def test_get_url_from_singlefile():
@pytest.mark.asyncio
async def test_get_custom_archive_directory(test_async_client, test_directory, test_config):
async def test_get_custom_archive_directory(async_client, test_directory, test_config):
"""Custom directory can be used for archive directory."""
# Default location.
assert lib.get_archive_directory() == (test_directory / 'archive')

View file

@ -7,7 +7,7 @@ from wrolpi.common import get_wrolpi_config
@pytest.mark.asyncio
async def test_archive_download_destination(test_async_client, test_session, test_directory, archive_factory, fake_now):
async def test_archive_download_destination(async_client, test_session, test_directory, archive_factory, fake_now):
fake_now(datetime(2000, 1, 2))
wrolpi_config = get_wrolpi_config()

View file

@ -180,7 +180,7 @@ def save_inventories_file():
config.inventories = inventories
config.version = version
config.save()
config.background_save.activate_switch()
@before_startup

View file

@ -76,33 +76,33 @@ def test_item_api(test_session, init_test_inventory, test_client):
@pytest.mark.asyncio
async def test_inventory_api(test_session, test_async_client):
async def test_inventory_api(test_session, async_client):
"""An Inventory can be created."""
inventory = dict(name='foo')
request, response = await test_async_client.post('/api/inventory', content=json.dumps(inventory))
request, response = await async_client.post('/api/inventory', content=json.dumps(inventory))
assert response.status_code == HTTPStatus.CREATED, response.status_code
request, response = await test_async_client.get('/api/inventory')
request, response = await async_client.get('/api/inventory')
assert response.status_code == HTTPStatus.OK
inventory = response.json['inventories'][0]
inventory_id = inventory['id']
request, response = await test_async_client.get(f'/api/inventory/{inventory_id}')
request, response = await async_client.get(f'/api/inventory/{inventory_id}')
assert response.status_code == HTTPStatus.OK
request, response = await test_async_client.get('/api/inventory/categories')
request, response = await async_client.get('/api/inventory/categories')
assert response.status_code == HTTPStatus.OK
request, response = await test_async_client.get('/api/inventory/brands')
request, response = await async_client.get('/api/inventory/brands')
assert response.status_code == HTTPStatus.OK
inventory = dict(name='new name')
request, response = await test_async_client.put(f'/api/inventory/{inventory_id}', content=json.dumps(inventory))
request, response = await async_client.put(f'/api/inventory/{inventory_id}', content=json.dumps(inventory))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = await test_async_client.delete(f'/api/inventory/{inventory_id}')
request, response = await async_client.delete(f'/api/inventory/{inventory_id}')
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = await test_async_client.get('/api/inventory')
request, response = await async_client.get('/api/inventory')
assert response.status_code == HTTPStatus.OK
assert len(response.json['inventories']) == 0

View file

@ -9,6 +9,7 @@ from pint import Quantity
from wrolpi.common import Base
from wrolpi.db import get_db_session
from wrolpi.switches import await_switches
from wrolpi.test.common import PytestCase
from .. import init
from ..common import sum_by_key, get_inventory_by_category, get_inventory_by_subcategory, get_inventory_by_name, \
@ -273,7 +274,8 @@ def test_no_inventories(test_session, test_directory):
pass
def test_inventories_version(test_async_client, test_session, test_directory, init_test_inventory):
@pytest.mark.asyncio
async def test_inventories_version(async_client, test_session, test_directory, init_test_inventory):
"""You can't save over a newer version of an inventory."""
config = get_inventories_config()
@ -284,10 +286,12 @@ def test_inventories_version(test_async_client, test_session, test_directory, in
# Version is set to 1 on first save.
save_inventories_file()
await await_switches()
assert config.version == 1
# Version is incremented when saving.
save_inventories_file()
await await_switches()
assert config.version == 2
# Version is greater than what will be saved.
@ -299,7 +303,7 @@ def test_inventories_version(test_async_client, test_session, test_directory, in
@pytest.mark.asyncio
async def test_inventories_config(test_async_client, test_session, test_directory, init_test_inventory):
async def test_inventories_config(async_client, test_session, test_directory, init_test_inventory):
for item in TEST_ITEMS:
item = Item(**item)
test_session.add(item)

View file

@ -1,6 +1,8 @@
import asyncio
from typing import List, Tuple
from sqlalchemy.orm import Session
from modules.videos.models import Video
from wrolpi.common import logger, limit_concurrent, register_modeler, register_refresh_cleanup
from wrolpi.db import get_db_curs, get_db_session
@ -33,14 +35,19 @@ async def video_modeler():
video_id = None
try:
if not video:
video = Video(file_group=file_group)
video = Video(file_group=file_group, file_group_id=file_group.id)
session.add(video)
session.flush([video])
video_id = video.id
if not Session.object_session(video):
session.add(video)
video.flush()
# Extract ffprobe data.
await video.get_ffprobe_json()
video.flush(session)
# Validate and index subtitles.
video.validate()
video.validate(session)
processed += 1
except Exception as e:
if PYTEST:
raise

View file

@ -8,8 +8,7 @@ from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.api_utils import json_response
from wrolpi.common import logger, wrol_mode_check, make_media_directory, \
get_media_directory, run_after, get_relative_to_media_directory
from wrolpi.common import logger, wrol_mode_check, get_media_directory, get_relative_to_media_directory
from wrolpi.downloader import download_manager
from wrolpi.events import Events
from wrolpi.schema import JSONErrorResponse
@ -74,11 +73,11 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
@openapi.response(HTTPStatus.NO_CONTENT)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
@validate(schema.ChannelPutRequest)
@run_after(save_channels_config)
@wrol_mode_check
async def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest):
body.directory = pathlib.Path(body.directory) if body.directory else None
channel = await lib.update_channel(data=body, channel_id=channel_id)
save_channels_config.activate_switch()
return response.raw('', HTTPStatus.NO_CONTENT,
headers={'Location': f'/api/videos/channels/{channel.id}'})

View file

@ -6,7 +6,7 @@ from sqlalchemy import or_, func, desc, asc
from sqlalchemy.orm import Session
from wrolpi import flags
from wrolpi.common import run_after, logger, \
from wrolpi.common import logger, \
get_media_directory, wrol_mode_check, background_task
from wrolpi.db import get_db_curs, optional_session, get_db_session
from wrolpi.downloader import save_downloads_config, download_manager
@ -114,7 +114,6 @@ def get_channel(session: Session, *, channel_id: int = None, source_id: str = No
return channel
@run_after(save_channels_config)
@optional_session
async def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
"""Update a Channel's DB record"""
@ -150,10 +149,11 @@ async def update_channel(session: Session, *, data: schema.ChannelPutRequest, ch
else:
background_task(_())
save_channels_config.activate_switch()
return channel
@run_after(save_channels_config)
@optional_session
def create_channel(session: Session, data: schema.ChannelPostRequest, return_dict: bool = True) -> Union[Channel, dict]:
"""
@ -180,10 +180,11 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic
session.commit()
save_channels_config.activate_switch()
return channel.dict() if return_dict else channel
@run_after(save_channels_config)
@optional_session(commit=True)
def delete_channel(session: Session, *, channel_id: int):
channel = Channel.find_by_id(channel_id, session=session)
@ -191,6 +192,8 @@ def delete_channel(session: Session, *, channel_id: int):
channel_dict = channel.dict()
channel.delete_with_videos()
save_channels_config.activate_switch()
return channel_dict
@ -230,8 +233,8 @@ async def create_channel_download(channel_id: int, url: str, frequency: int, set
download = channel.get_or_create_download(url, frequency, session, reset_attempts=True)
download.settings = settings
save_channels_config()
await save_downloads_config()
save_channels_config.activate_switch()
save_downloads_config.activate_switch()
return download
@ -250,8 +253,8 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f
download_manager.remove_from_skip_list(url)
save_channels_config()
await save_downloads_config()
save_channels_config.activate_switch()
save_downloads_config.activate_switch()
return download
@ -287,6 +290,8 @@ async def tag_channel(tag_name: str | None, directory: pathlib.Path | None, chan
await coro
else:
background_task(coro)
else:
save_channels_config.activate_switch()
else:
logger.info(f'Tagging {channel} with {tag_name}')

View file

@ -99,7 +99,7 @@ def test_video_file_name(test_session, simple_video, test_client):
@pytest.mark.asyncio
async def test_channel_conflicts(test_async_client, test_session, test_directory):
async def test_channel_conflicts(async_client, test_session, test_directory):
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory).mkdir()
new_channel = dict(
@ -109,7 +109,7 @@ async def test_channel_conflicts(test_async_client, test_session, test_directory
)
async def _post_channel(channel):
return await test_async_client.post('/api/videos/channels', content=json.dumps(channel))
return await async_client.post('/api/videos/channels', content=json.dumps(channel))
# Create it
request, response = await _post_channel(new_channel)
@ -189,7 +189,7 @@ def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_direc
@pytest.mark.asyncio
async def test_channel_download_requires_refresh(
test_async_client, test_session, mock_video_extract_info, download_channel, video_download_manager,
async_client, test_session, mock_video_extract_info, download_channel, video_download_manager,
video_factory, events_history):
"""A Channel cannot be downloaded until it has been refreshed.
@ -206,7 +206,7 @@ async def test_channel_download_requires_refresh(
assert channel.refreshed == expected
assert bool(channel.info_json) == expected
await test_async_client.sanic_app.dispatch('wrolpi.download.')
await async_client.sanic_app.dispatch('wrolpi.download.')
assert_refreshed(False)
test_session.commit()
@ -259,7 +259,7 @@ def test_channel_by_id(test_session, test_client, simple_channel, simple_video):
@pytest.mark.asyncio
async def test_channel_crud(test_session, test_async_client, test_directory, test_download_manager):
async def test_channel_crud(test_session, async_client, test_directory, test_download_manager):
channel_directory = test_directory / 'channel directory'
channel_directory.mkdir()
@ -270,15 +270,15 @@ async def test_channel_crud(test_session, test_async_client, test_directory, tes
)
# Channel doesn't exist
request, response = await test_async_client.get('/api/videos/channels/examplechannel1')
request, response = await async_client.get('/api/videos/channels/examplechannel1')
assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
# Create it
request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(new_channel))
request, response = await async_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.CREATED, response.content
location = response.headers['Location']
request, response = await test_async_client.get(location)
request, response = await async_client.get(location)
assert response.status_code == HTTPStatus.OK, response.json
created = response.json['channel']
assert created
@ -294,13 +294,13 @@ async def test_channel_crud(test_session, test_async_client, test_directory, tes
f'Channel directory is absolute: {created["directory"]}'
# Can't create it again
request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(new_channel))
request, response = await async_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.BAD_REQUEST
async def put_and_fetch(new_channel_):
request, response = await test_async_client.put(location, content=json.dumps(new_channel_))
request, response = await async_client.put(location, content=json.dumps(new_channel_))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = await test_async_client.get(location)
request, response = await async_client.get(location)
assert response.status_code == HTTPStatus.OK
channel = response.json['channel']
return channel
@ -315,22 +315,22 @@ async def test_channel_crud(test_session, test_async_client, test_directory, tes
assert channel['url'] == 'https://example.com/channel1'
# Can't update channel that doesn't exist
request, response = await test_async_client.put('/api/videos/channels/doesnt_exist',
request, response = await async_client.put('/api/videos/channels/doesnt_exist',
content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NOT_FOUND
# Delete the new channel
request, response = await test_async_client.delete(location)
request, response = await async_client.delete(location)
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Download).count() == 0
# Cant delete it again
request, response = await test_async_client.delete(location)
request, response = await async_client.delete(location)
assert response.status_code == HTTPStatus.NOT_FOUND
@pytest.mark.asyncio
async def test_change_channel_url(test_async_client, test_session, test_download_manager, download_channel,
async def test_change_channel_url(async_client, test_session, test_download_manager, download_channel,
test_download_manager_config):
# Change the Channel's URL.
download_channel.update({'url': 'https://example.com/new-url'})
@ -411,12 +411,12 @@ async def test_get_channel_videos_pagination(test_session, simple_channel, video
@pytest.mark.asyncio
async def test_create_channel_download_api(test_async_client, test_session, simple_channel, tag_factory):
async def test_create_channel_download_api(async_client, test_session, simple_channel, tag_factory):
tag = await tag_factory()
# Create Download.
body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}}
request, response = await test_async_client.post(
request, response = await async_client.post(
f'/api/videos/channels/{simple_channel.id}/download', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content
download: Download = test_session.query(Download).one()
@ -425,7 +425,7 @@ async def test_create_channel_download_api(test_async_client, test_session, simp
# Create another Download. Previous Downloads is untouched.
body = {'url': 'https://example.com/2', 'frequency': 55, 'settings': {'tag_names': [tag.name]}}
request, response = await test_async_client.post(
request, response = await async_client.post(
f'/api/videos/channels/{simple_channel.id}/download', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content
download1, download2 = test_session.query(Download).order_by(Download.url).all()
@ -436,7 +436,7 @@ async def test_create_channel_download_api(test_async_client, test_session, simp
# Change frequency of first Download.
body = {'url': 'https://example.com/1', 'frequency': 123, 'settings': {'title_include': 'foo,bar'}}
request, response = await test_async_client.put(
request, response = await async_client.put(
f'/api/videos/channels/{simple_channel.id}/download/{download1.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content
download1, download2 = test_session.query(Download).order_by(Download.url).all()
@ -447,14 +447,14 @@ async def test_create_channel_download_api(test_async_client, test_session, simp
assert len(simple_channel.downloads) == 2
# Deleting Download does not delete Channel.
request, response = await test_async_client.delete(f'/api/download/{download1.id}')
request, response = await async_client.delete(f'/api/download/{download1.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Download).count() == 1
assert test_session.query(Channel).count() == 1
@pytest.mark.asyncio
async def test_tag_channel(test_async_client, test_session, test_directory, channel_factory, tag_factory, video_factory,
async def test_tag_channel(async_client, test_session, test_directory, channel_factory, tag_factory, video_factory,
test_channels_config, test_download_manager, test_downloader):
"""A single Tag can be applied to a Channel."""
# Create channel directory in the usual videos directory.
@ -485,7 +485,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
assert [i.path for i in test_session.query(Directory)] == [channel.directory, ]
body = dict(tag_name=tag.name, directory='videos/Tag Name/Channel Name')
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
request, response = await async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT
channel = test_session.query(Channel).one()
@ -518,7 +518,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
# Remove tag, Channel/Videos should be moved back.
body = dict(tag_name=None, directory='videos/Channel Name')
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
request, response = await async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
channel = test_session.query(Channel).one()
assert channel.tag_name is None
@ -535,7 +535,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
# Channel can be Tagged, without moving directories.
body = dict(tag_name=tag.name)
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
request, response = await async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
channel = test_session.query(Channel).one()
assert channel.tag_name == 'Tag Name'
@ -550,7 +550,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
@pytest.mark.asyncio
async def test_move_channel(test_async_client, test_session, test_directory, channel_factory, tag_factory,
async def test_move_channel(async_client, test_session, test_directory, channel_factory, tag_factory,
video_factory, test_channels_config, test_download_manager, test_downloader):
"""A Channel can be moved."""
channel = channel_factory(name='Channel Name')
@ -566,7 +566,7 @@ async def test_move_channel(test_async_client, test_session, test_directory, cha
name=channel.name,
directory=str(test_directory / 'videos/New Directory'),
)
request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body)
request, response = await async_client.put(f'/api/videos/channels/{channel.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
channel = Channel.find_by_id(channel.id)
assert str(channel.directory) == str(test_directory / 'videos/New Directory')
@ -576,7 +576,7 @@ async def test_move_channel(test_async_client, test_session, test_directory, cha
@pytest.mark.asyncio
async def test_move_channel_after_terminal_move(test_async_client, test_session, test_directory, channel_factory,
async def test_move_channel_after_terminal_move(async_client, test_session, test_directory, channel_factory,
tag_factory, video_factory, test_channels_config, test_download_manager,
test_downloader):
"""A Channel can be moved in the UI after being moved in the Terminal."""
@ -594,7 +594,7 @@ async def test_move_channel_after_terminal_move(test_async_client, test_session,
name=channel.name,
directory=str(test_directory / 'videos/New Directory'),
)
request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body)
request, response = await async_client.put(f'/api/videos/channels/{channel.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
channel = Channel.find_by_id(channel.id)
vid1 = test_session.query(Video).one()
@ -603,7 +603,7 @@ async def test_move_channel_after_terminal_move(test_async_client, test_session,
@pytest.mark.asyncio
async def test_search_tagged_channels(test_async_client, test_session, channel_factory, tag_factory,
async def test_search_tagged_channels(async_client, test_session, channel_factory, tag_factory,
video_factory):
"""Tagged Channels can be searched."""
tag = await tag_factory()
@ -611,7 +611,7 @@ async def test_search_tagged_channels(test_async_client, test_session, channel_f
# Can search when empty.
body = dict(tag_names=[tag.name])
request, response = await test_async_client.post('/api/videos/channels/search', json=body)
request, response = await async_client.post('/api/videos/channels/search', json=body)
assert response.status_code == HTTPStatus.OK
assert len(response.json['channels']) == 0, 'No Channels should be tagged'
@ -620,7 +620,7 @@ async def test_search_tagged_channels(test_async_client, test_session, channel_f
test_session.commit()
body = dict(tag_names=[tag.name])
request, response = await test_async_client.post('/api/videos/channels/search', json=body)
request, response = await async_client.post('/api/videos/channels/search', json=body)
assert response.status_code == HTTPStatus.OK
assert len(response.json['channels']) == 1, 'Only one Channel is tagged'
assert response.json['channels'][0]['id'] == channel1.id

View file

@ -8,6 +8,7 @@ from modules.videos.channel import lib
from modules.videos.errors import UnknownChannel
from modules.videos.lib import save_channels_config, import_channels_config
from modules.videos.models import Channel
from wrolpi.switches import await_switches
@pytest.mark.parametrize('params', [
@ -66,7 +67,7 @@ def test_get_channel(test_session, test_directory, channel_factory):
@pytest.mark.asyncio
async def test_channels_no_url(test_async_client, test_session, test_directory, test_channels_config):
async def test_channels_no_url(async_client, test_session, test_directory, test_channels_config):
"""Test that a Channel's URL is coerced to None if it is empty."""
channel1_directory = test_directory / 'channel1'
channel1_directory.mkdir()
@ -91,7 +92,7 @@ async def test_channels_no_url(test_async_client, test_session, test_directory,
assert len(channels) == 2
assert all(i.url is None for i in channels), [(i.name, i.url) for i in channels]
save_channels_config()
save_channels_config.activate_switch()
import_channels_config()
@ -128,20 +129,21 @@ async def test_search_channels_by_name(test_session, channel_factory, video_fact
@pytest.mark.asyncio
async def test_tag_channel_existing(test_async_client, test_session, test_directory, channel_factory, tag_factory,
async def test_tag_channel_existing(async_client, test_session, test_directory, channel_factory, tag_factory,
video_factory, test_channels_config):
"""A Channel can already exist in a Tag's directory and still be tagged."""
"""A Channel already in a Tag's directory can still be tagged."""
channel_directory = test_directory / 'videos/one/Channel Name'
channel = channel_factory(name='Channel Name', download_frequency=120, directory=channel_directory)
video_factory(channel_id=channel.id)
test_session.commit()
await await_switches()
assert str(channel.directory).startswith(f'{test_directory}/videos/one/')
tag = await tag_factory()
await lib.tag_channel(tag.name, channel_directory, channel.id, test_session)
await await_switches()
# Channel's tag is saved to the config.
save_channels_config(test_session)
with test_channels_config.open() as fh:
config = yaml.load(fh, Loader=yaml.Loader)
assert config['channels'][0]['tag_name'] == tag.name

View file

@ -20,7 +20,7 @@ def test_delete_channel_no_url(test_session, test_client, channel_factory):
@pytest.mark.asyncio
async def test_delete_channel_with_videos(test_session, test_async_client, channel_factory, video_factory):
async def test_delete_channel_with_videos(test_session, async_client, channel_factory, video_factory):
"""Videos are disowned when their Channel is deleted."""
channel = channel_factory()
video = video_factory(channel_id=channel.id)
@ -28,7 +28,7 @@ async def test_delete_channel_with_videos(test_session, test_async_client, chann
test_session.commit()
# Delete the Channel.
request, response = await test_async_client.delete(f'/api/videos/channels/{channel.id}')
request, response = await async_client.delete(f'/api/videos/channels/{channel.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
# Video still exists, but has no channel.
@ -39,7 +39,7 @@ async def test_delete_channel_with_videos(test_session, test_async_client, chann
@pytest.mark.asyncio
async def test_nested_channel_directories(test_session, test_async_client, test_directory, channel_factory,
async def test_nested_channel_directories(test_session, async_client, test_directory, channel_factory,
video_factory):
"""Channel directories cannot contain another Channel's directory."""
(test_directory / 'foo').mkdir()
@ -54,7 +54,7 @@ async def test_nested_channel_directories(test_session, test_async_client, test_
name='channel 2',
directory=str(channel2_directory),
)
request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(content))
request, response = await async_client.post('/api/videos/channels', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json['cause'] and \
response.json['cause']['message'] == 'The directory is already used by another channel.'
@ -64,7 +64,7 @@ async def test_nested_channel_directories(test_session, test_async_client, test_
name='channel 3',
directory=str(test_directory),
)
request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(content))
request, response = await async_client.post('/api/videos/channels', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json['cause'] and \
response.json['cause']['message'] == 'The directory is already used by another channel.'

View file

@ -246,7 +246,7 @@ def video_with_search_factory(test_session, test_directory, video_file_factory):
@pytest.fixture
def assert_video_search(test_async_client):
def assert_video_search(async_client):
"""A fixture which performs a video search request against the API.
If assert_* params are passed, the response is checked."""
@ -276,7 +276,7 @@ def assert_video_search(test_async_client):
if channel_id is not None:
content['channel_id'] = channel_id
request, response = await test_async_client.post('/api/videos/search', content=json.dumps(content))
request, response = await async_client.post('/api/videos/search', content=json.dumps(content))
assert response.json
assert response.status_code == HTTPStatus.OK

View file

@ -3,12 +3,10 @@ import dataclasses
import html
import pathlib
import re
import traceback
from typing import Generator, Type
from typing import Tuple, Optional
import pytz
from sqlalchemy import or_
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
@ -18,10 +16,11 @@ from wrolpi.captions import extract_captions
from wrolpi.common import ConfigFile, limit_concurrent, get_wrolpi_config, extract_domain, \
escape_file_name, get_media_directory, background_task, Base
from wrolpi.dates import Seconds, from_timestamp
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.db import get_db_curs, get_db_session
from wrolpi.downloader import Download
from wrolpi.errors import UnknownDirectory
from wrolpi.files.lib import split_path_stem_and_suffix
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST
from .common import is_valid_poster, convert_image, \
generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
@ -364,19 +363,23 @@ def get_channels_config_from_db(session: Session) -> dict:
return dict(channels=channels)
@optional_session()
def save_channels_config(session: Session = None):
@register_switch_handler('save_channels_config')
def save_channels_config():
"""Get the Channel information from the DB, save it to the config."""
config = get_channels_config_from_db(session)
channels_config = get_channels_config()
with get_db_session() as session:
config = get_channels_config_from_db(session)
channels_config = get_channels_config()
if PYTEST and not channels_config:
logger.warning('Refusing to save channels config because test did not initialize a test config!')
return
channels_config.update(config)
logger.info('save_channels_config completed')
save_channels_config: ActivateSwitchMethod
channel_import_logger = logger.getChild('channel_import')
@ -391,7 +394,7 @@ async def fetch_channel_source_id(channel_id: int):
channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
else:
session.commit()
save_channels_config(session)
save_channels_config.activate_switch()
except Exception as e:
logger.error(f'Failed to get Channel source id of id={channel_id}', exc_info=e)

View file

@ -220,7 +220,7 @@ class Video(ModelHelper, Base):
return previous_video, next_video
def validate(self):
def validate(self, session: Session = None):
"""Perform a validation of this video and it's files."""
if not self.file_group.primary_path:
# Can't validate if there is no video file.
@ -245,7 +245,7 @@ class Video(ModelHelper, Base):
self.have_comments = True
# If this Video is in a Channel's directory, then it is part of that Channel.
session: Session = Session.object_session(self)
session: Session = session or Session.object_session(self)
if session and (channel := Channel.get_by_path(self.video_path.parent, session)):
self.channel = channel
logger.debug(f'{self} has Channel {channel}')
@ -371,7 +371,6 @@ class Video(ModelHelper, Base):
if not self.ffprobe_json:
from modules.videos.common import ffprobe_json
self.ffprobe_json = await ffprobe_json(self.video_path)
self.flush()
return self.ffprobe_json
@ -819,9 +818,9 @@ class Channel(ModelHelper, Base):
# Save configs before move, this is because move imports configs.
from modules.videos.lib import save_channels_config
await save_downloads_config(session)
save_channels_config(session)
save_tags_config(session)
save_downloads_config.activate_switch()
save_channels_config.activate_switch()
save_tags_config.activate_switch()
# Move the contents of the Channel directory into the destination directory.
logger.info(f'Moving {self} from {repr(str(old_directory))}')

View file

@ -14,7 +14,7 @@ from wrolpi.files.models import FileGroup
@pytest.mark.asyncio
async def test_refresh_videos_index(test_async_client, test_session, test_directory, video_factory):
async def test_refresh_videos_index(async_client, test_session, test_directory, video_factory):
"""The video modeler indexes video data into the Video's FileGroup."""
video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True)
test_session.commit()
@ -162,14 +162,14 @@ def test_api_download(test_session, test_client, test_directory):
@pytest.mark.asyncio
async def test_api_download_video_tags(test_session, test_async_client, tag_factory):
async def test_api_download_video_tags(test_session, async_client, tag_factory):
"""A user can request Tags for a video being downloaded."""
tag1 = await tag_factory()
tag2 = await tag_factory()
content = dict(urls=['https://example.com/video1', ], downloader='video',
settings={'tag_names': [tag1.name, tag2.name]})
request, response = await test_async_client.post('/api/download', content=json.dumps(content))
request, response = await async_client.post('/api/download', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
download = test_session.query(Download).one()
@ -264,7 +264,7 @@ async def test_search_videos(test_session, video_factory, assert_video_search, s
@pytest.mark.asyncio
async def test_delete_video(test_async_client, test_session, video_factory, test_download_manager):
async def test_delete_video(async_client, test_session, video_factory, test_download_manager):
"""When a Video record is deleted, the FileGroup should be deleted. The URL is added to the
global skip list so the video is not downloaded again."""
video = video_factory(with_video_file=True, with_poster_ext='png')
@ -272,7 +272,7 @@ async def test_delete_video(test_async_client, test_session, video_factory, test
files = video.file_group.my_paths()
test_session.commit()
request, response = await test_async_client.delete(f'/api/videos/video/{video.id}')
request, response = await async_client.delete(f'/api/videos/video/{video.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Video).count() == 0, 'Video was not deleted.'
@ -282,28 +282,28 @@ async def test_delete_video(test_async_client, test_session, video_factory, test
@pytest.mark.asyncio
async def test_format_videos_description(test_async_client, test_session, test_directory, channel_factory, tag_factory):
async def test_format_videos_description(async_client, test_session, test_directory, channel_factory, tag_factory):
channel = channel_factory(name='Channel Name')
tag = await tag_factory()
test_session.commit()
body = dict()
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
request, response = await async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos'
body = dict(channel_id=channel.id)
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
request, response = await async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos/Channel Name'
body = dict(channel_id=channel.id, tag_name=tag.name)
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
request, response = await async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos/one/Channel Name'
get_wrolpi_config().videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s'
body = dict(channel_id=channel.id, tag_name=tag.name)
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
request, response = await async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos/one/example.com/Channel Name'

View file

@ -11,6 +11,7 @@ from modules.videos.models import Channel, Video
from wrolpi.common import get_absolute_media_path, get_wrolpi_config
from wrolpi.downloader import Download, DownloadFrequency
from wrolpi.files import lib as files_lib
from wrolpi.switches import await_switches
from wrolpi.vars import PROJECT_DIR
from .. import common
from ..common import convert_image, update_view_counts_and_censored, get_video_duration, generate_video_poster, \
@ -135,7 +136,7 @@ def test_generate_video_poster(video_file):
@pytest.mark.asyncio
async def test_import_channel_downloads(test_async_client, test_session, channel_factory, test_channels_config,
async def test_import_channel_downloads(async_client, test_session, channel_factory, test_channels_config,
tag_factory):
"""Importing the Channels' config should create any missing download records"""
channel1 = channel_factory(source_id='foo', url='https://example.com/channel1')
@ -155,6 +156,8 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
# Two Channels, but neither have Downloads.
save_channels_config()
await await_switches()
# Import after config is saved.
import_channels_config()
assert not channel1.downloads
assert not channel2.downloads
@ -167,6 +170,7 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
{'downloads': [
{'url': 'https://example.com/channel1', 'frequency': DownloadFrequency.weekly}
]})
await await_switches()
# Download record is created on import.
import_channels_config()
@ -189,6 +193,7 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
save_channels_config()
Download.find_by_url(channel2.url).delete(add_to_skip_list=False)
test_session.commit()
await await_switches()
# Missing Download is re-created on import.
import_channels_config()
@ -204,6 +209,7 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
channel2.get_or_create_download('https://example.com/channel2', 60, session=test_session)
test_session.commit()
save_channels_config()
await await_switches()
# Check config is written to match new URLs.
config = get_channels_config()
assert len(config.channels) == 2
@ -242,7 +248,7 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
@pytest.mark.asyncio
async def test_import_channel_delete_missing_channels(
test_async_client, test_session, channel_factory, test_channels_config):
async_client, test_session, channel_factory, test_channels_config):
"""The Channel import function deletes any Channels that are not in the config."""
# Create a DB and config with two Channels.
channel1 = channel_factory(source_id='foo')
@ -253,6 +259,7 @@ async def test_import_channel_delete_missing_channels(
test_session.delete(channel1)
test_session.delete(channel2)
test_session.commit()
await await_switches()
# Importing the config creates two Channels.
import_channels_config()
@ -265,6 +272,7 @@ async def test_import_channel_delete_missing_channels(
config_dict = config.dict()
config_dict['channels'] = [i for i in config.channels if i['directory'] != str(channel2.directory)]
config.update(config_dict)
await await_switches()
assert str(channel1.directory) in test_channels_config.read_text()
assert str(channel2.directory) not in test_channels_config.read_text()
@ -282,7 +290,7 @@ async def test_import_channel_delete_missing_channels(
@pytest.mark.asyncio
async def test_ffprobe_json(test_async_client, video_file, corrupted_video_file):
async def test_ffprobe_json(async_client, video_file, corrupted_video_file):
content = await common.ffprobe_json(video_file)
assert not content['chapters']
assert content['format']['duration'] == '5.312000'

View file

@ -178,7 +178,7 @@ def test_link_channel_and_downloads(test_session, channel_factory, test_download
@pytest.mark.asyncio
def test_link_channel_and_downloads_migration(test_async_client, test_session, channel_factory, test_download_manager):
def test_link_channel_and_downloads_migration(async_client, test_session, channel_factory, test_download_manager):
"""Test the 8d0d81bc9c34_channel_channel_downloads.py migration."""
# A simple Channel which has a download for its URL.
channel1 = channel_factory(url='https://example.com/channel1')
@ -220,7 +220,7 @@ def test_link_channel_and_downloads_migration(test_async_client, test_session, c
@pytest.mark.asyncio
async def test_format_videos_destination(test_async_client, test_directory):
async def test_format_videos_destination(async_client, test_directory):
"""Videos destination is formatted according to the WROLPiConfig."""
wrolpi_config = get_wrolpi_config()

View file

@ -19,7 +19,7 @@ async def test_ffprobe_stream_methods(simple_video):
@pytest.mark.asyncio
async def test_channel_move(test_async_client, test_session, test_directory, channel_factory, video_factory):
async def test_channel_move(async_client, test_session, test_directory, channel_factory, video_factory):
"""A Channel can be moved to another directory, any files in the Channel's directory are moved."""
channel = channel_factory(name='Channel Name')
video = video_factory(title='Vid', channel_id=channel.id)

View file

@ -30,7 +30,7 @@ def test_delete_video_no_channel(test_session, simple_video):
@pytest.mark.asyncio
async def test_delete_video_with_tag(test_async_client, test_session, video_factory, tag_factory):
async def test_delete_video_with_tag(async_client, test_session, video_factory, tag_factory):
"""You cannot delete a video with a tag."""
video = video_factory(with_video_file=True, with_poster_ext='png')
tag = await tag_factory()
@ -47,7 +47,7 @@ async def test_delete_video_with_tag(test_async_client, test_session, video_fact
@pytest.mark.asyncio
async def test_video_channel_refresh(test_async_client, test_session, test_directory, channel_factory, video_factory):
async def test_video_channel_refresh(async_client, test_session, test_directory, channel_factory, video_factory):
"""A Video will be associated with a Channel when it's files are in that Channel's directory."""
# Create a video file in this channel's directory.
channel = channel_factory()
@ -134,7 +134,7 @@ async def test_delete_duplicate_video(test_session, channel_factory, video_facto
@pytest.mark.asyncio
async def test_delete_renamed_video(test_async_client, test_session, channel_factory, video_factory, tag_factory):
async def test_delete_renamed_video(async_client, test_session, channel_factory, video_factory, tag_factory):
"""If duplicate Video's exist, delete all the videos that do not have the new title."""
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_Some new title.mp4'

View file

@ -12,6 +12,7 @@ from wrolpi.events import Events
from wrolpi.schema import JSONErrorResponse
from . import lib
from .. import schema
from ..lib import save_channels_config
video_bp = Blueprint('Video', '/api/videos')
@ -59,7 +60,6 @@ async def search_videos(_: Request, body: schema.VideoSearchRequest):
@openapi.response(HTTPStatus.NO_CONTENT)
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@wrol_mode_check
@run_after(lib.save_channels_config)
async def video_delete(_: Request, video_ids: str):
try:
video_ids = [int(i) for i in str(video_ids).split(',')]
@ -68,5 +68,6 @@ async def video_delete(_: Request, video_ids: str):
lib.delete_videos(*video_ids)
save_channels_config.activate_switch()
Events.send_deleted(f'Deleted {len(video_ids)} videos')
return response.empty()

View file

@ -9,7 +9,7 @@ from modules.videos.models import Video
from wrolpi.dates import now
def test_get_video_prev_next(test_async_client, test_session, channel_factory, video_factory):
def test_get_video_prev_next(async_client, test_session, channel_factory, video_factory):
"""
Test that the previous and next videos will be retrieved when fetching a video.
"""
@ -69,7 +69,7 @@ def test_get_video_prev_next(test_async_client, test_session, channel_factory, v
raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e
def test_get_video_prev_next_no_upload_date(test_async_client, test_session, video_factory, channel_factory):
def test_get_video_prev_next_no_upload_date(async_client, test_session, video_factory, channel_factory):
"""
Previous and Next Videos should be those next to a Video file when videos do not have upload dates.
"""
@ -108,7 +108,7 @@ def test_get_video_prev_next_no_upload_date(test_async_client, test_session, vid
@pytest.mark.asyncio
async def test_video_delete(test_async_client, test_session, channel_factory, video_factory, test_download_manager):
async def test_video_delete(async_client, test_session, channel_factory, video_factory, test_download_manager):
"""Video.delete() removes the video's files, but leave the DB record."""
channel1, channel2 = channel_factory(), channel_factory()
vid1 = video_factory(channel_id=channel1.id, with_video_file=True, with_caption_file=True,
@ -125,7 +125,7 @@ async def test_video_delete(test_async_client, test_session, channel_factory, vi
assert vid1_video_path.is_file() and vid1_caption_path.is_file()
assert vid2_video_path.is_file() and vid2_info_json_path.is_file()
request, response = await test_async_client.delete(f'/api/videos/video/{vid1.id}')
request, response = await async_client.delete(f'/api/videos/video/{vid1.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
# Video was added to skip list.
@ -135,7 +135,7 @@ async def test_video_delete(test_async_client, test_session, channel_factory, vi
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
assert vid2_video_path.is_file() and vid2_info_json_path.is_file()
request, response = await test_async_client.delete(f'/api/videos/video/{vid2.id}')
request, response = await async_client.delete(f'/api/videos/video/{vid2.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_download_manager.is_skipped(vid1.file_group.url, vid2.file_group.url)
@ -144,12 +144,12 @@ async def test_video_delete(test_async_client, test_session, channel_factory, vi
assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False
# 3 does not exist.
request, response = await test_async_client.delete(f'/api/videos/video/3')
request, response = await async_client.delete(f'/api/videos/video/3')
assert response.status_code == HTTPStatus.NOT_FOUND
@pytest.mark.asyncio
async def test_wrol_mode(test_async_client, simple_channel, simple_video, wrol_mode_fixture,
async def test_wrol_mode(async_client, simple_channel, simple_video, wrol_mode_fixture,
test_download_manager, tag_factory):
"""Many methods are blocked when WROL Mode is enabled."""
channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory)))
@ -158,29 +158,29 @@ async def test_wrol_mode(test_async_client, simple_channel, simple_video, wrol_m
await wrol_mode_fixture(True)
# Can't create, update, or delete a channel.
_, resp = await test_async_client.post('/api/videos/channels', content=channel)
_, resp = await async_client.post('/api/videos/channels', content=channel)
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == 'WROL_MODE_ENABLED'
_, resp = await test_async_client.put(f'/api/videos/channels/{simple_channel.id}', content=channel)
_, resp = await async_client.put(f'/api/videos/channels/{simple_channel.id}', content=channel)
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == 'WROL_MODE_ENABLED'
_, resp = await test_async_client.delete(f'/api/videos/channels/{simple_channel.id}')
_, resp = await async_client.delete(f'/api/videos/channels/{simple_channel.id}')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == 'WROL_MODE_ENABLED'
# Can't delete a video
_, resp = await test_async_client.delete(f'/api/videos/video/{simple_video.id}')
_, resp = await async_client.delete(f'/api/videos/video/{simple_video.id}')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == 'WROL_MODE_ENABLED'
# Can't refresh
_, resp = await test_async_client.post('/api/files/refresh')
_, resp = await async_client.post('/api/files/refresh')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == 'WROL_MODE_ENABLED'
# THE REST OF THESE METHODS ARE ALLOWED
content = dict(file_group_id=simple_video.file_group_id, tag_name=tag.name)
_, resp = await test_async_client.post('/api/files/tag', content=json.dumps(content))
_, resp = await async_client.post('/api/files/tag', content=json.dumps(content))
assert resp.status_code == HTTPStatus.CREATED
assert test_download_manager.stopped.is_set()

View file

@ -100,14 +100,14 @@ def zim_factory(test_session, zim_path_factory, test_directory):
@pytest.fixture
def assert_zim_search(test_async_client, test_directory):
def assert_zim_search(async_client, test_directory):
from http import HTTPStatus
async def _(search_str: str, zim_id: int, expected: Dict, tag_names: List[str] = [],
expected_status_code: int = HTTPStatus.OK):
content = dict(search_str=search_str, tag_names=tag_names)
assert isinstance(zim_id, int), 'This fixture only supports Zim ids of type integer'
request, response = await test_async_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content))
request, response = await async_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content))
assert response.status_code == expected_status_code
result = response.json['zim']

View file

@ -192,7 +192,7 @@ class Zim(Base):
else Tag.get_by_id(tag_id_or_name, session)
tag_zim_entry = TagZimEntry(tag=tag, zim=self, zim_entry=zim_entry)
session.add(tag_zim_entry)
tags.save_tags_config()
tags.save_tags_config.activate_switch()
return tag_zim_entry
def untag_entry(self, tag_id_or_name: int | str, zim_entry: str):
@ -207,7 +207,7 @@ class Zim(Base):
session.delete(tag_zim_entry)
else:
raise UnknownZimTagEntry(f'Could not find at {repr(str(zim_entry))}')
tags.save_tags_config()
tags.save_tags_config.activate_switch()
@staticmethod
def _entries_with_tags_process(limit: int, offset: int, params: dict, session: Session, tags_sub_select: str):

View file

@ -9,7 +9,7 @@ from wrolpi.downloader import Download
@pytest.mark.asyncio
async def test_zim_search(test_async_client, test_session, test_zim, tag_factory, assert_zim_search):
async def test_zim_search(async_client, test_session, test_zim, tag_factory, assert_zim_search):
"""Entries can be tagged and retrieved using their containing Zim."""
tag1, tag2 = await tag_factory(), await tag_factory()
test_zim.tag_entry(tag1.name, 'one')
@ -80,7 +80,7 @@ async def test_zim_search(test_async_client, test_session, test_zim, tag_factory
@pytest.mark.asyncio
async def test_entries_by_tag(test_async_client, test_session, test_zim, tag_factory):
async def test_entries_by_tag(async_client, test_session, test_zim, tag_factory):
"""Entries can be tagged and retrieved using their containing Zim."""
tag1, tag2 = await tag_factory(), await tag_factory()
test_zim.tag_entry(tag1.name, 'one')
@ -99,37 +99,37 @@ async def test_entries_by_tag(test_async_client, test_session, test_zim, tag_fac
@pytest.mark.asyncio
async def test_zim_crud(test_async_client, test_session, test_directory, test_zim):
request, response = await test_async_client.get('/api/zim')
async def test_zim_crud(async_client, test_session, test_directory, test_zim):
request, response = await async_client.get('/api/zim')
assert response.status_code == HTTPStatus.OK
assert len(response.json['zims']) == 1
assert response.json['zims'][0]['path'] == test_zim.path.name
request, response = await test_async_client.delete('/api/zim/1')
request, response = await async_client.delete('/api/zim/1')
assert response.status_code == HTTPStatus.NO_CONTENT
assert not test_zim.path.exists()
assert test_session.query(Zim).count() == 0
@pytest.mark.asyncio
async def test_zim_tag_and_untag(test_async_client, test_session, test_zim, tag_factory):
async def test_zim_tag_and_untag(async_client, test_session, test_zim, tag_factory):
"""Zim entries can be Tagged and Untagged."""
tag1, tag2 = await tag_factory(), await tag_factory()
test_session.commit()
# Entries can be tagged in the API.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
request, response = await test_async_client.post('/api/zim/tag', content=json.dumps(content))
request, response = await async_client.post('/api/zim/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
# Entries can be untagged in the API.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content))
request, response = await async_client.post('/api/zim/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
# Cannot untag twice.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content))
request, response = await async_client.post('/api/zim/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NOT_FOUND
@ -199,7 +199,7 @@ def test_get_zim_entry(test_session, test_client, test_zim):
@pytest.mark.asyncio
async def test_find_outdated_zim_files(test_session, test_directory, test_zim_bytes, test_async_client):
async def test_find_outdated_zim_files(test_session, test_directory, test_zim_bytes, async_client):
"""Outdated Zim files are found and reported. They can automatically be deleted."""
# Can search empty directory.
outdated, current = lib.find_outdated_zim_files()
@ -217,11 +217,11 @@ async def test_find_outdated_zim_files(test_session, test_directory, test_zim_by
asdf.write_bytes(test_zim_bytes)
empty = test_directory / 'zims/wikipedia_en_all_maxi_2023-03.zim' # empty files should be ignored.
empty.touch()
request, response = await test_async_client.get('/api/zim/outdated')
request, response = await async_client.get('/api/zim/outdated')
assert response.status_code == HTTPStatus.OK
assert response.json['outdated'] == [str(wikipedia_one.relative_to(test_directory)), ]
request, response = await test_async_client.delete('/api/zim/outdated')
request, response = await async_client.delete('/api/zim/outdated')
assert response.status_code == HTTPStatus.NO_CONTENT
# Only the outdated was deleted.
assert wikibooks_one.is_file()
@ -232,12 +232,12 @@ async def test_find_outdated_zim_files(test_session, test_directory, test_zim_by
@pytest.mark.asyncio
async def test_search_estimates(test_session, test_async_client, zim_factory):
async def test_search_estimates(test_session, async_client, zim_factory):
zim_factory('test zim')
test_session.commit()
body = dict(search_str='one')
request, response = await test_async_client.post('/api/zim/search_estimates', json=body)
request, response = await async_client.post('/api/zim/search_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert len(response.json['zims_estimates']) == 1
assert response.json['zims_estimates'][0]['estimate'] == 2

View file

@ -12,10 +12,11 @@ from wrolpi.common import DownloadFileInfo, get_wrolpi_config
from wrolpi.downloader import Download, import_downloads_config, save_downloads_config
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.switches import await_switches
@pytest.mark.asyncio
async def test_get_zim(test_async_client, test_session, zim_path_factory):
async def test_get_zim(async_client, test_session, zim_path_factory):
zim_path_factory()
await files_lib.refresh_files()
@ -26,7 +27,7 @@ async def test_get_zim(test_async_client, test_session, zim_path_factory):
@pytest.mark.asyncio
async def test_zim_get_entry(test_async_client, test_session, zim_path_factory):
async def test_zim_get_entry(async_client, test_session, zim_path_factory):
zim_path_factory()
await files_lib.refresh_files()
@ -50,14 +51,14 @@ async def test_zim_get_entry(test_async_client, test_session, zim_path_factory):
@pytest.mark.asyncio
async def test_zim_bad_entry(test_async_client, test_session, test_zim, tag_factory):
async def test_zim_bad_entry(async_client, test_session, test_zim, tag_factory):
tag1 = await tag_factory()
with pytest.raises(UnknownZimEntry):
test_zim.tag_entry(tag1.name, 'path does not exist')
@pytest.mark.asyncio
async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, tag_factory):
async def test_zim_get_entries_tags(async_client, test_session, test_zim, tag_factory):
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
test_zim.tag_entry(tag1.name, 'one')
test_zim.tag_entry(tag2.name, 'one')
@ -69,7 +70,7 @@ async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, t
@pytest.mark.asyncio
async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory, tag_factory):
async def test_tag_zim_entry_model(async_client, test_session, zim_factory, tag_factory):
tag = await tag_factory()
tze = zim_factory('zim1').tag_entry(tag.name, 'one')
# Test TagZimEntry.__repr__
@ -77,7 +78,7 @@ async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory,
@pytest.mark.asyncio
async def test_zim_entries_with_tags(test_async_client, test_session, zim_factory, tag_factory):
async def test_zim_entries_with_tags(async_client, test_session, zim_factory, tag_factory):
"""Can fetch Zim Entries that have Tags. Can filter only those entries that are tagged on a specific Zim."""
tag1, tag2 = await tag_factory(), await tag_factory()
zim1, zim2 = zim_factory('zim1'), zim_factory('zim2')
@ -126,7 +127,8 @@ def test_get_unique_paths():
@pytest.mark.asyncio
async def test_zim_tags_config(test_session, test_directory, test_zim, tag_factory, test_tags_config, fake_now):
async def test_zim_tags_config(async_client, test_session, test_directory, test_zim, tag_factory, test_tags_config,
fake_now):
fake_now(datetime.datetime(2000, 1, 1, 0, 0, 0, 1))
config = tags.get_tags_config()
assert not config.tag_zims
@ -137,6 +139,7 @@ async def test_zim_tags_config(test_session, test_directory, test_zim, tag_facto
test_zim.tag_entry(tag2.name, 'one')
test_zim.tag_entry(tag1.name, 'two')
test_session.commit()
await await_switches()
# Tags are created in empty database.
assert {i[0] for i in test_session.query(tags.Tag.id)} == {1, 2}
@ -150,6 +153,7 @@ async def test_zim_tags_config(test_session, test_directory, test_zim, tag_facto
# Remove a tag, the config should change.
test_zim.untag_entry(tag1.name, 'one')
test_session.commit()
await await_switches()
config = tags.get_tags_config()
assert config.tag_zims == [
@ -246,7 +250,7 @@ async def test_zim_download(test_session, kiwix_download_zim, test_directory, te
@pytest.mark.asyncio
async def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_factory, test_tags_config):
async def test_zim_tag_migration(async_client, test_session, test_directory, zim_factory, tag_factory, test_tags_config):
"""When an outdated Zim file is deleted, the Tags should be moved to the latest Zim."""
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # The outdated Zim.
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim')
@ -261,6 +265,7 @@ async def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_
zim3.tag_entry(tag2.name, 'one')
test_session.commit()
assert test_session.query(TagZimEntry).count() == 4
await await_switches()
# All Zims have been tagged.
tags_config = tags.get_tags_config()
@ -352,7 +357,7 @@ async def test_zim_subscription_download_import(test_session, test_downloader_co
assert download.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
# Save config file, delete all entries.
await save_downloads_config()
save_downloads_config()
test_session.query(Download).delete()
test_session.query(ZimSubscription).delete()
test_session.commit()