mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
FileWorker: Async queue-based file synchronization system
Introduces FileWorker for background file operations with dual-queue architecture (public/private). Handles refresh, move, and rename tasks with progress tracking, job completion monitoring, and batch processing. Compares DB FileGroups to filesystem state using FileGroupDiff to detect new/modified/deleted files. Designed for memory efficiency on Raspberry Pi.
This commit is contained in:
parent
1d9b794fca
commit
173284699d
49 changed files with 4256 additions and 861 deletions
|
|
@ -48,9 +48,9 @@ function FlagsMessages() {
|
|||
let refreshingMessage;
|
||||
let refreshRequiredMessage;
|
||||
|
||||
// Do not tell the maintainer to refresh the files if they are already refreshing.
|
||||
if (flags.refreshing) {
|
||||
// Actively refreshing.
|
||||
// Do not tell the maintainer to refresh the files if the FileWorker is busy.
|
||||
if (flags.file_worker_busy) {
|
||||
// FileWorker is busy.
|
||||
refreshingMessage = <Message icon>
|
||||
<Icon name='circle notched' loading/>
|
||||
<Message.Content>
|
||||
|
|
|
|||
|
|
@ -1174,10 +1174,10 @@ export async function deleteFile(paths) {
|
|||
}
|
||||
|
||||
export async function fetchFilesProgress() {
|
||||
const response = await apiGet(`${API_URI}/files/refresh_progress`);
|
||||
const response = await apiGet(`${API_URI}/files/worker_status`);
|
||||
if (response.ok) {
|
||||
const json = await response.json();
|
||||
return json['progress'];
|
||||
return json['status'];
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -542,50 +542,49 @@ export function FilesSearchView({
|
|||
</>
|
||||
}
|
||||
|
||||
function getPhaseLabel(status) {
|
||||
switch (status) {
|
||||
case 'counting':
|
||||
return 'Counting';
|
||||
case 'comparing':
|
||||
case 'upserting':
|
||||
case 'deleting':
|
||||
return 'Refreshing';
|
||||
case 'modeling':
|
||||
return 'Modeling';
|
||||
case 'indexing':
|
||||
return 'Indexing';
|
||||
case 'planning':
|
||||
return 'Move: Planning';
|
||||
case 'moving':
|
||||
return 'Move: Moving';
|
||||
case 'reverting':
|
||||
return 'Reverting';
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function FilesRefreshProgress() {
|
||||
const {progress} = useFilesProgressInterval();
|
||||
|
||||
if (!progress) {
|
||||
return;
|
||||
if (!progress || progress.status === 'idle') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const {
|
||||
discovery,
|
||||
refreshing,
|
||||
modeling,
|
||||
indexing,
|
||||
cleanup,
|
||||
indexed,
|
||||
unindexed,
|
||||
counted_files,
|
||||
total_file_groups,
|
||||
modeled
|
||||
} = progress;
|
||||
const {status, operation_total, operation_processed, operation_percent} = progress;
|
||||
const phaseLabel = getPhaseLabel(status);
|
||||
|
||||
if (refreshing) {
|
||||
// Default is Counting / Step 1. Move progress bar toward the middle to avoid clipping out of the screen.
|
||||
let params = {value: Math.floor(counted_files / 2), total: counted_files, progress: 'ratio'};
|
||||
let label = 'Refresh: Counting';
|
||||
|
||||
if (discovery) {
|
||||
params['value'] = total_file_groups;
|
||||
params['total'] = counted_files;
|
||||
label = 'Refresh: Discovery'
|
||||
} else if (modeling) {
|
||||
params['value'] = modeled;
|
||||
params['total'] = total_file_groups;
|
||||
label = 'Refresh: Modeling';
|
||||
} else if (indexing) {
|
||||
params['value'] = unindexed;
|
||||
params['total'] = unindexed + indexed;
|
||||
label = 'Refresh: Indexing';
|
||||
} else if (cleanup) {
|
||||
params['value'] = 3;
|
||||
params['total'] = 4;
|
||||
label = 'Refresh: Cleanup';
|
||||
}
|
||||
return <Progress active color='violet' {...params}>{label}</Progress>
|
||||
if (!phaseLabel) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let label = `Refresh: ${phaseLabel}`;
|
||||
if (operation_total > 0) {
|
||||
label = `${label} (${operation_processed.toLocaleString()} / ${operation_total.toLocaleString()})`;
|
||||
}
|
||||
|
||||
return <Progress active color='violet' percent={operation_percent || 0} progress>{label}</Progress>;
|
||||
}
|
||||
|
||||
function FilesPage() {
|
||||
|
|
@ -610,7 +609,7 @@ export function FilesRoute() {
|
|||
}
|
||||
|
||||
const useRefresh = () => {
|
||||
const refreshing = useStatusFlag('refreshing');
|
||||
const refreshing = useStatusFlag('file_worker_busy');
|
||||
const refreshingDirectory = useStatusFlag('refreshing_directory');
|
||||
const wrolModeEnabled = useWROLMode();
|
||||
|
||||
|
|
|
|||
|
|
@ -167,7 +167,7 @@ export function NavBar() {
|
|||
|
||||
let processingLink;
|
||||
if (status && status.flags) {
|
||||
if (status.flags.refreshing) {
|
||||
if (status.flags.file_worker_busy) {
|
||||
processingLink = '/files';
|
||||
} else if (status.flags.map_importing) {
|
||||
processingLink = '/map/manage';
|
||||
|
|
|
|||
|
|
@ -783,7 +783,7 @@ export const useFilesProgress = () => {
|
|||
export const useFilesProgressInterval = () => {
|
||||
const {progress, fetchFilesProgress} = useFilesProgress();
|
||||
|
||||
useRecurringTimeout(fetchFilesProgress, 1000 * 3);
|
||||
useRecurringTimeout(fetchFilesProgress, 1000);
|
||||
|
||||
return {progress, fetchFilesProgress};
|
||||
}
|
||||
|
|
|
|||
9
main.py
9
main.py
|
|
@ -9,11 +9,11 @@ from contextlib import contextmanager
|
|||
from sanic import Sanic
|
||||
from sanic.signals import Event
|
||||
|
||||
from modules.archive.lib import import_domains_config
|
||||
from modules.inventory import init_inventory
|
||||
from modules.inventory.common import import_inventories_config
|
||||
from modules.videos.lib import import_channels_config, get_videos_downloader_config
|
||||
from wrolpi import flags
|
||||
from modules.archive.lib import import_domains_config
|
||||
from wrolpi import root_api # noqa
|
||||
from wrolpi import tags
|
||||
from wrolpi.api_utils import api_app, perpetual_signal
|
||||
|
|
@ -24,6 +24,7 @@ from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initi
|
|||
from wrolpi.dates import Seconds
|
||||
from wrolpi.downloader import import_downloads_config, download_manager, get_download_manager_config
|
||||
from wrolpi.errors import WROLModeEnabled
|
||||
from wrolpi.files.worker import file_worker
|
||||
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, INTERNET_SERVER
|
||||
from wrolpi.version import get_version_string
|
||||
|
||||
|
|
@ -312,6 +313,12 @@ async def start_single_tasks(app: Sanic):
|
|||
logger.debug(f'start_single_tasks done')
|
||||
|
||||
|
||||
@perpetual_signal(sleep=1)
|
||||
async def perpetual_file_worker_queue():
|
||||
file_worker.transfer_queue()
|
||||
await file_worker.process_queue()
|
||||
|
||||
|
||||
@perpetual_signal(sleep=1)
|
||||
async def perpetual_check_log_level():
|
||||
"""Copies global log level into this Sanic worker's logger."""
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ async def test_archives_search_no_query(test_session, archive_factory, async_cli
|
|||
@skip_circleci
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_upload(test_session, async_client, singlefile_contents_factory, make_multipart_form,
|
||||
await_switches, events_history):
|
||||
await_switches, events_fixture):
|
||||
"""Test converting a SingleFile from the SingleFile browser extension to an Archive."""
|
||||
singlefile_contents = singlefile_contents_factory(title='upload title', url='https://example.com/single-file-url')
|
||||
forms = [
|
||||
|
|
@ -204,8 +204,7 @@ async def test_archive_upload(test_session, async_client, singlefile_contents_fa
|
|||
assert archive.screenshot_path.is_file(), 'Screenshot should have been created'
|
||||
assert archive.file_group.title == 'upload title', 'Title was not correct'
|
||||
assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted'
|
||||
events = list(events_history)
|
||||
assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.'
|
||||
events_fixture.assert_has_event('upload_archive')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ from wrolpi.api_utils import CustomJSONEncoder
|
|||
from wrolpi.collections import Collection
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.test.common import skip_circleci
|
||||
|
||||
|
|
@ -85,7 +84,7 @@ async def test_relationships(async_client, test_session, example_singlefile):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_title(async_client, test_session, archive_factory, singlefile_contents_factory):
|
||||
async def test_archive_title(async_client, test_session, archive_factory, singlefile_contents_factory, refresh_files):
|
||||
"""An Archive's title can be fetched in multiple ways. This tests from most to least reliable."""
|
||||
# Create some test files, delete all records for a fresh refresh.
|
||||
archive_factory(
|
||||
|
|
@ -98,7 +97,7 @@ async def test_archive_title(async_client, test_session, archive_factory, single
|
|||
async def reset_and_get_archive():
|
||||
test_session.query(FileGroup).delete()
|
||||
test_session.commit()
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
return test_session.query(Archive).one()
|
||||
|
||||
archive1: Archive = await reset_and_get_archive()
|
||||
|
|
@ -175,7 +174,8 @@ async def test_archive_refresh_deleted_archive(async_client, test_session, archi
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fills_contents_with_refresh(async_client, test_session, archive_factory, singlefile_contents_factory):
|
||||
async def test_fills_contents_with_refresh(async_client, test_session, archive_factory, singlefile_contents_factory,
|
||||
refresh_files):
|
||||
"""Refreshing archives fills in any missing contents."""
|
||||
archive1 = archive_factory('example.com', 'https://example.com/one')
|
||||
archive2 = archive_factory('example.com', 'https://example.com/one')
|
||||
|
|
@ -208,7 +208,7 @@ async def test_fills_contents_with_refresh(async_client, test_session, archive_f
|
|||
assert not archive4.file_group.d_text
|
||||
|
||||
# Fill the contents.
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
# The archives will be renamed with their title.
|
||||
archive1, archive2, archive3, archive4 = test_session.query(Archive).order_by(Archive.id)
|
||||
assert not archive4.file_group.d_text
|
||||
|
|
@ -291,7 +291,7 @@ def test_get_domains(test_session, archive_factory):
|
|||
|
||||
@skip_circleci
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_archive(test_session, test_directory, fake_now):
|
||||
async def test_new_archive(test_session, test_directory, fake_now, async_client):
|
||||
singlefile, readability, screenshot = make_fake_archive_result()
|
||||
fake_now(datetime(2000, 1, 1))
|
||||
archive1 = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
|
|
@ -334,7 +334,7 @@ async def test_new_archive(test_session, test_directory, fake_now):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_title_from_html(test_directory, test_session, fake_now):
|
||||
async def test_get_title_from_html(test_directory, test_session, fake_now, async_client):
|
||||
fake_now(datetime(2000, 1, 1))
|
||||
singlefile, readability, screenshot = make_fake_archive_result()
|
||||
archive = await model_archive_result('https://example.com', singlefile, readability, screenshot)
|
||||
|
|
@ -584,11 +584,11 @@ async def test_refresh_archives(test_session, test_directory, async_client, make
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_archives_index(test_session, make_files_structure):
|
||||
async def test_refresh_archives_index(test_session, make_files_structure, refresh_files):
|
||||
"""Archives are indexed using ArchiveIndexer and archive_modeler."""
|
||||
# The start of a typical singlefile html file.
|
||||
singlefile_contents = '''<!DOCTYPE html> <html lang="en"><!--
|
||||
Page saved with SingleFile
|
||||
Page saved with SingleFile
|
||||
url: https://example.com
|
||||
saved date: Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)
|
||||
--><head><meta charset="utf-8">
|
||||
|
|
@ -609,7 +609,7 @@ async def test_refresh_archives_index(test_session, make_files_structure):
|
|||
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
|
||||
})
|
||||
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
archive: Archive = test_session.query(Archive).one()
|
||||
assert archive.singlefile_path == singlefile
|
||||
|
|
@ -625,7 +625,7 @@ async def test_refresh_archives_index(test_session, make_files_structure):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_meta(async_client, test_session, make_files_structure):
|
||||
async def test_archive_meta(async_client, test_session, make_files_structure, refresh_files):
|
||||
# The start of a typical singlefile html file.
|
||||
singlefile_contents = '''<!DOCTYPE html> <html lang="en">
|
||||
<script data-vue-meta="ssr" type="application/ld+json">
|
||||
|
|
@ -640,7 +640,7 @@ async def test_archive_meta(async_client, test_session, make_files_structure):
|
|||
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
|
||||
})
|
||||
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
archive: Archive = test_session.query(Archive).one()
|
||||
assert archive.singlefile_path == singlefile
|
||||
|
|
@ -655,19 +655,19 @@ async def test_archive_meta(async_client, test_session, make_files_structure):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_archives_deleted_singlefile(async_client, test_session, make_files_structure,
|
||||
singlefile_contents_factory):
|
||||
singlefile_contents_factory, refresh_files):
|
||||
"""Removing a Singlefile file from a FileGroup makes that group no longer an Archive."""
|
||||
singlefile, readability = make_files_structure({
|
||||
'2022-09-04-16-20-11_The Title.html': singlefile_contents_factory(),
|
||||
'2022-09-04-16-20-11_The Title.readability.json': '{"url": "https://example.com"}',
|
||||
})
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert test_session.query(FileGroup).one().model == 'archive'
|
||||
assert test_session.query(Archive).count() == 1
|
||||
|
||||
# Remove singlefile, FileGroup is no longer an Archive.
|
||||
singlefile.unlink()
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert test_session.query(FileGroup).one().model is None
|
||||
assert test_session.query(Archive).count() == 0
|
||||
|
||||
|
|
@ -709,7 +709,7 @@ def test_archive_order(test_session, test_directory, archive_factory):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_download_index(test_session, test_directory, image_bytes_factory):
|
||||
async def test_archive_download_index(test_session, test_directory, image_bytes_factory, async_client):
|
||||
"""An Archive is indexed when it is downloaded."""
|
||||
singlefile = '<html><title>the singlefile</title></html>'
|
||||
readability = dict(
|
||||
|
|
@ -732,7 +732,7 @@ async def test_archive_download_index(test_session, test_directory, image_bytes_
|
|||
'Did not store the screenshot'
|
||||
|
||||
|
||||
def test_archive_history(test_session, archive_factory):
|
||||
def test_archive_history(test_session, archive_factory, async_client):
|
||||
"""Archive's can have a history of other archives.
|
||||
|
||||
Archive's with an empty URL are not associated."""
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@ def video_cleanup():
|
|||
''')
|
||||
|
||||
|
||||
|
||||
def claim_videos_for_channels(channel_ids: List[int]):
|
||||
"""
|
||||
Assign Videos to specific Channels based on directory paths.
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from wrolpi.common import logger, \
|
|||
get_media_directory, wrol_mode_check, background_task
|
||||
from wrolpi.db import get_db_curs, get_db_session
|
||||
from wrolpi.downloader import save_downloads_config, download_manager, Download
|
||||
from wrolpi.errors import APIError, ValidationError, RefreshConflict
|
||||
from wrolpi.errors import APIError, ValidationError, FileWorkerConflict
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.tags import Tag
|
||||
from .. import schema
|
||||
|
|
@ -269,8 +269,8 @@ async def tag_channel(session: Session, tag_name: str | None, directory: pathlib
|
|||
|
||||
Move the Channel to the new directory, if provided."""
|
||||
|
||||
if directory and flags.refreshing.is_set():
|
||||
raise RefreshConflict('Refusing to move channel while file refresh is in progress')
|
||||
if directory and flags.file_worker_busy.is_set():
|
||||
raise FileWorkerConflict('Refusing to move channel while FileWorker is busy')
|
||||
|
||||
channel = Channel.find_by_id(session, channel_id)
|
||||
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ async def test_channel_empty_url_doesnt_conflict(async_client, test_session, tes
|
|||
@pytest.mark.asyncio
|
||||
async def test_channel_download_requires_refresh(
|
||||
async_client, test_session, mock_video_extract_info, download_channel, video_download_manager,
|
||||
video_factory, events_history, await_switches):
|
||||
video_factory, events_fixture, await_switches, await_background_tasks):
|
||||
"""A Channel cannot be downloaded until it has been refreshed.
|
||||
|
||||
Videos already downloaded are not downloaded again."""
|
||||
|
|
@ -228,6 +228,8 @@ async def test_channel_download_requires_refresh(
|
|||
await video_download_manager.do_downloads()
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
await await_switches()
|
||||
# Wait for background tasks including channel refresh which processes the file worker queue
|
||||
await await_background_tasks()
|
||||
|
||||
# Channel was refreshed before downloading videos.
|
||||
assert_refreshed(True)
|
||||
|
|
@ -235,8 +237,10 @@ async def test_channel_download_requires_refresh(
|
|||
# Only the missing video was downloaded.
|
||||
assert downloaded_urls == ['https://example.com/2']
|
||||
|
||||
# Should not send refresh events because downloads are automated.
|
||||
assert list(events_history) == []
|
||||
# Should not send user-facing events - only internal FileWorker events are allowed.
|
||||
user_facing_events = ['user_notify_message', 'directory_refresh', 'files_refreshed']
|
||||
for event_name in user_facing_events:
|
||||
events_fixture.assert_no_event(event_name)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -13,7 +13,8 @@ from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_re
|
|||
background_task, replace_file
|
||||
from wrolpi.db import get_db_curs, get_db_session
|
||||
from wrolpi.downloader import Download
|
||||
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
|
||||
from wrolpi.files.lib import split_path_stem_and_suffix
|
||||
from wrolpi.files.worker import file_worker
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import Tag, TagFile
|
||||
from wrolpi.vars import PYTEST, VIDEO_INFO_JSON_KEYS_TO_CLEAN
|
||||
|
|
@ -900,9 +901,10 @@ class Channel(ModelHelper, Base):
|
|||
# Perform info_json in background task. Channel will be marked as refreshed after this completes.
|
||||
async def _():
|
||||
# Refresh all files within this channel's directory first.
|
||||
from modules.videos.common import update_view_counts_and_censored
|
||||
await refresh_files([directory], send_events=send_events)
|
||||
job_id = file_worker.queue_refresh([directory])
|
||||
await file_worker.wait_for_job(job_id)
|
||||
# Update view count second.
|
||||
from modules.videos.common import update_view_counts_and_censored
|
||||
await update_view_counts_and_censored(id_)
|
||||
with get_db_session(commit=True) as session_:
|
||||
channel_ = cls.find_by_id(session_, id_)
|
||||
|
|
|
|||
|
|
@ -9,12 +9,11 @@ from modules.videos.video import lib as video_lib
|
|||
from wrolpi.common import get_wrolpi_config
|
||||
from wrolpi.db import get_db_curs
|
||||
from wrolpi.downloader import Download
|
||||
from wrolpi.files.lib import refresh_files
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_videos_index(async_client, test_session, test_directory, video_factory):
|
||||
async def test_refresh_videos_index(async_client, test_session, test_directory, video_factory, refresh_files):
|
||||
"""The video modeler indexes video data into the Video's FileGroup."""
|
||||
video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True)
|
||||
test_session.commit()
|
||||
|
|
@ -54,12 +53,12 @@ async def test_refresh_videos(async_client, test_session, test_directory, simple
|
|||
with_info_json=True, with_poster_ext='jpg')
|
||||
test_session.commit()
|
||||
video1.file_group.size = video1.file_group.data = None
|
||||
video1.file_group.paths = list()
|
||||
video1.file_group.files = []
|
||||
# video2 is in the test directory.
|
||||
video2 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True,
|
||||
with_poster_ext='jpg')
|
||||
video2.file_group.data = None
|
||||
video2.file_group.paths = []
|
||||
video2.file_group.files = []
|
||||
test_session.commit()
|
||||
|
||||
assert not video1.file_group.size, 'video1 should not have size during creation'
|
||||
|
|
@ -76,7 +75,7 @@ async def test_refresh_videos(async_client, test_session, test_directory, simple
|
|||
with get_db_curs(commit=True) as curs:
|
||||
stmt = "INSERT INTO file_group (mimetype, directory, primary_path, indexed, files, model)" \
|
||||
" values ('video/mp4', %(directory)s, %(primary_path)s, true, %(files)s, 'video') RETURNING id"
|
||||
params = {'directory': str(test_directory), 'primary_path': str(video_file), 'files': list()}
|
||||
params = {'directory': str(test_directory), 'primary_path': str(video_file), 'files': '[]'}
|
||||
curs.execute(stmt, params)
|
||||
video4_id = curs.fetchall()[0][0]
|
||||
stmt = "INSERT INTO video (file_group_id) values (%(video_id)s)"
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ from modules.videos.models import Channel, Video
|
|||
from wrolpi.collections import Collection
|
||||
from wrolpi.common import get_absolute_media_path, get_wrolpi_config
|
||||
from wrolpi.downloader import Download, DownloadFrequency
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
from .. import common
|
||||
from ..common import convert_image, update_view_counts_and_censored, extract_video_duration, generate_video_poster, \
|
||||
|
|
@ -411,11 +410,11 @@ async def test_ffprobe_json(async_client, video_file, corrupted_video_file):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_ffprobe_json(async_client, test_session, video_file):
|
||||
async def test_video_ffprobe_json(async_client, test_session, video_file, refresh_files):
|
||||
"""ffprobe data is extracted when a video is modeled."""
|
||||
with mock.patch('modules.videos.lib.extract_video_duration') as mock_extract_video_duration:
|
||||
mock_extract_video_duration.side_effect = Exception('duration should be from ffprobe json')
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
video = test_session.query(Video).one()
|
||||
assert video.ffprobe_json
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ from modules.videos.lib import parse_video_file_name, validate_video, get_statis
|
|||
from modules.videos.models import Video, Channel
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
from wrolpi.downloader import Download, RSSDownloader
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
|
@ -142,7 +141,8 @@ async def test_get_statistics(test_session, video_factory, channel_factory):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orphaned_files(async_client, test_session, make_files_structure, test_directory, video_factory):
|
||||
async def test_orphaned_files(async_client, test_session, make_files_structure, test_directory, video_factory,
|
||||
refresh_files):
|
||||
# A Video without associated files is not orphaned.
|
||||
vid1 = video_factory(title='vid1', with_video_file=True)
|
||||
# The video files will be removed...
|
||||
|
|
@ -159,7 +159,7 @@ async def test_orphaned_files(async_client, test_session, make_files_structure,
|
|||
# Remove vid3 video. Poster is now orphaned.
|
||||
vid3.video_path.unlink()
|
||||
vid3_poster_path = vid3.poster_path
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
test_session.commit()
|
||||
|
||||
# 6 files when two video files are deleted.
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import pytest
|
|||
|
||||
from modules.videos.models import Video
|
||||
from wrolpi.errors import FileGroupIsTagged
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
||||
|
|
@ -47,7 +46,8 @@ async def test_delete_video_with_tag(async_client, test_session, video_factory,
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_channel_refresh(async_client, test_session, test_directory, channel_factory, video_factory):
|
||||
async def test_video_channel_refresh(async_client, test_session, test_directory, channel_factory, video_factory,
|
||||
refresh_files):
|
||||
"""A Video will be associated with a Channel when it's files are in that Channel's directory."""
|
||||
# Create a video file in this channel's directory.
|
||||
channel = channel_factory()
|
||||
|
|
@ -60,7 +60,7 @@ async def test_video_channel_refresh(async_client, test_session, test_directory,
|
|||
test_session.query(FileGroup).delete()
|
||||
test_session.commit()
|
||||
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
assert test_session.query(FileGroup).count() == 2
|
||||
assert test_session.query(Video).count() == 2
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@ from wrolpi.dates import Seconds
|
|||
from wrolpi.events import Events
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.files.lib import refresh_files, split_file_name_words
|
||||
from wrolpi.files.lib import split_file_name_words
|
||||
from wrolpi.files.worker import file_worker
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.vars import PYTEST, DOCKERIZED
|
||||
|
||||
|
|
@ -550,8 +551,8 @@ async def remove_outdated_zim_files(path: pathlib.Path = None) -> int:
|
|||
file.unlink()
|
||||
deleted_count += 1
|
||||
|
||||
# Must refresh synchronously to clean up DB before returning.
|
||||
await refresh_files(outdated)
|
||||
# Refresh synchronously to clean up DB before returning.
|
||||
await file_worker.refresh_sync(list(outdated))
|
||||
|
||||
await restart_kiwix()
|
||||
|
||||
|
|
|
|||
|
|
@ -10,15 +10,14 @@ from modules.zim.models import TagZimEntry, ZimSubscription
|
|||
from wrolpi import tags, flags
|
||||
from wrolpi.common import DownloadFileInfo, get_wrolpi_config
|
||||
from wrolpi.downloader import Download, import_downloads_config, get_download_manager_config
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.test.common import skip_circleci
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_zim(async_client, test_session, zim_path_factory):
|
||||
async def test_get_zim(async_client, test_session, zim_path_factory, refresh_files):
|
||||
zim_path_factory()
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
assert lib.get_zim(test_session, 1)
|
||||
|
||||
|
|
@ -27,9 +26,9 @@ async def test_get_zim(async_client, test_session, zim_path_factory):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_get_entry(async_client, test_session, zim_path_factory):
|
||||
async def test_zim_get_entry(async_client, test_session, zim_path_factory, refresh_files):
|
||||
zim_path_factory()
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
entry: Entry = lib.get_entry('one', 1)
|
||||
assert entry.path == 'one'
|
||||
|
|
@ -389,13 +388,13 @@ async def test_zim_subscription_download_import(async_client, test_session):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zim_modeler(async_client, test_session, zim_factory):
|
||||
async def test_zim_modeler(async_client, test_session, zim_factory, refresh_files):
|
||||
"""Zim modeler sets FileGroup title and a_text."""
|
||||
zim = zim_factory('the_zim_title.zim')
|
||||
assert zim.file_group.indexed is False
|
||||
test_session.commit()
|
||||
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
fg: FileGroup = test_session.query(FileGroup).one()
|
||||
assert fg.indexed is True
|
||||
|
|
|
|||
|
|
@ -3,4 +3,7 @@ asyncio_mode = auto
|
|||
testpaths =
|
||||
wrolpi
|
||||
modules
|
||||
asyncio_default_fixture_loop_scope = function
|
||||
asyncio_default_fixture_loop_scope = function
|
||||
# Do not remove timeout - tests should not run forever
|
||||
timeout = 20
|
||||
timeout_method = signal
|
||||
|
|
|
|||
|
|
@ -15,14 +15,15 @@ psutil==6.0.0
|
|||
psycopg2-binary==2.9.9
|
||||
pypdf==6.4.0
|
||||
pytest-asyncio==0.24.0
|
||||
pytest-xdist==3.5.0
|
||||
pytest==8.3.3
|
||||
python-docx==1.1.2
|
||||
python-dotenv==1.0.1
|
||||
python-magic~=0.4.27
|
||||
pytz==2024.2
|
||||
sanic-ext==23.6.0
|
||||
sanic-testing==22.6.0 # 23.6.0 has error with "expandtabs"
|
||||
sanic==23.6.0 # This is the latest version of Sanic that avoids this bug: https://github.com/sanic-org/sanic/issues/2921
|
||||
sanic-ext==25.12.0
|
||||
sanic-testing==24.6.0
|
||||
sanic==25.12.0
|
||||
selenium==4.9.1
|
||||
srt==3.5.3
|
||||
tqdm==4.67.1
|
||||
|
|
|
|||
|
|
@ -201,7 +201,9 @@ def perpetual_signal(event: str = None, sleep: int | float = 1, run_while_testin
|
|||
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
elapsed = int(time() - start)
|
||||
logger.trace(f'perpetual_signal {event_} took {elapsed} seconds')
|
||||
if not cancelled:
|
||||
if not cancelled and not PYTEST:
|
||||
# In PYTEST mode, don't self-dispatch - the test fixture handles this.
|
||||
# This prevents "Task was destroyed but it is pending!" errors during teardown.
|
||||
await asyncio.sleep(sleep)
|
||||
await api_app.dispatch(event_)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from wrolpi import flags
|
|||
from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
|
||||
from wrolpi.db import get_db_session
|
||||
import pathlib
|
||||
from wrolpi.errors import RefreshConflict
|
||||
from wrolpi.errors import FileWorkerConflict
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.tags import Tag
|
||||
|
|
@ -286,7 +286,7 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None:
|
|||
ValidationError: If collection has no directory
|
||||
"""
|
||||
# Local import to avoid circular import: collections -> files -> collections
|
||||
from wrolpi.files.lib import refresh_files
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
||||
with get_db_session() as session:
|
||||
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
|
||||
|
|
@ -303,7 +303,7 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None:
|
|||
directory = collection.directory
|
||||
|
||||
# Refresh files asynchronously
|
||||
asyncio.ensure_future(refresh_files([directory], send_events=send_events))
|
||||
file_worker.queue_refresh([directory])
|
||||
|
||||
if send_events:
|
||||
relative_dir = get_relative_to_media_directory(directory)
|
||||
|
|
@ -350,7 +350,7 @@ async def tag_collection(
|
|||
Raises:
|
||||
UnknownCollection: If collection not found
|
||||
ValidationError: If tagging requirements not met
|
||||
RefreshConflict: If a file refresh is in progress and directory change is requested
|
||||
FileWorkerConflict: If a file operation is in progress and directory change is requested
|
||||
"""
|
||||
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
|
||||
|
||||
|
|
@ -381,8 +381,8 @@ async def tag_collection(
|
|||
logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, '
|
||||
f'{old_directory} -> {target_directory}')
|
||||
|
||||
if need_to_move and flags.refreshing.is_set():
|
||||
raise RefreshConflict('Refusing to move collection while file refresh is in progress')
|
||||
if need_to_move and flags.file_worker_busy.is_set():
|
||||
raise FileWorkerConflict('Refusing to move collection while FileWorker is busy')
|
||||
|
||||
session.flush()
|
||||
|
||||
|
|
@ -443,8 +443,8 @@ async def tag_collection(
|
|||
logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, '
|
||||
f'{old_directory} -> {target_directory}')
|
||||
|
||||
if need_to_move and flags.refreshing.is_set():
|
||||
raise RefreshConflict('Refusing to move collection while file refresh is in progress')
|
||||
if need_to_move and flags.file_worker_busy.is_set():
|
||||
raise FileWorkerConflict('Refusing to move collection while FileWorker is busy')
|
||||
|
||||
# Apply the tag
|
||||
collection.tag_id = tag.id
|
||||
|
|
|
|||
|
|
@ -11,8 +11,9 @@ from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_re
|
|||
from wrolpi.downloader import Download, save_downloads_config
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.files.lib import move as move_files, refresh_files
|
||||
from wrolpi.files.worker import file_worker
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.files.lib import delete_directory
|
||||
from wrolpi.media_path import MediaPathType
|
||||
from wrolpi.tags import Tag
|
||||
from wrolpi.tags import save_tags_config
|
||||
|
|
@ -942,7 +943,7 @@ class Collection(ModelHelper, Base):
|
|||
|
||||
self.directory = directory
|
||||
|
||||
with flags.refreshing:
|
||||
with flags.file_worker_busy:
|
||||
# Update download destinations before moving files.
|
||||
change_download_destinations(old_directory, directory)
|
||||
session.commit()
|
||||
|
|
@ -956,14 +957,16 @@ class Collection(ModelHelper, Base):
|
|||
try:
|
||||
if not old_directory.exists():
|
||||
# Old directory does not exist; refresh both
|
||||
await refresh_files([old_directory, directory])
|
||||
file_worker.queue_refresh([old_directory, directory])
|
||||
if send_events:
|
||||
Events.send_file_move_completed(f'Collection {repr(self.name)} was moved (directory missing)')
|
||||
else:
|
||||
files_to_move = list(old_directory.iterdir())
|
||||
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
logger.trace(f'move_collection: moving {len(files_to_move)} items from {old_directory}')
|
||||
await move_files(session, directory, *files_to_move)
|
||||
if files_to_move:
|
||||
job_id = file_worker.queue_move(directory, files_to_move)
|
||||
await file_worker.wait_for_job(job_id)
|
||||
if send_events:
|
||||
Events.send_file_move_completed(f'Collection {repr(self.name)} was moved')
|
||||
except Exception as e:
|
||||
|
|
@ -978,7 +981,7 @@ class Collection(ModelHelper, Base):
|
|||
finally:
|
||||
session.commit()
|
||||
if old_directory.exists() and not next(iter(old_directory.iterdir()), None):
|
||||
old_directory.rmdir()
|
||||
delete_directory(old_directory)
|
||||
else:
|
||||
# Files already moved, just send event
|
||||
if send_events:
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@ def test_set_tag_on_directory_less_domain_collection(test_session):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tag_collection_removes_tag_and_updates_directory(test_session, test_directory, tag_factory,
|
||||
await_switches):
|
||||
await_switches, await_background_tasks):
|
||||
"""When removing a tag via tag_collection, the directory should be updated if provided."""
|
||||
from wrolpi.collections.lib import tag_collection
|
||||
|
||||
|
|
@ -207,7 +207,8 @@ async def test_tag_collection_removes_tag_and_updates_directory(test_session, te
|
|||
directory=str(untagged_dir), # New directory
|
||||
)
|
||||
test_session.commit()
|
||||
await await_switches()
|
||||
await await_switches(timeout=2)
|
||||
await await_background_tasks(timeout=2)
|
||||
|
||||
# Verify tag was removed
|
||||
assert collection.tag_id is None
|
||||
|
|
|
|||
|
|
@ -1801,14 +1801,16 @@ def background_task(coro) -> Task:
|
|||
|
||||
The task is stored in a global set of background tasks so the task will not be discarded by the garbage collector.
|
||||
"""
|
||||
# Get the coroutine name for better logging (used in task name for debugging stuck tasks)
|
||||
coro_name = getattr(coro, '__qualname__', None) or getattr(coro, '__name__', None) or str(coro)
|
||||
|
||||
async def error_logger():
|
||||
try:
|
||||
await coro
|
||||
except Exception as e:
|
||||
logger_.error('Background task had error:', exc_info=e)
|
||||
logger_.error(f'background_task: error in {coro_name}:', exc_info=e)
|
||||
|
||||
task = asyncio.create_task(error_logger())
|
||||
task = asyncio.create_task(error_logger(), name=f'bg:{coro_name}')
|
||||
add_background_task(task)
|
||||
return task
|
||||
|
||||
|
|
@ -1822,13 +1824,66 @@ async def cancel_background_tasks():
|
|||
await asyncio.gather(*BACKGROUND_TASKS)
|
||||
|
||||
|
||||
async def await_background_tasks():
|
||||
"""Awaits all background tasks, used only for testing."""
|
||||
async def await_background_tasks(timeout: int = 15):
|
||||
"""Awaits all background tasks, used only for testing.
|
||||
|
||||
Processes the FileWorker queue concurrently with background tasks to avoid
|
||||
deadlocks when background tasks wait for file worker jobs to complete.
|
||||
|
||||
Args:
|
||||
timeout: Maximum seconds to wait for background tasks (default 15s)
|
||||
"""
|
||||
if not PYTEST:
|
||||
raise RuntimeError('await_background_tasks should only be used for testing')
|
||||
|
||||
for background_task in BACKGROUND_TASKS.copy():
|
||||
await background_task
|
||||
from wrolpi.files.worker import file_worker
|
||||
import time
|
||||
|
||||
iteration = 0
|
||||
max_iterations = 1000 # Safety limit to prevent infinite loops
|
||||
start_time = time.time()
|
||||
|
||||
while BACKGROUND_TASKS or file_worker.public_queue.qsize() > 0 or not file_worker.private_queue.empty():
|
||||
iteration += 1
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
if elapsed > timeout:
|
||||
task_names = [t.get_name() if hasattr(t, 'get_name') else str(t) for t in BACKGROUND_TASKS]
|
||||
logger_.error(f'await_background_tasks: TIMEOUT after {elapsed:.1f}s, {iteration} iterations')
|
||||
logger_.error(f' BACKGROUND_TASKS={len(BACKGROUND_TASKS)}: {task_names}')
|
||||
logger_.error(f' public_queue={file_worker.public_queue.qsize()}, private_queue_empty={file_worker.private_queue.empty()}')
|
||||
# Cancel stuck tasks and clear to prevent cascading failures
|
||||
for task in list(BACKGROUND_TASKS):
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
BACKGROUND_TASKS.clear()
|
||||
break
|
||||
|
||||
if iteration > max_iterations:
|
||||
task_names = [t.get_name() if hasattr(t, 'get_name') else str(t) for t in BACKGROUND_TASKS]
|
||||
logger_.error(f'await_background_tasks: SAFETY LIMIT REACHED after {iteration} iterations')
|
||||
logger_.error(f' BACKGROUND_TASKS={len(BACKGROUND_TASKS)}: {task_names}')
|
||||
logger_.error(f' public_queue={file_worker.public_queue.qsize()}, private_queue_empty={file_worker.private_queue.empty()}')
|
||||
break
|
||||
|
||||
# Process file worker queue
|
||||
file_worker.transfer_queue()
|
||||
while not file_worker.private_queue.empty():
|
||||
await file_worker.process_queue()
|
||||
|
||||
if not BACKGROUND_TASKS:
|
||||
continue
|
||||
|
||||
# Wait for any background task to complete (with timeout so we can keep processing queue)
|
||||
done, pending = await asyncio.wait(
|
||||
BACKGROUND_TASKS.copy(),
|
||||
timeout=0.1,
|
||||
return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
|
||||
for task in done:
|
||||
await task # Re-await to propagate exceptions
|
||||
BACKGROUND_TASKS.discard(task)
|
||||
|
||||
|
||||
def get_warn_once(message: str, logger__: logging.Logger, level=logging.ERROR):
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import pathlib
|
|||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import zipfile
|
||||
from abc import ABC
|
||||
from datetime import datetime
|
||||
|
|
@ -36,7 +37,7 @@ from wrolpi import flags
|
|||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.cmd import CommandResult
|
||||
from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode, timer, TRACE_LEVEL
|
||||
from wrolpi.common import logger, await_background_tasks as await_background_tasks_
|
||||
from wrolpi.common import logger, await_background_tasks as await_background_tasks_, BACKGROUND_TASKS
|
||||
from wrolpi.common import set_test_media_directory, Base, set_test_config
|
||||
from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts
|
||||
from wrolpi.dates import set_test_now
|
||||
|
|
@ -111,6 +112,43 @@ def test_debug_logger(request):
|
|||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clear_background_tasks(request):
|
||||
"""Clear BACKGROUND_TASKS before each test and fail if orphaned tasks remain after.
|
||||
|
||||
Tests must properly await all background tasks they create. Orphaned tasks indicate
|
||||
a test is not properly cleaning up, which can cause cascading failures in other tests.
|
||||
"""
|
||||
# Clear before test - cancel any orphaned tasks from previous tests
|
||||
if BACKGROUND_TASKS:
|
||||
logger.warning(f'clear_background_tasks: clearing {len(BACKGROUND_TASKS)} orphaned tasks BEFORE test')
|
||||
for task in list(BACKGROUND_TASKS):
|
||||
task_name = task.get_name() if hasattr(task, 'get_name') else str(task)
|
||||
logger.warning(f' orphaned task from previous test: {task_name}')
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
BACKGROUND_TASKS.clear()
|
||||
|
||||
yield
|
||||
|
||||
# After test - fail if orphaned tasks remain
|
||||
if BACKGROUND_TASKS:
|
||||
orphaned_tasks = []
|
||||
for task in list(BACKGROUND_TASKS):
|
||||
task_name = task.get_name() if hasattr(task, 'get_name') else str(task)
|
||||
orphaned_tasks.append(f'{task_name} (done={task.done()})')
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
BACKGROUND_TASKS.clear()
|
||||
|
||||
# Fail the test
|
||||
pytest.fail(
|
||||
f'Test left {len(orphaned_tasks)} orphaned background task(s): {", ".join(orphaned_tasks)}. '
|
||||
f'Tests must await all background tasks they create using await_background_tasks or '
|
||||
f'the async_client fixture.'
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_directory() -> Generator[Path, Any, None]:
|
||||
"""
|
||||
|
|
@ -210,10 +248,11 @@ class AutoAwaitTestClient:
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
async def async_client(test_directory, test_session) -> AsyncGenerator[AutoAwaitTestClient, Any]:
|
||||
async def async_client(test_directory, test_session, clear_background_tasks) -> AsyncGenerator[AutoAwaitTestClient, Any]:
|
||||
"""Get an Async Sanic Test Client with all default routes attached.
|
||||
|
||||
Depends on test_session to ensure the database mock is in place for session middleware.
|
||||
Depends on clear_background_tasks to detect orphaned background tasks.
|
||||
|
||||
This client automatically awaits switches and background tasks after
|
||||
POST, PUT, PATCH, and DELETE requests, and expires session objects.
|
||||
|
|
@ -229,7 +268,6 @@ async def async_client(test_directory, test_session) -> AsyncGenerator[AutoAwait
|
|||
yield wrapped_client
|
||||
finally:
|
||||
api_app.stop()
|
||||
logger.debug('Destroying async_client')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -253,11 +291,14 @@ async def await_background_tasks(async_client, test_session):
|
|||
|
||||
After background tasks complete, session objects may be stale, so we automatically
|
||||
expire them to force a refresh on next access.
|
||||
|
||||
Args:
|
||||
timeout: Optional timeout in seconds (default 15s from await_background_tasks_)
|
||||
"""
|
||||
await async_client.get('/api')
|
||||
|
||||
async def _await_and_expire():
|
||||
await await_background_tasks_()
|
||||
async def _await_and_expire(timeout: int = 15):
|
||||
await await_background_tasks_(timeout=timeout)
|
||||
test_session.expire_all()
|
||||
|
||||
return _await_and_expire
|
||||
|
|
@ -584,10 +625,66 @@ def run_command_bad_result() -> CommandResult:
|
|||
return CommandResult(1, False, b'run_command_bad_result stdout', b'run_command_bad_result stderr', 0, 123)
|
||||
|
||||
|
||||
class EventsFixture:
|
||||
"""Fixture for testing events with rich assertions.
|
||||
|
||||
Only returns events created after this fixture was instantiated,
|
||||
making it safe for parallel test execution.
|
||||
"""
|
||||
|
||||
def __init__(self, events_history: list):
|
||||
self._history = events_history
|
||||
# Track the starting index to only see new events during this test
|
||||
self._start_index = len(events_history)
|
||||
|
||||
@property
|
||||
def events(self) -> List[dict]:
|
||||
"""Return events captured during this test (most recent first)."""
|
||||
# Only return events added after fixture creation
|
||||
new_events = list(self._history[self._start_index:])
|
||||
return list(reversed(new_events))
|
||||
|
||||
def get_by_name(self, event_name: str) -> List[dict]:
|
||||
"""Get all events with the given name."""
|
||||
return [e for e in self.events if e['event'] == event_name]
|
||||
|
||||
def get_by_subject(self, subject: str) -> List[dict]:
|
||||
"""Get all events with the given subject."""
|
||||
return [e for e in self.events if e['subject'] == subject]
|
||||
|
||||
def assert_has_event(self, event_name: str, message: str = None) -> None:
|
||||
"""Assert that an event with the given name exists."""
|
||||
matching = self.get_by_name(event_name)
|
||||
assert matching, f"No event found with name '{event_name}'. Events: {[e['event'] for e in self.events]}"
|
||||
if message:
|
||||
assert any(message in (e.get('message') or '') for e in matching), \
|
||||
f"No '{event_name}' event contains message '{message}'"
|
||||
|
||||
def assert_no_event(self, event_name: str) -> None:
|
||||
"""Assert that no event with the given name exists."""
|
||||
matching = self.get_by_name(event_name)
|
||||
assert not matching, f"Found unexpected event '{event_name}': {matching}"
|
||||
|
||||
def assert_events_contain(self, expected: List[dict]) -> None:
|
||||
"""Assert that events contain all expected events (order matters, most recent first)."""
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
for exp, actual in zip_longest(expected, self.events):
|
||||
assert actual is not None, f"Expected event {exp} but no more events found"
|
||||
assert_dict_contains(actual, exp)
|
||||
|
||||
def assert_event_count(self, count: int) -> None:
|
||||
"""Assert the total number of events."""
|
||||
assert len(self.events) == count, f"Expected {count} events, got {len(self.events)}"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def events_history(async_client):
|
||||
"""Give each test its own Events history."""
|
||||
yield api_app.shared_ctx.events_history
|
||||
def events_fixture(async_client) -> EventsFixture:
|
||||
"""Fixture for testing events with rich assertions.
|
||||
|
||||
Only sees events created after this fixture is instantiated,
|
||||
making it safe for parallel test execution.
|
||||
"""
|
||||
return EventsFixture(api_app.shared_ctx.events_history)
|
||||
|
||||
|
||||
FLAGS_LOCK = multiprocessing.Lock()
|
||||
|
|
@ -900,3 +997,34 @@ def mock_downloader_download_file():
|
|||
yield set_contents
|
||||
|
||||
|
||||
async def await_file_worker(timeout: int = 30):
|
||||
"""Wait for FileWorker to complete all pending tasks. Testing only."""
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
file_worker.transfer_queue()
|
||||
if file_worker.private_queue.qsize() > 0:
|
||||
await file_worker.process_queue()
|
||||
elif file_worker.public_queue.qsize() == 0:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
else:
|
||||
raise RuntimeError('Timed out waiting for file_worker')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def refresh_files(async_client):
|
||||
"""Test helper that queues and waits for file refresh."""
|
||||
|
||||
async def _refresh(paths=None):
|
||||
from wrolpi.common import get_media_directory
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
||||
paths = paths or [get_media_directory()]
|
||||
file_worker.queue_refresh(paths)
|
||||
await await_file_worker()
|
||||
|
||||
return _refresh
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -76,6 +76,10 @@ def attach_shared_contexts(app: Sanic):
|
|||
app.shared_ctx.events_lock = multiprocessing.Lock()
|
||||
app.shared_ctx.events_history = manager.list()
|
||||
|
||||
app.shared_ctx.file_worker_public_queue = manager.Queue()
|
||||
app.shared_ctx.file_worker_status = manager.dict()
|
||||
app.shared_ctx.file_worker_jobs = manager.dict()
|
||||
|
||||
reset_shared_contexts(app)
|
||||
|
||||
|
||||
|
|
@ -157,7 +161,29 @@ def reset_shared_contexts(app: Sanic):
|
|||
except queue.Empty:
|
||||
break
|
||||
|
||||
# File Worker
|
||||
while True:
|
||||
# Clear out any pending file tasks.
|
||||
try:
|
||||
app.shared_ctx.file_worker_public_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
app.shared_ctx.file_worker_status.clear()
|
||||
app.shared_ctx.file_worker_jobs.clear()
|
||||
app.shared_ctx.file_worker_status.update(dict(
|
||||
status='idle',
|
||||
task_type=None,
|
||||
paths=[],
|
||||
destination=None,
|
||||
error=None,
|
||||
operation_total=0,
|
||||
operation_processed=0,
|
||||
operation_percent=0,
|
||||
))
|
||||
|
||||
# Events.
|
||||
# Clear events_history in place to preserve references held by fixtures
|
||||
del app.shared_ctx.events_history[:]
|
||||
app.shared_ctx.single_tasks_started.clear()
|
||||
app.shared_ctx.flags_initialized.clear()
|
||||
app.shared_ctx.perpetual_tasks_started.clear()
|
||||
|
|
|
|||
|
|
@ -171,9 +171,9 @@ class InvalidDirectory(APIError):
|
|||
status_code = HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class RefreshConflict(APIError):
|
||||
code = 'REFRESH_CONFLICT'
|
||||
summary = 'Not possible during file refresh'
|
||||
class FileWorkerConflict(APIError):
|
||||
code = 'FILE_WORKER_CONFLICT'
|
||||
summary = 'Not possible while FileWorker is busy'
|
||||
status_code = HTTPStatus.CONFLICT
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -52,6 +52,10 @@ class Events:
|
|||
def send_directory_refresh(message: str = None):
|
||||
send_event('directory_refresh', message, subject='refresh')
|
||||
|
||||
@staticmethod
|
||||
def send_files_refreshed(message: str = None):
|
||||
send_event('files_refreshed', message, subject='refresh')
|
||||
|
||||
@staticmethod
|
||||
def send_deleted(message: str = None):
|
||||
send_event('deleted', message, subject='deleted')
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_
|
|||
background_task, walk, timer, TRACE_LEVEL, unique_by_predicate
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict
|
||||
from . import lib, schema
|
||||
from .worker import file_worker
|
||||
from ..api_utils import json_response, api_app
|
||||
from ..events import Events
|
||||
from ..schema import JSONErrorResponse
|
||||
|
|
@ -76,14 +77,21 @@ async def delete_file(_: Request, body: schema.DeleteRequest):
|
|||
)
|
||||
@wrol_mode_check
|
||||
async def refresh(request: Request):
|
||||
paths = None
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
||||
if request.body:
|
||||
media_directory = get_media_directory()
|
||||
if not isinstance(request.json['paths'], list):
|
||||
raise ValueError('Can only refresh a list')
|
||||
|
||||
paths = [media_directory / i for i in request.json['paths']]
|
||||
await lib.refresh_files(paths)
|
||||
# When user explicitly specifies files, don't expand to stem-mates.
|
||||
# This allows refreshing specific files without discovering related files.
|
||||
expand_stems = not all(p.is_file() for p in paths)
|
||||
else:
|
||||
paths = [get_media_directory()]
|
||||
expand_stems = True
|
||||
|
||||
file_worker.queue_refresh(paths, expand_stems=expand_stems)
|
||||
return response.empty()
|
||||
|
||||
|
||||
|
|
@ -98,6 +106,17 @@ async def refresh_progress(request: Request):
|
|||
))
|
||||
|
||||
|
||||
@files_bp.get('/worker_status')
|
||||
@openapi.definition(
|
||||
summary='Get the FileWorker status with detailed progress'
|
||||
)
|
||||
async def worker_status(request: Request):
|
||||
from wrolpi.files.worker import file_worker
|
||||
return json_response(dict(
|
||||
status=dict(file_worker.status),
|
||||
))
|
||||
|
||||
|
||||
@files_bp.post('/search')
|
||||
@openapi.definition(
|
||||
summary='Search Files',
|
||||
|
|
@ -239,7 +258,7 @@ async def post_create_directory(_: Request, body: schema.Directory):
|
|||
except FileExistsError:
|
||||
raise FileConflict(f'{path} already exists')
|
||||
|
||||
background_task(lib.refresh_files([path, ]))
|
||||
file_worker.queue_refresh([path])
|
||||
|
||||
return response.empty(HTTPStatus.CREATED)
|
||||
|
||||
|
|
@ -261,13 +280,7 @@ async def post_move(request: Request, body: schema.Move):
|
|||
if source.is_mount():
|
||||
raise FileConflict(f'Cannot move mounted directory!')
|
||||
|
||||
try:
|
||||
await lib.move(request.ctx.session, destination, *sources)
|
||||
Events.send_file_move_completed(f'Moving files to {body.destination} succeeded')
|
||||
except Exception as e:
|
||||
Events.send_file_move_failed(f'Moving files to {body.destination} failed!')
|
||||
raise FileConflict(f'Failed to move {sources} to {destination}') from e
|
||||
|
||||
file_worker.queue_move(destination, sources)
|
||||
return response.empty(HTTPStatus.NO_CONTENT)
|
||||
|
||||
|
||||
|
|
@ -296,7 +309,7 @@ async def post_rename(request: Request, body: schema.Rename):
|
|||
# Refreshing individual files causes the FileGroup's files list to be overwritten
|
||||
# with only the specified file, losing track of associated files.
|
||||
parent_dir = new_path.parent
|
||||
background_task(lib.refresh_files([parent_dir]))
|
||||
file_worker.queue_refresh([parent_dir])
|
||||
|
||||
return response.empty(HTTPStatus.NO_CONTENT)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import re
|
|||
import shutil
|
||||
import subprocess
|
||||
import urllib.parse
|
||||
from collections import OrderedDict
|
||||
from itertools import zip_longest
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple, Union, Dict, Generator, Iterable, Set
|
||||
|
|
@ -23,11 +22,11 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from wrolpi import flags
|
||||
from wrolpi.cmd import which
|
||||
from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \
|
||||
partition, cancelable_wrapper, \
|
||||
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, \
|
||||
from wrolpi.common import get_media_directory, wrol_mode_check, logger, \
|
||||
partition, \
|
||||
get_files_and_directories, chunks_by_stem, walk, \
|
||||
get_wrolpi_config, \
|
||||
timer, chunks, unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory
|
||||
unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory
|
||||
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
|
||||
from wrolpi.db import get_db_session, get_db_curs, mogrify
|
||||
from wrolpi.downloader import download_manager, Download
|
||||
|
|
@ -50,10 +49,10 @@ except ImportError:
|
|||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
|
||||
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'search_files',
|
||||
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics',
|
||||
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
|
||||
'move', 'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href']
|
||||
'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href']
|
||||
|
||||
|
||||
def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
|
||||
|
|
@ -255,10 +254,11 @@ async def delete(*paths: Union[str, pathlib.Path]):
|
|||
else:
|
||||
path.unlink()
|
||||
|
||||
# Must refresh synchronously to clean up DB before returning.
|
||||
# Refresh synchronously to clean up DB before returning.
|
||||
# This ensures deleted FileGroups are removed before the caller continues
|
||||
# (e.g., upload API recreating the file).
|
||||
await refresh_files(paths)
|
||||
from wrolpi.files.worker import file_worker
|
||||
await file_worker.refresh_sync(list(paths))
|
||||
|
||||
|
||||
def _mimetype_suffix_map(path: Path, mimetype: str):
|
||||
|
|
@ -537,7 +537,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
|
|||
idempotency = str(idempotency)
|
||||
|
||||
non_primary_files = set()
|
||||
for chunk in chunks_by_stem(files, 100):
|
||||
for chunk in chunks_by_stem(files, 500):
|
||||
# Group all files by their shared full-stem. `chunk_by_stem` already sorted the files.
|
||||
grouped = group_files_by_stem(chunk, pre_sorted=True)
|
||||
# Convert the groups into file_groups. Extract common information into `file_group.data`.
|
||||
|
|
@ -724,95 +724,6 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim
|
|||
curs.execute(stmt)
|
||||
|
||||
|
||||
@limit_concurrent(1) # Only one refresh at a time.
|
||||
@wrol_mode_check
|
||||
@cancelable_wrapper
|
||||
async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = True):
|
||||
"""Find, model, and index all files in the media directory."""
|
||||
from wrolpi.api_utils import api_app
|
||||
if isinstance(paths, str):
|
||||
paths = [pathlib.Path(paths), ]
|
||||
if isinstance(paths, pathlib.Path):
|
||||
paths = [paths, ]
|
||||
|
||||
idempotency = now()
|
||||
api_app.shared_ctx.refresh['idempotency'] = idempotency
|
||||
|
||||
refreshing_all_files = False
|
||||
|
||||
with flags.refreshing, timer('refresh_files', 'info'):
|
||||
api_app.shared_ctx.refresh['counted_files'] = 0
|
||||
if not paths:
|
||||
refresh_logger.warning('Refreshing all files')
|
||||
refreshing_all_files = True
|
||||
else:
|
||||
refresh_msg = ", ".join(list(map(str, paths)))
|
||||
refresh_logger.warning(f'Refreshing {refresh_msg[:1000]}')
|
||||
if send_events:
|
||||
Events.send_global_refresh_started()
|
||||
|
||||
# Add all files in the media directory to the DB.
|
||||
paths = paths or [get_media_directory()]
|
||||
|
||||
directories = list(filter(lambda i: i.is_dir(), paths))
|
||||
found_directories = set()
|
||||
if directories:
|
||||
with flags.refresh_counting:
|
||||
while directories:
|
||||
directory = directories.pop()
|
||||
try:
|
||||
files, dirs = get_files_and_directories(directory)
|
||||
except PermissionError as e:
|
||||
refresh_logger.error(f'Error refreshing {directory}', exc_info=e)
|
||||
continue
|
||||
directories.extend(dirs)
|
||||
found_directories |= set(dirs)
|
||||
add_files_to_refresh_count(len(files))
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
refresh_logger.info(f'Counted {api_app.shared_ctx.refresh["counted_files"]} files')
|
||||
|
||||
with flags.refresh_discovery:
|
||||
await refresh_discover_paths(paths, idempotency)
|
||||
if send_events:
|
||||
Events.send_global_refresh_discovery_completed()
|
||||
|
||||
# Model all files that have not been indexed.
|
||||
with flags.refresh_modeling:
|
||||
await apply_modelers()
|
||||
if send_events:
|
||||
Events.send_global_refresh_modeling_completed()
|
||||
|
||||
# Index the rest of the files that were not indexed by modelers.
|
||||
with flags.refresh_indexing:
|
||||
await apply_indexers()
|
||||
if send_events:
|
||||
Events.send_global_refresh_indexing_completed()
|
||||
|
||||
# Cleanup any outdated file data.
|
||||
with flags.refresh_cleanup:
|
||||
await apply_refresh_cleanup()
|
||||
|
||||
with get_db_session() as session:
|
||||
parent_directories = {i[0] for i in session.query(Directory.path).filter(Directory.path.in_(paths))}
|
||||
parent_directories |= set(filter(lambda i: i.is_dir(), paths))
|
||||
upsert_directories(parent_directories, found_directories)
|
||||
|
||||
if send_events:
|
||||
Events.send_global_after_refresh_completed()
|
||||
|
||||
refresh_logger.warning('Done refreshing Files')
|
||||
|
||||
if refreshing_all_files:
|
||||
# Only set refresh_complete flag if all files have been refreshed.
|
||||
flags.refresh_complete.set()
|
||||
if send_events:
|
||||
Events.send_refresh_completed()
|
||||
|
||||
api_app.shared_ctx.refresh['counted_files'] = 0
|
||||
api_app.shared_ctx.refresh['idempotency'] = None
|
||||
|
||||
|
||||
async def apply_indexers():
|
||||
"""Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function."""
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
|
@ -1058,8 +969,9 @@ def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List
|
|||
results = session.query(FileGroup, Video) \
|
||||
.filter(FileGroup.id.in_(ordered_ids)) \
|
||||
.outerjoin(Video, Video.file_group_id == FileGroup.id)
|
||||
# Order FileGroups by their location in ordered_ids.
|
||||
file_groups: List[Tuple[FileGroup, Video]] = sorted(results, key=lambda i: ordered_ids.index(i[0].id))
|
||||
# Order FileGroups by their location in ordered_ids. Use dict for O(1) lookups instead of O(n) list.index().
|
||||
id_to_order = {id_: idx for idx, id_ in enumerate(ordered_ids)}
|
||||
file_groups: List[Tuple[FileGroup, Video]] = sorted(results, key=lambda i: id_to_order.get(i[0].id, 0))
|
||||
results = list()
|
||||
for extra, (file_group, video) in zip_longest(extras, file_groups):
|
||||
video: Video
|
||||
|
|
@ -1320,14 +1232,14 @@ def get_refresh_progress() -> RefreshProgress:
|
|||
|
||||
progress = RefreshProgress(
|
||||
counted_files=api_app.shared_ctx.refresh.get('counted_files', 0),
|
||||
counting=flags.refresh_counting.is_set(),
|
||||
discovery=flags.refresh_discovery.is_set(),
|
||||
counting=flags.file_worker_counting.is_set(),
|
||||
discovery=flags.file_worker_discovery.is_set(),
|
||||
indexed=int(results['indexed'] or 0),
|
||||
indexing=flags.refresh_indexing.is_set(),
|
||||
indexing=flags.file_worker_indexing.is_set(),
|
||||
modeled=int(results['modeled'] or 0),
|
||||
modeling=flags.refresh_modeling.is_set(),
|
||||
cleanup=flags.refresh_cleanup.is_set(),
|
||||
refreshing=flags.refreshing.is_set(),
|
||||
modeling=flags.file_worker_modeling.is_set(),
|
||||
cleanup=flags.file_worker_cleanup.is_set(),
|
||||
refreshing=flags.file_worker_busy.is_set(),
|
||||
total_file_groups=int(results['total_file_groups'] or 0),
|
||||
unindexed=int(results['unindexed'] or 0),
|
||||
)
|
||||
|
|
@ -1453,260 +1365,35 @@ def _bulk_update_file_groups_db(session: Session, chunk_plan: Dict[pathlib.Path,
|
|||
curs.execute(sql)
|
||||
|
||||
|
||||
def add_files_to_refresh_count(count: int):
|
||||
from wrolpi.api_utils import api_app
|
||||
count = api_app.shared_ctx.refresh.get('counted_files', 0) + count
|
||||
api_app.shared_ctx.refresh['counted_files'] = count
|
||||
|
||||
|
||||
async def _move(session: Session, destination: pathlib.Path, *sources: pathlib.Path) \
|
||||
-> OrderedDict[pathlib.Path, pathlib.Path]:
|
||||
media_directory = get_media_directory()
|
||||
for source in sources:
|
||||
if not str(source).startswith(str(media_directory)):
|
||||
raise FileNotFoundError(f'{source} is not within the media directory')
|
||||
if not str(destination).startswith(str(media_directory)):
|
||||
raise FileNotFoundError(f'{destination} is not within the media directory')
|
||||
|
||||
destination_existed = destination.is_dir()
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# The files that will be moved. [ (old_file, new_file), ... ]
|
||||
plan: Dict[pathlib.Path, pathlib.Path] = dict()
|
||||
# Directories that will need to be cleaned up.
|
||||
old_directories = unique_by_predicate(i if i.is_dir() else i.parent for i in sources)
|
||||
|
||||
def add_file_group_to_plan(file_group_: FileGroup):
|
||||
primary_path_ = file_group_.primary_path
|
||||
if primary_path_ not in plan:
|
||||
new_primary_path_ = destination / primary_path_.name
|
||||
if new_primary_path_.exists():
|
||||
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}')
|
||||
plan[primary_path_] = new_primary_path_
|
||||
|
||||
def add_source_file_group_to_plan(file_group_: FileGroup, source_: pathlib.Path):
|
||||
primary_path_ = file_group_.primary_path
|
||||
if primary_path_ not in plan:
|
||||
new_primary_path_ = destination / source_.name / primary_path_.relative_to(source)
|
||||
if new_primary_path_.exists():
|
||||
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}')
|
||||
plan[primary_path_] = new_primary_path_
|
||||
|
||||
# Sources may be a list of files, we only want to issue a query for one file in each FileGroup.
|
||||
sources = get_unique_files_by_stem(sources)
|
||||
logger.info(f'move got {len(sources)} to move')
|
||||
|
||||
with flags.refresh_counting:
|
||||
for source in sources:
|
||||
if source.is_file():
|
||||
# Get any FileGroups that share the source's stem.
|
||||
files = glob_shared_stem(source)
|
||||
file_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(files)).all()
|
||||
if not file_groups:
|
||||
# No FileGroups for this source file, create one.
|
||||
try:
|
||||
fg = FileGroup.from_paths(session, *files)
|
||||
file_groups = [fg, ]
|
||||
except NoPrimaryFile:
|
||||
for file in files:
|
||||
fg = FileGroup.from_paths(session, file)
|
||||
add_file_group_to_plan(fg)
|
||||
add_files_to_refresh_count(len(file_groups))
|
||||
for fg in file_groups:
|
||||
add_file_group_to_plan(fg)
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
elif source.is_dir():
|
||||
# Use indexed directory column for efficient lookup
|
||||
# Match exact directory or subdirectories
|
||||
stmt = '''SELECT primary_path
|
||||
FROM file_group
|
||||
WHERE directory = :dir
|
||||
OR directory LIKE :dir_pattern'''
|
||||
source_str = str(source)
|
||||
for (primary_path,) in session.execute(
|
||||
stmt, dict(dir=source_str, dir_pattern=f'{source_str}/%')):
|
||||
primary_path = pathlib.Path(primary_path)
|
||||
if primary_path not in plan:
|
||||
new_primary_path = destination / source.name / primary_path.relative_to(source)
|
||||
if new_primary_path.exists():
|
||||
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path}')
|
||||
plan[primary_path] = new_primary_path
|
||||
add_files_to_refresh_count(1)
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
# Move directories of this source.
|
||||
for directory in (i for i in walk(source) if i.is_dir()):
|
||||
new_directory = destination / source.name / directory.relative_to(source)
|
||||
plan[directory] = new_directory
|
||||
|
||||
# Find any files in this directory that are not yet in the DB and add them to the plan.
|
||||
files = {i for i in walk(source) if i.is_file()}
|
||||
if missing_files := files - set(plan.keys()):
|
||||
for paths in group_files_by_stem(missing_files):
|
||||
try:
|
||||
# Only create the FileGroup if a primary_path can be found.
|
||||
get_primary_file(paths)
|
||||
file_group = FileGroup.from_paths(session, *paths)
|
||||
add_files_to_refresh_count(1)
|
||||
add_source_file_group_to_plan(file_group, source)
|
||||
except NoPrimaryFile:
|
||||
# No primary path for these paths, create FileGroups for each.
|
||||
add_files_to_refresh_count(1)
|
||||
for file in paths:
|
||||
fg = FileGroup.from_paths(session, file)
|
||||
add_source_file_group_to_plan(fg, source)
|
||||
|
||||
# Sleep to catch cancel.
|
||||
await asyncio.sleep(0)
|
||||
else:
|
||||
raise UnknownFile(f'Unknown path type, cannot move: {source}')
|
||||
|
||||
# Sort plan by the deepest files first.
|
||||
plan: OrderedDict = OrderedDict(sorted(plan.items(), key=lambda i: (len(i[0].parents), i[0].name), reverse=True))
|
||||
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
logger.trace(f'_move: plan created with {len(plan)} items to {destination}')
|
||||
|
||||
# Revert plan is built out as files are moved.
|
||||
revert_plan = OrderedDict()
|
||||
new_directories = set()
|
||||
|
||||
def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path], is_revert: bool = False):
|
||||
"""Apply the move plan to all the FileGroups.
|
||||
|
||||
@warning: Cannot be cancelled!"""
|
||||
nonlocal new_directories
|
||||
# Query existing directories ONCE before processing chunks (not per-chunk).
|
||||
existing_directories = {i[0] for i in session.query(Directory.path)}
|
||||
inserted_directories = set()
|
||||
|
||||
# Move FileGroups in groups.
|
||||
for chunk in chunks(plan_.items(), MOVE_CHUNK_SIZE):
|
||||
old_paths = [i for i, j in chunk]
|
||||
old_files, old_directories_ = partition(lambda i: i.is_file(), old_paths)
|
||||
file_groups_ = session.query(FileGroup).filter(FileGroup.primary_path.in_(old_files)).all()
|
||||
if len(file_groups_) != len(old_files):
|
||||
# We ensured there are FileGroups while building the plan, maybe user deleted a file?
|
||||
raise RuntimeError('Could not get all FileGroups for move')
|
||||
|
||||
# Build chunk plan for bulk update
|
||||
chunk_plan = {}
|
||||
for file_group_ in file_groups_:
|
||||
old_file = file_group_.primary_path
|
||||
new_primary_path_ = plan_[old_file]
|
||||
parent = new_primary_path_.parent
|
||||
parent.mkdir(parents=True, exist_ok=True)
|
||||
if parent not in new_directories:
|
||||
new_directories.add(parent)
|
||||
# Move physical files
|
||||
_move_file_group_files(file_group_, new_primary_path_)
|
||||
chunk_plan[old_file] = new_primary_path_
|
||||
revert_plan[new_primary_path_] = old_file
|
||||
for directory_ in old_directories_:
|
||||
delete_directory(directory_)
|
||||
|
||||
# Find directories that need to be inserted (not existing and not already inserted).
|
||||
missing_directories = new_directories - existing_directories - inserted_directories
|
||||
if missing_directories:
|
||||
session.add_all([Directory(path=d, name=d.name) for d in missing_directories])
|
||||
inserted_directories.update(missing_directories)
|
||||
|
||||
# Batch update all FileGroups in this chunk with raw SQL
|
||||
_bulk_update_file_groups_db(session, chunk_plan)
|
||||
# Flush Directory inserts
|
||||
session.flush()
|
||||
|
||||
with flags.refresh_discovery:
|
||||
try:
|
||||
do_plan(plan)
|
||||
for source in sources:
|
||||
if source.is_dir():
|
||||
delete_directory(source)
|
||||
logger.info(f'Move execution completed')
|
||||
except Exception as e:
|
||||
logger.error(f'_move failed, reverting {len(revert_plan)} items', exc_info=e)
|
||||
new_directories = set()
|
||||
# Move files back. Get copy because do_plan will change the revert plan.
|
||||
do_plan(revert_plan.copy(), is_revert=True)
|
||||
# Delete the directories that were created.
|
||||
directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True)
|
||||
for path in directories:
|
||||
if path.is_dir():
|
||||
delete_directory(path)
|
||||
# Remove destination only if it did not exist at the start.
|
||||
if destination.is_dir() and destination_existed is False:
|
||||
delete_directory(destination)
|
||||
raise
|
||||
|
||||
# Clean up any old Directory records.
|
||||
for old_directory in old_directories:
|
||||
# Delete directories that do no exist, or are empty.
|
||||
if not old_directory.is_dir() or next(old_directory.iterdir(), None) is None:
|
||||
directories = session.query(Directory).filter(Directory.path.like(str(old_directory)))
|
||||
for directory in directories:
|
||||
session.delete(directory)
|
||||
|
||||
return plan
|
||||
|
||||
|
||||
def delete_directory(directory: pathlib.Path, recursive: bool = False):
|
||||
"""Remove a directory, remove it's Directory record.
|
||||
|
||||
Will refuse to delete a directory if it contains Tagged Files."""
|
||||
if recursive:
|
||||
with get_db_session() as session:
|
||||
# Use indexed directory column for efficient lookup
|
||||
directory_str = str(directory)
|
||||
tagged = session.query(FileGroup) \
|
||||
.filter(or_(
|
||||
FileGroup.directory == directory_str,
|
||||
FileGroup.directory.like(f'{directory_str}/%')
|
||||
)) \
|
||||
.join(TagFile, TagFile.file_group_id == FileGroup.id) \
|
||||
.limit(1).one_or_none()
|
||||
if tagged:
|
||||
raise FileGroupIsTagged(f'Cannot delete {tagged} because it is tagged')
|
||||
shutil.rmtree(directory)
|
||||
else:
|
||||
directory.rmdir()
|
||||
Will refuse to delete a directory if it contains Tagged Files.
|
||||
|
||||
This function is idempotent - safe to call even if the directory was already deleted."""
|
||||
if directory.exists():
|
||||
if recursive:
|
||||
with get_db_session() as session:
|
||||
# Use indexed directory column for efficient lookup
|
||||
directory_str = str(directory)
|
||||
tagged = session.query(FileGroup) \
|
||||
.filter(or_(
|
||||
FileGroup.directory == directory_str,
|
||||
FileGroup.directory.like(f'{directory_str}/%')
|
||||
)) \
|
||||
.join(TagFile, TagFile.file_group_id == FileGroup.id) \
|
||||
.limit(1).one_or_none()
|
||||
if tagged:
|
||||
raise FileGroupIsTagged(f'Cannot delete {tagged} because it is tagged')
|
||||
shutil.rmtree(directory)
|
||||
else:
|
||||
directory.rmdir()
|
||||
# Always clean up DB record even if directory was already deleted
|
||||
with get_db_curs(commit=True) as curs:
|
||||
stmt = 'DELETE FROM directory WHERE path=%s'
|
||||
curs.execute(stmt, (str(directory),))
|
||||
|
||||
|
||||
async def move(session: Session, destination: pathlib.Path, *sources: pathlib.Path) \
|
||||
-> OrderedDict[pathlib.Path, pathlib.Path]:
|
||||
"""Moves a file or directory (recursively), preserving applied Tags.
|
||||
|
||||
If the move fails at any point, the files will be returned to their previous locations.
|
||||
|
||||
Returns an OrderedDict, the key is the file's old location, the value is where the file was moved.
|
||||
"""
|
||||
with flags.refreshing:
|
||||
from wrolpi.api_utils import api_app
|
||||
api_app.shared_ctx.refresh['counted_files'] = 0
|
||||
|
||||
# Move the files, if this fails it will revert itself and raise an error.
|
||||
with timer('moving files', 'info'):
|
||||
plan = await _move(session, destination, *sources)
|
||||
session.commit()
|
||||
|
||||
with flags.refresh_indexing:
|
||||
await apply_indexers()
|
||||
|
||||
with flags.refresh_modeling:
|
||||
await apply_modelers()
|
||||
|
||||
with flags.refresh_cleanup:
|
||||
await apply_refresh_cleanup()
|
||||
# Save tags now that files have been moved.
|
||||
save_tags_config.activate_switch()
|
||||
|
||||
return plan
|
||||
|
||||
|
||||
async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
|
||||
"""Rename a file (and it's associated files). Preserve any tags.
|
||||
|
||||
|
|
@ -1742,6 +1429,8 @@ async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
|
|||
async def rename_directory(session: Session, directory: pathlib.Path, new_name: str, send_events: bool = False) \
|
||||
-> pathlib.Path:
|
||||
"""Rename a directory. This is done by moving all files into the new directory, and removing the old directory."""
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
||||
new_directory = directory.with_name(new_name)
|
||||
if new_directory.exists():
|
||||
raise FileConflict(f'Cannot rename {directory} to {new_directory} because it already exists.')
|
||||
|
|
@ -1758,7 +1447,8 @@ async def rename_directory(session: Session, directory: pathlib.Path, new_name:
|
|||
# Regular directory rename - move all paths into the new directory.
|
||||
paths = list(directory.iterdir())
|
||||
try:
|
||||
await move(session, new_directory, *paths)
|
||||
job_id = file_worker.queue_move(new_directory, paths)
|
||||
await file_worker.wait_for_job(job_id)
|
||||
if send_events:
|
||||
Events.send_file_move_completed(f'Directory has been renamed: {directory}')
|
||||
except Exception:
|
||||
|
|
@ -1766,7 +1456,7 @@ async def rename_directory(session: Session, directory: pathlib.Path, new_name:
|
|||
Events.send_file_move_failed(f'Directory rename has failed: {directory}')
|
||||
raise
|
||||
|
||||
# Update nested Collections (files already moved by move() above)
|
||||
# Update nested Collections (files already moved by file_worker above)
|
||||
directory_str = str(directory)
|
||||
nested_collections = session.query(Collection).filter(
|
||||
Collection.directory.like(f'{directory_str}/%')
|
||||
|
|
@ -1776,8 +1466,9 @@ async def rename_directory(session: Session, directory: pathlib.Path, new_name:
|
|||
new_coll_dir = new_directory / relative
|
||||
await coll.move_collection(new_coll_dir, session, send_events=send_events, with_files=False)
|
||||
|
||||
# Remove the old directory.
|
||||
delete_directory(directory)
|
||||
# Remove the old directory (recursive=True to clean up any empty subdirectories
|
||||
# that remained after files were moved out).
|
||||
delete_directory(directory, recursive=True)
|
||||
|
||||
return new_directory
|
||||
|
||||
|
|
|
|||
|
|
@ -165,12 +165,12 @@ async def test_delete_wrol_mode(async_client, wrol_mode_fixture):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_files_search_recent(test_session, test_directory, async_client, video_file_factory):
|
||||
async def test_files_search_recent(test_session, test_directory, async_client, video_file_factory, refresh_files):
|
||||
"""File search can return the most recently viewed files."""
|
||||
video_file_factory(test_directory / 'foo.mp4'), video_file_factory(test_directory / 'bar.mp4')
|
||||
baz = (test_directory / 'baz.txt')
|
||||
baz.write_text('baz contents')
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
# baz is most recently viewed, foo has not been viewed.
|
||||
bar, baz, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
|
||||
|
|
@ -230,7 +230,7 @@ async def test_files_search(test_session, async_client, make_files_structure, as
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_files_search_any_tag(async_client, test_session, make_files_structure, tag_factory):
|
||||
async def test_files_search_any_tag(async_client, test_session, make_files_structure, tag_factory, refresh_files):
|
||||
one, two = await tag_factory(), await tag_factory()
|
||||
files = [
|
||||
'foo.txt',
|
||||
|
|
@ -238,7 +238,7 @@ async def test_files_search_any_tag(async_client, test_session, make_files_struc
|
|||
'bar.txt',
|
||||
]
|
||||
make_files_structure(files)
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
bar, foobar, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
|
||||
assert bar.primary_path.name == 'bar.txt' \
|
||||
and foo.primary_path.name == 'foo.txt' \
|
||||
|
|
@ -388,10 +388,10 @@ async def test_file_group_tag(async_client):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_directories(async_client, test_session, test_directory, make_files_structure,
|
||||
assert_directories):
|
||||
assert_directories, refresh_files):
|
||||
"""Directories can be searched by name."""
|
||||
make_files_structure(['foo/one.txt', 'foo/two.txt', 'bar/one.txt'])
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert_directories({'foo', 'bar'})
|
||||
# Create directory that was not refreshed.
|
||||
(test_directory / 'baz').mkdir()
|
||||
|
|
@ -745,11 +745,12 @@ async def test_post_upload_new_directory(test_session, async_client, test_direct
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_directory_crud(test_session, async_client, test_directory, assert_directories, assert_files):
|
||||
async def test_directory_crud(test_session, async_client, test_directory, assert_directories, assert_files,
|
||||
refresh_files):
|
||||
"""A directory can be created in a subdirectory. Errors are returned if there are conflicts."""
|
||||
foo = test_directory / 'foo'
|
||||
foo.mkdir()
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert_directories({'foo', })
|
||||
|
||||
# Create a subdirectory.
|
||||
|
|
@ -796,7 +797,7 @@ async def test_directory_crud(test_session, async_client, test_directory, assert
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move(test_session, test_directory, make_files_structure, async_client):
|
||||
async def test_move(test_session, test_directory, make_files_structure, async_client, await_background_tasks):
|
||||
"""Files can be moved up and down the media directory. Destination directories should already exist."""
|
||||
baz, bar = make_files_structure({
|
||||
'foo/bar.txt': 'bar',
|
||||
|
|
@ -815,6 +816,7 @@ async def test_move(test_session, test_directory, make_files_structure, async_cl
|
|||
content = dict(paths=['foo', 'baz.txt'], destination='qux')
|
||||
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
await await_background_tasks()
|
||||
assert not bar.exists()
|
||||
assert (test_directory / 'qux/foo/bar.txt').is_file()
|
||||
|
||||
|
|
@ -822,6 +824,7 @@ async def test_move(test_session, test_directory, make_files_structure, async_cl
|
|||
content = dict(paths=['qux/foo/bar.txt', ], destination='qux')
|
||||
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
await await_background_tasks()
|
||||
assert not bar.exists()
|
||||
assert (test_directory / 'qux/bar.txt').is_file()
|
||||
assert (test_directory / 'qux/foo').is_dir()
|
||||
|
|
@ -830,30 +833,33 @@ async def test_move(test_session, test_directory, make_files_structure, async_cl
|
|||
content = dict(paths=['qux/bar.txt', ], destination='')
|
||||
request, response = await async_client.post('/api/files/move', json=content)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
await await_background_tasks()
|
||||
assert (test_directory / 'bar.txt').is_file()
|
||||
|
||||
# mv bar.txt quux/quuz/bar.txt
|
||||
content = dict(paths=['bar.txt', ], destination='quux/quuz')
|
||||
request, response = await async_client.post('/api/files/move', json=content)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
await await_background_tasks()
|
||||
assert (test_directory / 'quux/quuz/bar.txt').is_file()
|
||||
|
||||
# mv quux/quuz/bar.txt quux/bar.txt
|
||||
content = dict(paths=['quux/quuz/bar.txt', ], destination='quux')
|
||||
request, response = await async_client.post('/api/files/move', json=content)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
await await_background_tasks()
|
||||
assert (test_directory / 'quux/bar.txt').is_file()
|
||||
assert (test_directory / 'qux/foo').is_dir()
|
||||
assert (test_directory / 'quux/quuz').is_dir()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_file(test_session, test_directory, make_files_structure, async_client):
|
||||
async def test_rename_file(test_session, test_directory, make_files_structure, async_client, refresh_files):
|
||||
"""A FileGroup can be renamed. The title and search index is updated."""
|
||||
foo_file, = make_files_structure({
|
||||
'foo/bar/baz.txt': 'asdf',
|
||||
})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
foo_fg = test_session.query(FileGroup).one()
|
||||
assert foo_fg.a_text == 'baz txt'
|
||||
assert foo_fg.d_text == 'asdf'
|
||||
|
|
@ -871,7 +877,7 @@ async def test_rename_file(test_session, test_directory, make_files_structure, a
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_file_with_associated_files_via_api(test_session, test_directory, make_files_structure,
|
||||
async_client, video_bytes, srt_text):
|
||||
async_client, video_bytes, srt_text, refresh_files):
|
||||
"""Renaming a FileGroup via API should rename all associated files.
|
||||
|
||||
Regression test: When renaming "example.mp4" to "example 2.mp4" and back via API,
|
||||
|
|
@ -881,7 +887,7 @@ async def test_rename_file_with_associated_files_via_api(test_session, test_dire
|
|||
'example.mp4': video_bytes,
|
||||
'example.srt': srt_text,
|
||||
})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
# Verify initial state
|
||||
fg = test_session.query(FileGroup).one()
|
||||
|
|
@ -932,11 +938,11 @@ async def test_rename_file_with_associated_files_via_api(test_session, test_dire
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_directory(test_session, test_directory, make_files_structure, async_client):
|
||||
async def test_rename_directory(test_session, test_directory, make_files_structure, async_client, refresh_files):
|
||||
make_files_structure({
|
||||
'foo/bar/baz.txt': 'asdf',
|
||||
})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
bar, foo = test_session.query(Directory).order_by(Directory.name).all()
|
||||
assert foo.name == 'foo' and bar.name == 'bar'
|
||||
|
||||
|
|
@ -952,7 +958,8 @@ async def test_rename_directory(test_session, test_directory, make_files_structu
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_collection_directory(test_session, test_directory, make_files_structure, async_client):
|
||||
async def test_rename_collection_directory(test_session, test_directory, make_files_structure, async_client,
|
||||
refresh_files):
|
||||
"""Renaming a Collection's directory updates the Collection.directory."""
|
||||
from wrolpi.collections.models import Collection
|
||||
|
||||
|
|
@ -965,7 +972,7 @@ async def test_rename_collection_directory(test_session, test_directory, make_fi
|
|||
test_session.commit()
|
||||
collection_id = collection.id
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
# Rename the directory
|
||||
content = dict(path='my_collection', new_name='renamed_collection')
|
||||
|
|
@ -980,7 +987,7 @@ async def test_rename_collection_directory(test_session, test_directory, make_fi
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_directory_with_nested_collections(test_session, test_directory, async_client):
|
||||
async def test_rename_directory_with_nested_collections(test_session, test_directory, async_client, refresh_files):
|
||||
"""Renaming a parent directory updates nested Collection directories and download destinations."""
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.downloader import Download
|
||||
|
|
@ -1009,7 +1016,7 @@ async def test_rename_directory_with_nested_collections(test_session, test_direc
|
|||
test_session.commit()
|
||||
coll_a_id, coll_b_id, download_id = coll_a.id, coll_b.id, download.id
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
# Rename the parent directory
|
||||
content = dict(path='videos', new_name='my_videos')
|
||||
|
|
@ -1035,9 +1042,9 @@ async def test_rename_directory_with_nested_collections(test_session, test_direc
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_directory_recursive(test_session, test_directory, make_files_structure, async_client,
|
||||
assert_files):
|
||||
assert_files, refresh_files):
|
||||
make_files_structure(['dir/foo', 'dir/bar', 'empty'])
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert test_session.query(FileGroup).count() == 3
|
||||
|
||||
content = {'paths': ['dir/', ]}
|
||||
|
|
@ -1050,10 +1057,10 @@ async def test_delete_directory_recursive(test_session, test_directory, make_fil
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_file(test_session, async_client, test_directory, make_files_structure):
|
||||
async def test_get_file(test_session, async_client, test_directory, make_files_structure, refresh_files):
|
||||
"""Can get info about a single file."""
|
||||
make_files_structure({'foo/bar.txt': 'foo contents'})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
content = dict(file='foo/bar.txt')
|
||||
request, response = await async_client.post('/api/files/file', content=json.dumps(content))
|
||||
|
|
@ -1068,7 +1075,7 @@ async def test_get_file(test_session, async_client, test_directory, make_files_s
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ignore_directory(test_session, async_client, test_directory, make_files_structure, test_wrolpi_config,
|
||||
await_switches):
|
||||
await_switches, refresh_files):
|
||||
"""A maintainer can ignore/un-ignore directories. The files in the directory should not be refreshed."""
|
||||
# Remove any default ignored directories.
|
||||
get_wrolpi_config().ignored_directories = []
|
||||
|
|
@ -1082,7 +1089,7 @@ async def test_ignore_directory(test_session, async_client, test_directory, make
|
|||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(get_wrolpi_config().ignored_directories) == 1
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
files = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
|
||||
assert {i.primary_path.name for i in files} == {'foo.txt', 'bar.txt', 'wrolpi.yaml'}
|
||||
|
|
@ -1092,7 +1099,7 @@ async def test_ignore_directory(test_session, async_client, test_directory, make
|
|||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(get_wrolpi_config().ignored_directories) == 0
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
files = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
|
||||
# Ignore configs.
|
||||
|
|
|
|||
|
|
@ -5,13 +5,13 @@ from http import HTTPStatus
|
|||
import pytest
|
||||
|
||||
from wrolpi.files.ebooks import EBook, EPUB_MIMETYPE
|
||||
from wrolpi.files.lib import refresh_files, move
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.files.worker import file_worker
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index(async_client, test_session, test_directory, example_epub, example_mobi):
|
||||
async def test_index(async_client, test_session, test_directory, example_epub, example_mobi, refresh_files):
|
||||
"""Ebooks can be indexed.
|
||||
|
||||
Covers can be discovered."""
|
||||
|
|
@ -57,7 +57,8 @@ async def test_index(async_client, test_session, test_directory, example_epub, e
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_local_cover(test_session, test_directory, example_epub, image_bytes_factory, await_switches):
|
||||
async def test_discover_local_cover(test_session, test_directory, example_epub, image_bytes_factory, await_switches,
|
||||
refresh_files):
|
||||
cover_path = example_epub.with_suffix('.jpg')
|
||||
cover_path.write_bytes(image_bytes_factory())
|
||||
await refresh_files()
|
||||
|
|
@ -69,7 +70,7 @@ async def test_discover_local_cover(test_session, test_directory, example_epub,
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_cover(test_session, test_directory, example_epub, await_switches):
|
||||
async def test_extract_cover(test_session, test_directory, example_epub, await_switches, refresh_files):
|
||||
"""First image is extracted from the Ebook and used as the cover."""
|
||||
await refresh_files()
|
||||
|
||||
|
|
@ -115,7 +116,7 @@ async def test_search_ebooks(test_session, async_client, example_epub):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_discover_calibre_cover(test_session, async_client, test_directory, example_epub, example_mobi,
|
||||
image_file):
|
||||
image_file, refresh_files):
|
||||
"""Calibre puts a cover near an ebook file, test if it can be found."""
|
||||
# Create a Calibre metadata file.
|
||||
metadata = test_directory / 'metadata.opf'
|
||||
|
|
@ -139,7 +140,8 @@ async def test_discover_calibre_cover(test_session, async_client, test_directory
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_ebook(async_client, test_session, test_directory, example_epub, image_file, tag_factory):
|
||||
async def test_move_ebook(async_client, test_session, test_directory, example_epub, image_file, tag_factory,
|
||||
refresh_files, await_background_tasks):
|
||||
"""An ebook is re-indexed when moved."""
|
||||
tag = await tag_factory()
|
||||
shutil.move(image_file, example_epub.with_suffix('.jpg'))
|
||||
|
|
@ -158,7 +160,9 @@ async def test_move_ebook(async_client, test_session, test_directory, example_ep
|
|||
|
||||
new_directory = test_directory / 'new'
|
||||
new_directory.mkdir()
|
||||
await move(test_session, new_directory, ebook.file_group.primary_path)
|
||||
file_worker.queue_move(new_directory, [ebook.file_group.primary_path])
|
||||
await await_background_tasks()
|
||||
test_session.expire_all()
|
||||
|
||||
ebook: EBook = test_session.query(EBook).one()
|
||||
assert ebook.file_group.primary_path == test_directory / 'new/ebook example.epub'
|
||||
|
|
|
|||
|
|
@ -107,11 +107,12 @@ async def test_delete_file_link(async_client, test_session, test_directory):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_tagged(await_switches, test_session, make_files_structure, tag_factory, video_bytes):
|
||||
async def test_delete_tagged(await_switches, test_session, make_files_structure, tag_factory, video_bytes,
|
||||
refresh_files):
|
||||
"""Cannot delete a file that has been tagged."""
|
||||
tag = await tag_factory()
|
||||
make_files_structure({'foo/bar.txt': 'asdf', 'foo/bar.mp4': video_bytes})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
# Both files end up in a group.
|
||||
bar = test_session.query(FileGroup).one()
|
||||
bar.add_tag(test_session, tag.id)
|
||||
|
|
@ -170,7 +171,7 @@ def test_split_path_stem_and_suffix(path, full, expected):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_files(async_client, test_session, make_files_structure, assert_file_groups):
|
||||
async def test_refresh_files(async_client, test_session, make_files_structure, assert_file_groups, refresh_files):
|
||||
"""All files in the media directory should be found when calling `refresh_files`"""
|
||||
files = make_files_structure([
|
||||
'foo.txt',
|
||||
|
|
@ -179,7 +180,7 @@ async def test_refresh_files(async_client, test_session, make_files_structure, a
|
|||
])
|
||||
foo, bar, baz = files
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert_file_groups([
|
||||
{'primary_path': 'foo.txt', 'indexed': True},
|
||||
{'primary_path': 'bar.txt', 'indexed': True},
|
||||
|
|
@ -187,23 +188,23 @@ async def test_refresh_files(async_client, test_session, make_files_structure, a
|
|||
|
||||
baz.unlink()
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert_file_groups([{'primary_path': 'foo.txt', 'indexed': True}, {'primary_path': 'bar.txt', 'indexed': True}])
|
||||
|
||||
foo.unlink()
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert_file_groups([{'primary_path': 'bar.txt', 'indexed': True}])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_group_location(async_client, test_session, make_files_structure):
|
||||
async def test_file_group_location(async_client, test_session, make_files_structure, refresh_files):
|
||||
"""FileGroup can return the URL necessary to preview the FileGroup."""
|
||||
make_files_structure({
|
||||
'foo/one.txt': 'one',
|
||||
'two.txt': 'two',
|
||||
})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
one, two = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
|
||||
assert one.location == '/files?folders=foo&preview=foo%2Fone.txt'
|
||||
|
|
@ -212,10 +213,10 @@ async def test_file_group_location(async_client, test_session, make_files_struct
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_bogus_files(async_client, test_session, make_files_structure, test_directory,
|
||||
assert_file_groups, insert_file_group):
|
||||
assert_file_groups, insert_file_group, refresh_files):
|
||||
"""Bogus files are removed during a refresh."""
|
||||
make_files_structure(['does exist.txt'])
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
insert_file_group([test_directory / 'does not exist.txt'])
|
||||
test_session.commit()
|
||||
|
||||
|
|
@ -225,17 +226,17 @@ async def test_refresh_bogus_files(async_client, test_session, make_files_struct
|
|||
{'primary_path': 'does not exist.txt', 'indexed': True},
|
||||
])
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
assert test_session.query(FileGroup).count() == 1, 'Bogus file was not removed'
|
||||
assert_file_groups([{'primary_path': 'does exist.txt', 'indexed': True}])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_empty_media_directory(async_client, test_session, test_directory):
|
||||
async def test_refresh_empty_media_directory(async_client, test_session, test_directory, refresh_files):
|
||||
"""refresh_paths will refuse to refresh with an empty media directory."""
|
||||
with pytest.raises(UnknownDirectory):
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -392,54 +393,31 @@ async def test_file_group_tag(test_session, make_files_structure, test_directory
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_a_text_no_indexer(async_client, test_session, make_files_structure):
|
||||
async def test_refresh_a_text_no_indexer(async_client, test_session, make_files_structure, refresh_files):
|
||||
"""File.a_text is filled even if the file does not match an Indexer."""
|
||||
make_files_structure(['foo', 'bar-bar'])
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
files = {i.a_text for i in test_session.query(FileGroup)}
|
||||
assert files == {'bar bar bar-bar', 'foo'}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_many_files(async_client, test_session, make_files_structure):
|
||||
async def test_refresh_many_files(async_client, test_session, make_files_structure, refresh_files):
|
||||
"""Used to profile file refreshing"""
|
||||
file_count = 10_000
|
||||
make_files_structure([f'{uuid4()}.txt' for _ in range(file_count)])
|
||||
with timer('first refresh'):
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert test_session.query(FileGroup).count() == file_count
|
||||
|
||||
with timer('second refresh'):
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert test_session.query(FileGroup).count() == file_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_cancel(async_client, test_session, make_files_structure, test_directory):
|
||||
"""Refresh tasks can be canceled."""
|
||||
# Creat a lot of files so the refresh will take too long.
|
||||
make_files_structure([f'{uuid4()}.txt' for _ in range(1_000)])
|
||||
|
||||
async def assert_cancel(task_):
|
||||
# Time the time it takes to cancel.
|
||||
before = datetime.now()
|
||||
# Sleep so the refresh task has time to run.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
# Cancel the refresh (it will be sleeping soon).
|
||||
task_.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task_
|
||||
assert (datetime.now() - before).total_seconds() < 1.2, 'Task took too long. Was the refresh canceled?'
|
||||
|
||||
task = asyncio.create_task(lib.refresh_files())
|
||||
await assert_cancel(task)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mime_type(async_client, test_session, make_files_structure, test_directory, assert_files):
|
||||
async def test_mime_type(async_client, test_session, make_files_structure, test_directory, assert_files, refresh_files):
|
||||
"""Files module uses the `file` command to get the mimetype of each file."""
|
||||
from PIL import Image
|
||||
|
||||
|
|
@ -453,7 +431,7 @@ async def test_mime_type(async_client, test_session, make_files_structure, test_
|
|||
Image.new('RGB', (25, 25), color='grey').save(bar)
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', baz)
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert_files([
|
||||
{'path': 'dir/foo text.txt'},
|
||||
{'path': 'dir/bar.jpeg'},
|
||||
|
|
@ -473,7 +451,7 @@ async def test_mime_type(async_client, test_session, make_files_structure, test_
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_files_indexer(async_client, test_session, make_files_structure, test_directory):
|
||||
async def test_files_indexer(async_client, test_session, make_files_structure, test_directory, refresh_files):
|
||||
"""An Indexer is provided for each file based on it's mimetype or contents."""
|
||||
source_files: List[str] = [
|
||||
'a text file.txt',
|
||||
|
|
@ -495,7 +473,7 @@ async def test_files_indexer(async_client, test_session, make_files_structure, t
|
|||
# Enable slow feature for testing.
|
||||
# TODO can this be sped up to always be included?
|
||||
with mock.patch('modules.videos.lib.EXTRACT_SUBTITLES', True):
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
text_file, zip_file, image_file, unknown_file, video_file \
|
||||
= test_session.query(FileGroup).order_by(FileGroup.primary_path)
|
||||
|
|
@ -525,11 +503,11 @@ async def test_files_indexer(async_client, test_session, make_files_structure, t
|
|||
|
||||
with mock.patch('modules.videos.models.Video.validate') as mock_validate:
|
||||
mock_validate.side_effect = Exception('This should not be called twice')
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
# Change the contents, the file should be re-indexed.
|
||||
text_path.write_text('new text contents')
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
files, total = lib.search_files('new', 10, 0)
|
||||
assert total == 1
|
||||
|
||||
|
|
@ -549,14 +527,14 @@ def test_split_file_name_words(name, expected):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_large_text_indexer(async_client, test_session, make_files_structure):
|
||||
async def test_large_text_indexer(async_client, test_session, make_files_structure, refresh_files):
|
||||
"""
|
||||
Large files have their indexes truncated.
|
||||
"""
|
||||
large, = make_files_structure({
|
||||
'large_file.txt': 'foo ' * 1_000_000,
|
||||
})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
|
||||
assert large.is_file() and large.stat().st_size == 4_000_000
|
||||
|
|
@ -696,7 +674,7 @@ async def test_get_refresh_progress(async_client, test_session):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_files_no_groups(async_client, test_session, test_directory, make_files_structure,
|
||||
zip_file_factory):
|
||||
zip_file_factory, refresh_files):
|
||||
"""Files that share a name, but cannot be grouped into a FileGroup have their own FileGroups."""
|
||||
foo_txt, foo_zip = make_files_structure({
|
||||
'foo.txt': 'text',
|
||||
|
|
@ -704,7 +682,7 @@ async def test_refresh_files_no_groups(async_client, test_session, test_director
|
|||
})
|
||||
assert foo_txt.stat().st_size and foo_zip.stat().st_size
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
# Two distinct FileGroups.
|
||||
assert test_session.query(FileGroup).count() == 2
|
||||
|
|
@ -714,7 +692,7 @@ async def test_refresh_files_no_groups(async_client, test_session, test_director
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_directories(test_session, test_directory, assert_directories, await_switches):
|
||||
async def test_refresh_directories(test_session, test_directory, assert_directories, await_switches, refresh_files):
|
||||
"""
|
||||
Directories are stored when they are discovered. They are removed when they can no longer be found.
|
||||
"""
|
||||
|
|
@ -725,21 +703,21 @@ async def test_refresh_directories(test_session, test_directory, assert_director
|
|||
bar.mkdir()
|
||||
baz.mkdir()
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert_directories({'foo', 'bar', 'baz'})
|
||||
|
||||
# Deleted directory is removed.
|
||||
foo.rmdir()
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert_directories({'bar', 'baz'})
|
||||
|
||||
bar.rmdir()
|
||||
await lib.refresh_files([bar])
|
||||
await refresh_files([bar])
|
||||
assert_directories({'baz', })
|
||||
|
||||
# A new directory can be refreshed directly.
|
||||
foo.mkdir()
|
||||
await lib.refresh_files([foo])
|
||||
await refresh_files([foo])
|
||||
assert_directories({'foo', 'baz'})
|
||||
|
||||
|
||||
|
|
@ -784,115 +762,14 @@ async def test_file_group_merge(async_client, test_session, test_directory, make
|
|||
assert vid.mimetype == 'video/mp4'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move(async_client, test_session, test_directory, make_files_structure, video_bytes,
|
||||
singlefile_contents_factory):
|
||||
"""files.lib.move behaves likes posix mv"""
|
||||
make_files_structure({
|
||||
'foo/bar/video.mp4': video_bytes,
|
||||
'foo/bar/baz/archive.html': (singlefile_text := singlefile_contents_factory()),
|
||||
'foo/bytes.txt': b'text',
|
||||
'foo/text.txt': 'text',
|
||||
})
|
||||
foo = test_directory / 'foo'
|
||||
qux = test_directory / 'qux'
|
||||
await lib.refresh_files()
|
||||
|
||||
# mv foo qux
|
||||
plan = await lib.move(test_session, qux, foo)
|
||||
plan = [(str(i.relative_to(test_directory)), str(j.relative_to(test_directory))) for i, j in plan.items()]
|
||||
# The deepest files are moved first.
|
||||
assert plan == [('foo/bar/baz/archive.html', 'qux/foo/bar/baz/archive.html'),
|
||||
('foo/bar/video.mp4', 'qux/foo/bar/video.mp4'),
|
||||
('foo/bar/baz', 'qux/foo/bar/baz'),
|
||||
('foo/text.txt', 'qux/foo/text.txt'),
|
||||
('foo/bytes.txt', 'qux/foo/bytes.txt'),
|
||||
('foo/bar', 'qux/foo/bar'),
|
||||
]
|
||||
# Files were moved.
|
||||
assert (qux / 'foo/bar/video.mp4').is_file() and (qux / 'foo/bar/video.mp4').read_bytes() == video_bytes
|
||||
assert (qux / 'foo/bar/baz/archive.html').is_file() \
|
||||
and (qux / 'foo/bar/baz/archive.html').read_text() == singlefile_text
|
||||
assert (qux / 'foo/bytes.txt').is_file() and (qux / 'foo/bytes.txt').read_bytes() == b'text'
|
||||
assert (qux / 'foo/text.txt').is_file() and (qux / 'foo/text.txt').read_text() == 'text'
|
||||
# Directories were moved.
|
||||
assert not foo.exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_files(async_client, test_session, test_directory, make_files_structure):
|
||||
"""Files can be moved using files.lib.move."""
|
||||
one, two = make_files_structure({
|
||||
'foo/one.txt': 'one',
|
||||
'two.txt': 'two',
|
||||
})
|
||||
dest = test_directory / 'dest'
|
||||
|
||||
plan = await lib.move(test_session, dest, one, two)
|
||||
assert list(plan.items()) == [
|
||||
(test_directory / 'foo/one.txt', test_directory / 'dest/one.txt'),
|
||||
(test_directory / 'two.txt', test_directory / 'dest/two.txt'),
|
||||
]
|
||||
# one.txt is moved out of foo.
|
||||
assert not (test_directory / 'foo/one.txt').exists(), 'one.txt is lingering in source'
|
||||
assert not (test_directory / 'two.txt').exists(), 'two.txt is lingering in source'
|
||||
assert (dest / 'one.txt').read_text() == 'one', 'one.txt was not moved to destination'
|
||||
assert (dest / 'two.txt').read_text() == 'two', 'two.txt was not moved to destination'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_deep_directory(async_client, test_session, test_directory, make_files_structure):
|
||||
"""Moving files/directories from one deep directory to another is supported."""
|
||||
baz, foo, quuz_mp4, quuz_txt, quux = make_files_structure({
|
||||
'foo/foo.txt': 'foo',
|
||||
'foo/bar/baz.txt': 'baz',
|
||||
'foo/qux/quux.txt': 'quux',
|
||||
'foo/quuz.txt': 'quuz text', # this text file will be moved when it's MP4 is moved.
|
||||
'foo/quuz.mp4': 'quuz mp4',
|
||||
})
|
||||
assert foo.name == 'foo.txt' \
|
||||
and baz.name == 'baz.txt' \
|
||||
and quux.name == 'quux.txt' \
|
||||
and quuz_mp4.name == 'quuz.mp4' \
|
||||
and quuz_txt.name == 'quuz.txt', \
|
||||
'Test directory was not initiated correctly.'
|
||||
bar, qux = baz.parent, quux.parent
|
||||
dest = test_directory / 'deep/dest'
|
||||
|
||||
# mv foo/bar foo/qux foo/quuz.mp4 foo/quuz.txt deep/dest
|
||||
plan = await lib.move(test_session, dest, bar, qux, quuz_mp4, quuz_txt)
|
||||
assert list(plan.items()) == [
|
||||
(test_directory / 'foo/qux/quux.txt', test_directory / 'deep/dest/qux/quux.txt'),
|
||||
(test_directory / 'foo/bar/baz.txt', test_directory / 'deep/dest/bar/baz.txt'),
|
||||
(test_directory / 'foo/quuz.txt', test_directory / 'deep/dest/quuz.txt'),
|
||||
(test_directory / 'foo/quuz.mp4', test_directory / 'deep/dest/quuz.mp4'),
|
||||
]
|
||||
|
||||
# Unrelated files and directories are untouched.
|
||||
assert not (test_directory / 'deep/dest/foo').is_dir(), 'foo/ should not have been moved.'
|
||||
assert (test_directory / 'foo').is_dir(), 'foo/ should not be deleted.'
|
||||
assert (test_directory / 'foo/foo.txt').read_text() == 'foo', 'foo.txt should not be moved.'
|
||||
|
||||
# Files are moved, preserving the name of their directory.
|
||||
assert (test_directory / 'deep/dest/bar').is_dir(), 'foo/bar/ was not moved'
|
||||
assert (test_directory / 'deep/dest/bar/baz.txt').is_file(), 'foo/bar/baz.txt was not moved'
|
||||
assert (test_directory / 'deep/dest/bar/baz.txt').read_text() == 'baz', 'foo/bar/baz.txt has wrong contents.'
|
||||
assert (test_directory / 'deep/dest/qux/quux.txt').read_text() == 'quux', 'quux.txt has the wrong contents'
|
||||
assert (test_directory / 'deep/dest/quuz.txt').read_text() == 'quuz text', 'quuz.txt has the wrong contents'
|
||||
assert (test_directory / 'deep/dest/quuz.mp4').read_text() == 'quuz mp4', 'quuz.mp4 has the wrong contents'
|
||||
|
||||
# Old directory was removed.
|
||||
assert not (test_directory / 'foo/bar').is_dir(), 'bar directory was not moved'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_directory(async_client, test_session, test_directory, make_files_structure,
|
||||
assert_directories):
|
||||
assert_directories, refresh_files):
|
||||
"""A Directory record is deleted when it's directory is deleted."""
|
||||
make_files_structure({
|
||||
'foo/one.txt': 'one',
|
||||
})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert_directories({'foo'})
|
||||
|
||||
bar = test_directory / 'bar'
|
||||
|
|
@ -901,52 +778,15 @@ async def test_move_directory(async_client, test_session, test_directory, make_f
|
|||
assert_directories({'bar'})
|
||||
|
||||
|
||||
count = 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_error(async_client, test_session, test_directory, make_files_structure, video_bytes,
|
||||
singlefile_contents_factory):
|
||||
"""Files are restored when a move fails."""
|
||||
make_files_structure({
|
||||
'foo/bar/video.mp4': video_bytes,
|
||||
'foo/bar/baz/archive.html': (singlefile_text := singlefile_contents_factory()),
|
||||
'foo/bytes.txt': b'text',
|
||||
'foo/text.txt': 'text',
|
||||
})
|
||||
foo = test_directory / 'foo'
|
||||
qux = test_directory / 'qux'
|
||||
|
||||
def mock_shutil_move(*args, **kwargs):
|
||||
# Mock the `shutil.move` with this function which will unexpectedly fail on the 3rd call.
|
||||
global count
|
||||
count += 1
|
||||
if count == 3:
|
||||
raise FileNotFoundError('fake file move failure')
|
||||
return shutil.move(*args, **kwargs)
|
||||
|
||||
with mock.patch('wrolpi.files.lib.shutil.move', mock_shutil_move), pytest.raises(FileNotFoundError):
|
||||
await lib.move(test_session, qux, foo)
|
||||
# The move failed, the files should be moved back.
|
||||
# foo was not deleted.
|
||||
assert foo.is_dir()
|
||||
assert (foo / 'bar/baz/archive.html').is_file() and (foo / 'bar/baz/archive.html').read_text() == singlefile_text
|
||||
assert (foo / 'bar/video.mp4').is_file() and (foo / 'bar/video.mp4').read_bytes() == video_bytes
|
||||
assert (foo / 'bytes.txt').is_file() and (foo / 'bytes.txt').read_bytes() == b'text'
|
||||
assert (foo / 'text.txt').is_file() and (foo / 'text.txt').read_text() == 'text'
|
||||
# Destination did not exist on move, so it was deleted.
|
||||
assert not qux.exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_group_move(async_client, test_session, make_files_structure, test_directory, video_bytes,
|
||||
srt_text):
|
||||
srt_text, refresh_files):
|
||||
"""Test FileGroup's move method"""
|
||||
video, srt = make_files_structure({
|
||||
'video.mp4': video_bytes,
|
||||
'video.srt': srt_text,
|
||||
})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
file_group: FileGroup = test_session.query(FileGroup).one()
|
||||
assert file_group.indexed is True
|
||||
|
||||
|
|
@ -962,14 +802,17 @@ async def test_file_group_move(async_client, test_session, make_files_structure,
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_tagged(async_client, test_session, test_directory, make_files_structure, tag_factory):
|
||||
async def test_move_tagged(async_client, test_session, test_directory, make_files_structure, tag_factory, refresh_files,
|
||||
await_background_tasks):
|
||||
"""A FileGroup's tag is preserved when moved or renamed."""
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
||||
tag = await tag_factory()
|
||||
foo, bar, = make_files_structure({
|
||||
'foo/foo.txt': 'foo',
|
||||
'foo/bar.txt': 'bar',
|
||||
})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
bar_file_group, foo_file_group = test_session.query(FileGroup).order_by(FileGroup.primary_path)
|
||||
bar_file_group.title = 'custom title' # Should not be overwritten.
|
||||
bar_file_group.add_tag(test_session, tag.id)
|
||||
|
|
@ -979,7 +822,10 @@ async def test_move_tagged(async_client, test_session, test_directory, make_file
|
|||
qux.mkdir()
|
||||
|
||||
# Move both files into qux. The Tag should also be moved.
|
||||
await lib.move(test_session, qux, bar, foo)
|
||||
file_worker.queue_move(qux, [bar, foo])
|
||||
await await_background_tasks()
|
||||
test_session.expire_all()
|
||||
|
||||
new_foo = qux / 'foo.txt'
|
||||
new_bar = qux / 'bar.txt'
|
||||
# Files were moved.
|
||||
|
|
@ -1009,7 +855,7 @@ async def test_move_tagged(async_client, test_session, test_directory, make_file
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_html_index(async_client, test_session, test_directory, make_files_structure):
|
||||
async def test_html_index(async_client, test_session, test_directory, make_files_structure, refresh_files):
|
||||
make_files_structure({
|
||||
'archive.html': '''<html>
|
||||
<title>The Title</title>
|
||||
|
|
@ -1041,7 +887,7 @@ async def test_html_index(async_client, test_session, test_directory, make_files
|
|||
'''
|
||||
})
|
||||
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
assert test_session.query(FileGroup).count() == 1, 'Too many files were refreshed.'
|
||||
archive_file: FileGroup = test_session.query(FileGroup).one()
|
||||
|
|
@ -1055,9 +901,9 @@ Outside element.'''
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_doc_indexer(async_client, test_session, example_doc):
|
||||
async def test_doc_indexer(async_client, test_session, example_doc, refresh_files):
|
||||
"""The contents of a Microsoft doc files can be indexed."""
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
doc = test_session.query(FileGroup).one()
|
||||
assert doc.title == 'example word.doc'
|
||||
|
|
@ -1069,9 +915,9 @@ async def test_doc_indexer(async_client, test_session, example_doc):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_docx_indexer(async_client, test_session, example_docx):
|
||||
async def test_docx_indexer(async_client, test_session, example_docx, refresh_files):
|
||||
"""The contents of a Microsoft docx files can be indexed."""
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
doc = test_session.query(FileGroup).one()
|
||||
assert doc.title == 'example word.docx'
|
||||
|
|
@ -1080,9 +926,9 @@ async def test_docx_indexer(async_client, test_session, example_docx):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_search_date_range(async_client, test_session, example_pdf, example_doc):
|
||||
async def test_file_search_date_range(async_client, test_session, example_pdf, example_doc, refresh_files):
|
||||
"""Test searching FileGroups by their published datetime."""
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
doc, pdf = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
|
||||
|
||||
files, total = lib.search_files('', 10, 0)
|
||||
|
|
@ -1153,40 +999,6 @@ async def test_upsert_file_video_with_channel(async_client, test_session, test_d
|
|||
assert video.info_json_path, 'Info json file not found near video file.'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_move_many_files(async_client, test_session, test_directory, make_files_structure):
|
||||
from wrolpi.db import get_db_curs
|
||||
|
||||
make_files_structure([f'foo/{i}.txt' for i in range(10_000)])
|
||||
await lib.refresh_files()
|
||||
|
||||
foo = test_directory / 'foo'
|
||||
bar = test_directory / 'bar'
|
||||
|
||||
# Track get_db_curs calls to verify commit=True is passed for bulk updates.
|
||||
# Without commit=True, raw SQL is rolled back in production.
|
||||
get_db_curs_calls = []
|
||||
original_get_db_curs = get_db_curs
|
||||
|
||||
def tracking_get_db_curs(*args, **kwargs):
|
||||
get_db_curs_calls.append(kwargs)
|
||||
return original_get_db_curs(*args, **kwargs)
|
||||
|
||||
with timer('test_move_many_files'), mock.patch('wrolpi.files.lib.get_db_curs', tracking_get_db_curs):
|
||||
# mv foo bar
|
||||
await lib.move(test_session, bar, foo)
|
||||
|
||||
assert not foo.is_dir()
|
||||
assert bar.is_dir()
|
||||
assert (bar / 'foo').is_dir()
|
||||
assert (bar / 'foo/0.txt').is_file()
|
||||
|
||||
# Verify bulk update used commit=True (required for raw SQL to persist in production)
|
||||
assert get_db_curs_calls, 'get_db_curs should have been called for bulk update'
|
||||
assert all(call.get('commit') is True for call in get_db_curs_calls), \
|
||||
'get_db_curs must be called with commit=True to persist raw SQL in production'
|
||||
|
||||
|
||||
@only_macos
|
||||
def test_get_real_path_name(test_directory):
|
||||
(test_directory / 'foo.txt').touch()
|
||||
|
|
@ -1443,7 +1255,7 @@ def test_sanitize_filename_surrogates_renames_file(test_directory):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_file_with_associated_files_twice(async_client, test_session, test_directory, make_files_structure,
|
||||
video_bytes, srt_text):
|
||||
video_bytes, srt_text, refresh_files):
|
||||
"""
|
||||
Renaming a FileGroup twice should rename all associated files both times.
|
||||
|
||||
|
|
@ -1455,7 +1267,7 @@ async def test_rename_file_with_associated_files_twice(async_client, test_sessio
|
|||
'example.mp4': video_bytes,
|
||||
'example.srt': srt_text,
|
||||
})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
# Verify initial state
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
|
|
@ -1509,7 +1321,7 @@ async def test_rename_file_with_associated_files_twice(async_client, test_sessio
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rename_non_primary_file_renames_filegroup(async_client, test_session, test_directory,
|
||||
make_files_structure, video_bytes, srt_text):
|
||||
make_files_structure, video_bytes, srt_text, refresh_files):
|
||||
"""
|
||||
Renaming a non-primary file (like a subtitle) should rename the entire FileGroup.
|
||||
|
||||
|
|
@ -1521,7 +1333,7 @@ async def test_rename_non_primary_file_renames_filegroup(async_client, test_sess
|
|||
'example.mp4': video_bytes,
|
||||
'example.srt': srt_text,
|
||||
})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
# Verify initial state
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import pytest
|
||||
|
||||
from wrolpi.errors import UnknownFile
|
||||
from wrolpi.files import lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
||||
|
|
@ -46,10 +45,10 @@ async def test_file_group_move(async_client, test_session, test_directory, video
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_tag_overload(async_client, test_session, make_files_structure, tag_factory):
|
||||
async def test_add_tag_overload(async_client, test_session, make_files_structure, tag_factory, refresh_files):
|
||||
"""FileGroup.add_tag and FileGroup.untag use `singeldispatch`."""
|
||||
make_files_structure({'foo.txt': 'foo contents'})
|
||||
await lib.refresh_files()
|
||||
await refresh_files()
|
||||
foo: FileGroup = test_session.query(FileGroup).one()
|
||||
|
||||
one, two = await tag_factory(), await tag_factory()
|
||||
|
|
|
|||
|
|
@ -10,13 +10,13 @@ from wrolpi.files.models import FileGroup
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_title(async_client, test_session, example_pdf):
|
||||
async def test_pdf_title(async_client, test_session, example_pdf, refresh_files):
|
||||
"""PDF modeler extracts a PDFs title and contents."""
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
with mock.patch('wrolpi.files.pdfs.get_pdf_metadata') as mock_get_pdf_metadata:
|
||||
mock_get_pdf_metadata.side_effect = Exception('Should not index twice')
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
|
||||
pdf: FileGroup = test_session.query(FileGroup).one()
|
||||
assert pdf.indexed is True
|
||||
|
|
|
|||
1277
wrolpi/files/test/test_worker.py
Normal file
1277
wrolpi/files/test/test_worker.py
Normal file
File diff suppressed because it is too large
Load diff
532
wrolpi/files/test/test_worker_move.py
Normal file
532
wrolpi/files/test/test_worker_move.py
Normal file
|
|
@ -0,0 +1,532 @@
|
|||
"""Tests for the FileWorker move task."""
|
||||
import pytest
|
||||
|
||||
from wrolpi.files.lib import _move_file_group_files
|
||||
from wrolpi.files.models import FileGroup, Directory
|
||||
from wrolpi.files.worker import file_worker, FileTask, FileTaskType
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_queue_move(async_client, test_session, test_directory, make_files_structure):
|
||||
"""FileWorker accepts and queues move tasks."""
|
||||
file1, = make_files_structure(['source/file1.txt'])
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
# Queue a move task
|
||||
file_worker.queue_move(dest, [file1.parent])
|
||||
|
||||
# Verify task is in the public queue
|
||||
assert file_worker.public_queue.qsize() == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_single_file(async_client, test_session, test_directory, make_files_structure):
|
||||
"""FileWorker moves a single file to destination."""
|
||||
file1, = make_files_structure(['source/file1.txt'])
|
||||
fg = FileGroup.from_paths(test_session, file1)
|
||||
test_session.commit()
|
||||
original_id = fg.id
|
||||
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
task = FileTask(FileTaskType.move, [file1], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Verify file was moved
|
||||
assert (dest / 'file1.txt').exists()
|
||||
assert not file1.exists()
|
||||
|
||||
# Verify FileGroup was updated
|
||||
test_session.expire_all()
|
||||
fg = test_session.query(FileGroup).filter(FileGroup.id == original_id).one()
|
||||
assert str(dest) in str(fg.directory)
|
||||
assert 'file1.txt' in str(fg.primary_path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_directory(async_client, test_session, test_directory, make_files_structure):
|
||||
"""FileWorker moves an entire directory to destination."""
|
||||
files = make_files_structure([
|
||||
'source/file1.txt',
|
||||
'source/file2.txt',
|
||||
'source/subdir/file3.txt',
|
||||
])
|
||||
|
||||
# Create FileGroups
|
||||
for f in files:
|
||||
FileGroup.from_paths(test_session, f)
|
||||
test_session.commit()
|
||||
|
||||
source_dir = test_directory / 'source'
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
task = FileTask(FileTaskType.move, [source_dir], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Verify files were moved
|
||||
assert (dest / 'source' / 'file1.txt').exists()
|
||||
assert (dest / 'source' / 'file2.txt').exists()
|
||||
assert (dest / 'source' / 'subdir' / 'file3.txt').exists()
|
||||
|
||||
# Verify original files are gone
|
||||
assert not (source_dir / 'file1.txt').exists()
|
||||
|
||||
# Verify FileGroups were updated
|
||||
test_session.expire_all()
|
||||
file_groups = test_session.query(FileGroup).all()
|
||||
assert len(file_groups) == 3
|
||||
for fg in file_groups:
|
||||
assert str(dest) in str(fg.directory)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_status_updates(async_client, test_session, test_directory, make_files_structure):
|
||||
"""FileWorker updates status during move operation."""
|
||||
files = make_files_structure([f'source/file{i}.txt' for i in range(5)])
|
||||
|
||||
for f in files:
|
||||
FileGroup.from_paths(test_session, f)
|
||||
test_session.commit()
|
||||
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
task = FileTask(FileTaskType.move, [test_directory / 'source'], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# After completion, status should be reset to idle
|
||||
status = file_worker.status
|
||||
assert status['status'] == 'idle'
|
||||
assert status['destination'] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_preserves_tags(
|
||||
async_client, test_session, test_directory, make_files_structure, tag_factory, await_switches
|
||||
):
|
||||
"""FileWorker preserves tags when moving files."""
|
||||
file1, = make_files_structure(['source/file1.txt'])
|
||||
fg = FileGroup.from_paths(test_session, file1)
|
||||
test_session.commit()
|
||||
|
||||
tag = await tag_factory()
|
||||
fg.add_tag(test_session, tag.id)
|
||||
test_session.commit()
|
||||
await await_switches()
|
||||
|
||||
original_id = fg.id
|
||||
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
task = FileTask(FileTaskType.move, [file1], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Verify tag is preserved
|
||||
test_session.expire_all()
|
||||
fg = test_session.query(FileGroup).filter(FileGroup.id == original_id).one()
|
||||
assert len(fg.tag_files) == 1
|
||||
assert fg.tag_files[0].tag_id == tag.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_with_shared_stem_files(
|
||||
async_client, test_session, test_directory, make_files_structure, video_file_factory
|
||||
):
|
||||
"""FileWorker moves all files sharing a stem together."""
|
||||
docs_dir = test_directory / 'source'
|
||||
docs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
video_path = video_file_factory(docs_dir / 'video1.mp4')
|
||||
info_path = docs_dir / 'video1.info.json'
|
||||
info_path.write_text('{"title": "test"}')
|
||||
|
||||
fg = FileGroup.from_paths(test_session, video_path)
|
||||
fg.files.append({'path': 'video1.info.json', 'mimetype': 'application/json'})
|
||||
test_session.commit()
|
||||
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
task = FileTask(FileTaskType.move, [video_path], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Verify both files were moved
|
||||
assert (dest / 'video1.mp4').exists()
|
||||
assert (dest / 'video1.info.json').exists()
|
||||
assert not video_path.exists()
|
||||
assert not info_path.exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_error_file_exists(
|
||||
async_client, test_session, test_directory, make_files_structure
|
||||
):
|
||||
"""FileWorker handles error when destination file already exists."""
|
||||
source_file, dest_file = make_files_structure([
|
||||
'source/file1.txt',
|
||||
'destination/file1.txt',
|
||||
])
|
||||
|
||||
fg = FileGroup.from_paths(test_session, source_file)
|
||||
test_session.commit()
|
||||
|
||||
dest = test_directory / 'destination'
|
||||
|
||||
task = FileTask(FileTaskType.move, [source_file], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Status should be reset to idle (error handled gracefully)
|
||||
status = file_worker.status
|
||||
assert status['status'] == 'idle'
|
||||
|
||||
# Source file should still exist (move failed)
|
||||
assert source_file.exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_invalid_source(async_client, test_session, test_directory):
|
||||
"""FileWorker handles invalid source paths gracefully."""
|
||||
nonexistent = test_directory / 'nonexistent'
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
task = FileTask(FileTaskType.move, [nonexistent], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Status should be reset to idle
|
||||
status = file_worker.status
|
||||
assert status['status'] == 'idle'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_bulk_files(async_client, test_session, test_directory, make_files_structure):
|
||||
"""FileWorker handles bulk moves efficiently with chunking."""
|
||||
# Create 150+ files to test chunking (MOVE_CHUNK_SIZE = 100)
|
||||
file_count = 150
|
||||
files = make_files_structure([f'source/file{i}.txt' for i in range(file_count)])
|
||||
|
||||
for f in files:
|
||||
FileGroup.from_paths(test_session, f)
|
||||
test_session.commit()
|
||||
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
task = FileTask(FileTaskType.move, [test_directory / 'source'], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Verify all files moved
|
||||
test_session.expire_all()
|
||||
moved_count = test_session.query(FileGroup).filter(
|
||||
FileGroup.directory.like(f'{dest}%')
|
||||
).count()
|
||||
assert moved_count == file_count
|
||||
|
||||
# Verify physical files exist
|
||||
assert (dest / 'source' / 'file0.txt').exists()
|
||||
assert (dest / 'source' / f'file{file_count - 1}.txt').exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_creates_destination_directories(
|
||||
async_client, test_session, test_directory, make_files_structure
|
||||
):
|
||||
"""FileWorker creates destination directories as needed."""
|
||||
file1, = make_files_structure(['source/file1.txt'])
|
||||
fg = FileGroup.from_paths(test_session, file1)
|
||||
test_session.commit()
|
||||
|
||||
# Destination doesn't exist yet
|
||||
dest = test_directory / 'new' / 'nested' / 'destination'
|
||||
assert not dest.exists()
|
||||
|
||||
task = FileTask(FileTaskType.move, [file1], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Verify destination was created and file moved
|
||||
assert dest.exists()
|
||||
assert (dest / 'file1.txt').exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_multiple_sources(
|
||||
async_client, test_session, test_directory, make_files_structure
|
||||
):
|
||||
"""FileWorker moves multiple source files to one destination."""
|
||||
file1, file2 = make_files_structure([
|
||||
'source1/file1.txt',
|
||||
'source2/file2.txt',
|
||||
])
|
||||
|
||||
fg1 = FileGroup.from_paths(test_session, file1)
|
||||
fg2 = FileGroup.from_paths(test_session, file2)
|
||||
test_session.commit()
|
||||
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
task = FileTask(FileTaskType.move, [file1, file2], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Verify both files moved
|
||||
assert (dest / 'file1.txt').exists()
|
||||
assert (dest / 'file2.txt').exists()
|
||||
assert not file1.exists()
|
||||
assert not file2.exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_already_moved_files(
|
||||
async_client, test_session, test_directory, make_files_structure
|
||||
):
|
||||
"""FileWorker handles files that were already moved on filesystem.
|
||||
|
||||
When a user manually moves files and then tells the worker to move them,
|
||||
the worker should detect the files are already at the destination and
|
||||
simply update the database records.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
# Create file and FileGroup at original location
|
||||
file1, = make_files_structure(['source/file1.txt'])
|
||||
fg = FileGroup.from_paths(test_session, file1)
|
||||
test_session.commit()
|
||||
original_id = fg.id
|
||||
|
||||
# User manually moves file on filesystem (outside of WROLPi)
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
shutil.move(str(file1), str(dest / 'file1.txt'))
|
||||
|
||||
# Verify: source gone, dest exists
|
||||
assert not file1.exists()
|
||||
assert (dest / 'file1.txt').exists()
|
||||
|
||||
# Tell worker to move (from old location to new)
|
||||
task = FileTask(FileTaskType.move, [file1], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Verify FileGroup was updated to new location
|
||||
test_session.expire_all()
|
||||
fg = test_session.query(FileGroup).filter(FileGroup.id == original_id).one()
|
||||
assert str(dest) in str(fg.directory)
|
||||
assert 'file1.txt' in str(fg.primary_path)
|
||||
|
||||
# Status should be idle (success)
|
||||
assert file_worker.status['status'] == 'idle'
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests for missing scenarios from files.lib.move
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_rollback_on_failure(
|
||||
async_client, test_session, test_directory, make_files_structure, monkeypatch
|
||||
):
|
||||
"""FileWorker rolls back moved files when an error occurs mid-move."""
|
||||
files = make_files_structure([
|
||||
'source/file1.txt',
|
||||
'source/file2.txt',
|
||||
'source/file3.txt',
|
||||
])
|
||||
|
||||
for f in files:
|
||||
FileGroup.from_paths(test_session, f)
|
||||
test_session.commit()
|
||||
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
# Patch _move_file_group_files to fail after first file
|
||||
call_count = 0
|
||||
original_move = _move_file_group_files
|
||||
|
||||
def failing_move(fg, new_path):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count > 1:
|
||||
raise IOError("Simulated disk failure")
|
||||
return original_move(fg, new_path)
|
||||
|
||||
monkeypatch.setattr('wrolpi.files.worker._move_file_group_files', failing_move)
|
||||
|
||||
task = FileTask(FileTaskType.move, [test_directory / 'source'], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# All source files should still exist (rolled back)
|
||||
assert files[0].exists(), f"file1.txt should exist after rollback"
|
||||
assert files[1].exists(), f"file2.txt should exist after rollback"
|
||||
assert files[2].exists(), f"file3.txt should exist after rollback"
|
||||
|
||||
# Destination should be empty or not exist
|
||||
dest_source = dest / 'source'
|
||||
assert not dest_source.exists() or not list(dest_source.iterdir()), \
|
||||
"Destination should be empty after rollback"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_unindexed_files(
|
||||
async_client, test_session, test_directory, make_files_structure
|
||||
):
|
||||
"""FileWorker creates FileGroups for files not yet in database."""
|
||||
# Create files but don't add them all to DB
|
||||
files = make_files_structure([
|
||||
'source/indexed.txt',
|
||||
'source/unindexed.txt',
|
||||
])
|
||||
|
||||
# Only index one file
|
||||
FileGroup.from_paths(test_session, files[0])
|
||||
test_session.commit()
|
||||
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
task = FileTask(FileTaskType.move, [test_directory / 'source'], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Both files should be moved
|
||||
assert (dest / 'source' / 'indexed.txt').exists()
|
||||
assert (dest / 'source' / 'unindexed.txt').exists()
|
||||
|
||||
# Both should have FileGroups now
|
||||
test_session.expire_all()
|
||||
file_groups = test_session.query(FileGroup).all()
|
||||
assert len(file_groups) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_preserves_existing_destination_on_failure(
|
||||
async_client, test_session, test_directory, make_files_structure, monkeypatch
|
||||
):
|
||||
"""FileWorker preserves existing destination directory on failure."""
|
||||
source_file, existing_file = make_files_structure([
|
||||
'source/file1.txt',
|
||||
'destination/existing.txt', # Pre-existing file
|
||||
])
|
||||
|
||||
FileGroup.from_paths(test_session, source_file)
|
||||
test_session.commit()
|
||||
|
||||
dest = test_directory / 'destination'
|
||||
|
||||
# Patch to fail
|
||||
def failing_move(fg, new_path):
|
||||
raise IOError("Simulated failure")
|
||||
|
||||
monkeypatch.setattr('wrolpi.files.worker._move_file_group_files', failing_move)
|
||||
|
||||
task = FileTask(FileTaskType.move, [source_file], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Destination should still exist with original file
|
||||
assert dest.exists(), "Existing destination should be preserved"
|
||||
assert existing_file.exists(), "Existing file should be preserved"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_removes_created_destination_on_failure(
|
||||
async_client, test_session, test_directory, make_files_structure, monkeypatch
|
||||
):
|
||||
"""FileWorker removes destination directory it created when move fails."""
|
||||
source_file, = make_files_structure(['source/file1.txt'])
|
||||
|
||||
FileGroup.from_paths(test_session, source_file)
|
||||
test_session.commit()
|
||||
|
||||
dest = test_directory / 'new_destination'
|
||||
assert not dest.exists() # Doesn't exist yet
|
||||
|
||||
# Patch to fail
|
||||
def failing_move(fg, new_path):
|
||||
raise IOError("Simulated failure")
|
||||
|
||||
monkeypatch.setattr('wrolpi.files.worker._move_file_group_files', failing_move)
|
||||
|
||||
task = FileTask(FileTaskType.move, [source_file], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Destination should NOT exist (was created by move and should be cleaned up)
|
||||
assert not dest.exists(), "Created destination should be removed on failure"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_worker_move_cleans_directory_records(
|
||||
async_client, test_session, test_directory, make_files_structure
|
||||
):
|
||||
"""FileWorker removes Directory records for directories that no longer exist.
|
||||
|
||||
Directories that still exist (even if empty) keep their DB records.
|
||||
"""
|
||||
files = make_files_structure([
|
||||
'source/subdir/file1.txt',
|
||||
])
|
||||
|
||||
# Create directory records
|
||||
source_dir = test_directory / 'source'
|
||||
subdir = source_dir / 'subdir'
|
||||
test_session.add(Directory(path=str(source_dir), name='source'))
|
||||
test_session.add(Directory(path=str(subdir), name='subdir'))
|
||||
|
||||
FileGroup.from_paths(test_session, files[0])
|
||||
test_session.commit()
|
||||
|
||||
dest = test_directory / 'destination'
|
||||
dest.mkdir()
|
||||
|
||||
task = FileTask(FileTaskType.move, [source_dir], destination=dest)
|
||||
file_worker.private_queue.put_nowait(task)
|
||||
|
||||
await file_worker.process_queue()
|
||||
|
||||
test_session.expire_all()
|
||||
|
||||
# subdir no longer exists - its DB record should be deleted
|
||||
subdir_record = test_session.query(Directory).filter(
|
||||
Directory.path == str(subdir)
|
||||
).one_or_none()
|
||||
assert subdir_record is None, "subdir Directory record should be deleted (directory no longer exists)"
|
||||
assert not subdir.exists(), "subdir should not exist on disk"
|
||||
|
||||
# source still exists (empty) - its DB record should remain
|
||||
source_record = test_session.query(Directory).filter(
|
||||
Directory.path == str(source_dir)
|
||||
).one_or_none()
|
||||
assert source_record is not None, "source Directory record should remain (directory still exists)"
|
||||
assert source_dir.exists(), "source should still exist on disk (empty)"
|
||||
1797
wrolpi/files/worker.py
Normal file
1797
wrolpi/files/worker.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -10,7 +10,7 @@ from wrolpi.common import logger, Base
|
|||
|
||||
__all__ = [
|
||||
'get_flags',
|
||||
'refreshing',
|
||||
'file_worker_busy',
|
||||
'refresh_complete',
|
||||
]
|
||||
|
||||
|
|
@ -106,8 +106,6 @@ class Flag:
|
|||
|
||||
# Set by `check_db_is_up`.
|
||||
db_up = Flag('db_up')
|
||||
# The global refresh is running.
|
||||
refreshing = Flag('refreshing')
|
||||
# Outdated Zims need to be removed.
|
||||
outdated_zims = Flag('outdated_zims', store_db=True)
|
||||
# Map Importing
|
||||
|
|
@ -115,12 +113,14 @@ map_importing = Flag('map_importing')
|
|||
# Used to disable or enable downloading when Internet is down.
|
||||
have_internet = Flag('have_internet')
|
||||
|
||||
# Steps of refreshing.
|
||||
refresh_counting = Flag('refresh_counting')
|
||||
refresh_discovery = Flag('refresh_discovery')
|
||||
refresh_modeling = Flag('refresh_modeling')
|
||||
refresh_indexing = Flag('refresh_indexing')
|
||||
refresh_cleanup = Flag('refresh_cleanup')
|
||||
# The FileWorker is busy (refreshing, moving, tagging, or renaming files).
|
||||
file_worker_busy = Flag('file_worker_busy')
|
||||
# Steps of FileWorker operations.
|
||||
file_worker_counting = Flag('file_worker_counting')
|
||||
file_worker_discovery = Flag('file_worker_discovery')
|
||||
file_worker_modeling = Flag('file_worker_modeling')
|
||||
file_worker_indexing = Flag('file_worker_indexing')
|
||||
file_worker_cleanup = Flag('file_worker_cleanup')
|
||||
|
||||
# The global refresh has been performed. This is False on a fresh instance of WROLPi.
|
||||
refresh_complete = Flag('refresh_complete', store_db=True)
|
||||
|
|
@ -130,16 +130,16 @@ def get_flags() -> dict:
|
|||
"""Return a list of all Flags which are set."""
|
||||
flags = dict(
|
||||
db_up=db_up.is_set(),
|
||||
file_worker_busy=file_worker_busy.is_set(),
|
||||
file_worker_cleanup=file_worker_cleanup.is_set(),
|
||||
file_worker_counting=file_worker_counting.is_set(),
|
||||
file_worker_discovery=file_worker_discovery.is_set(),
|
||||
file_worker_indexing=file_worker_indexing.is_set(),
|
||||
file_worker_modeling=file_worker_modeling.is_set(),
|
||||
have_internet=have_internet.is_set(),
|
||||
map_importing=map_importing.is_set(),
|
||||
outdated_zims=outdated_zims.is_set(),
|
||||
refresh_cleanup=refresh_cleanup.is_set(),
|
||||
refresh_complete=refresh_complete.is_set(),
|
||||
refresh_counting=refresh_counting.is_set(),
|
||||
refresh_discovery=refresh_discovery.is_set(),
|
||||
refresh_indexing=refresh_indexing.is_set(),
|
||||
refresh_modeling=refresh_modeling.is_set(),
|
||||
refreshing=refreshing.is_set(),
|
||||
have_internet=have_internet.is_set(),
|
||||
)
|
||||
return flags
|
||||
|
||||
|
|
|
|||
|
|
@ -510,7 +510,7 @@ async def feed(request: Request, query: schema.EventsRequest):
|
|||
# available.
|
||||
start = now()
|
||||
|
||||
after = None if query.after == 'None' else dates.strpdate(query.after)
|
||||
after = None if query.after in (None, 'None') else dates.strpdate(query.after)
|
||||
events = get_events(after)
|
||||
return json_response(dict(events=events, now=start))
|
||||
|
||||
|
|
|
|||
|
|
@ -84,16 +84,16 @@ class SettingsRequest:
|
|||
@dataclass
|
||||
class FlagsStatusResponse:
|
||||
db_up: bool
|
||||
file_worker_busy: bool
|
||||
file_worker_cleanup: bool
|
||||
file_worker_counting: bool
|
||||
file_worker_discovery: bool
|
||||
file_worker_indexing: bool
|
||||
file_worker_modeling: bool
|
||||
have_internet: bool
|
||||
map_importing: bool
|
||||
outdated_zims: bool
|
||||
refresh_cleanup: bool
|
||||
refresh_complete: bool
|
||||
refresh_counting: bool
|
||||
refresh_discovery: bool
|
||||
refresh_indexing: bool
|
||||
refresh_modeling: bool
|
||||
refreshing: bool
|
||||
have_internet: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ def register_switch_handler(switch_name: str):
|
|||
DEBUG_LOGGED = False
|
||||
|
||||
|
||||
@perpetual_signal(sleep=0.1, run_while_testing=True)
|
||||
@perpetual_signal(sleep=0.1, run_while_testing=False)
|
||||
async def switch_worker():
|
||||
"""Watches `api_app.shared_ctx.switches` for new activated switches, handles each one at a time."""
|
||||
global DEBUG_LOGGED
|
||||
|
|
@ -99,34 +99,36 @@ async def switch_worker():
|
|||
switch_name = None
|
||||
try:
|
||||
# Get new switch and its context. Handle each switch one at a time.
|
||||
switches_changed.wait(timeout=1)
|
||||
# During testing, don't block the event loop - check for pending switches directly.
|
||||
# In production, use blocking wait which is more efficient for long-running processes.
|
||||
if PYTEST:
|
||||
# Non-blocking: return immediately if no switches pending.
|
||||
# The perpetual_signal decorator will call again in 0.1s.
|
||||
if not api_app.shared_ctx.switches:
|
||||
return
|
||||
else:
|
||||
switches_changed.wait(timeout=1)
|
||||
switches: dict = api_app.shared_ctx.switches
|
||||
with api_app.shared_ctx.switches_lock:
|
||||
switch_name, context = switches.popitem()
|
||||
switches_keys = list(switches.keys())
|
||||
# Call handler with the stored context, await coroutine, if any.
|
||||
handler = SWITCH_HANDLERS[switch_name]
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(f'switch_worker handling {switch_name} of {switches_keys}')
|
||||
else:
|
||||
logger.info(f'switch_worker handling {switch_name} of {len(switches_keys)}')
|
||||
coro = handler(**context)
|
||||
if inspect.iscoroutine(coro):
|
||||
# Handler is async.
|
||||
await coro
|
||||
logger.debug(f'switch_worker completed {switch_name}')
|
||||
except TimeoutError:
|
||||
# `switches_changed` is not set.
|
||||
pass
|
||||
except StopIteration:
|
||||
# No switches to handle.
|
||||
pass
|
||||
except KeyError:
|
||||
except KeyError as e:
|
||||
# `switches` is empty, or `switch_name` not in `SWITCH_HANDLER`
|
||||
if switch_name:
|
||||
logger.critical(f'No switch handler defined for: {switch_name}')
|
||||
finally:
|
||||
# Clear only after task is complete to avoid
|
||||
# Clear only after task is complete to avoid race conditions
|
||||
switches_changed.clear()
|
||||
|
||||
|
||||
|
|
@ -137,13 +139,21 @@ async def await_switches(timeout: int = 10):
|
|||
count = 0
|
||||
while count < (timeout * 10):
|
||||
count += 1
|
||||
if api_app.shared_ctx.switches:
|
||||
await asyncio.sleep(0.1)
|
||||
continue
|
||||
if api_app.shared_ctx.switches_changed.is_set():
|
||||
switches_dict = dict(api_app.shared_ctx.switches) if api_app.shared_ctx.switches else {}
|
||||
switches_changed = api_app.shared_ctx.switches_changed.is_set()
|
||||
|
||||
if switches_dict or switches_changed:
|
||||
# Call switch_worker directly instead of using Sanic signal dispatch.
|
||||
# Signal dispatch causes issues with ASGI testing in Sanic 25.x.
|
||||
try:
|
||||
await switch_worker()
|
||||
except Exception as e:
|
||||
# Log errors but continue processing, matching perpetual_signal behavior
|
||||
logger.error(f'switch_worker had error: {e}')
|
||||
await asyncio.sleep(0.1)
|
||||
continue
|
||||
# All switches handled.
|
||||
break
|
||||
else:
|
||||
logger.error(f'await_switches: TIMEOUT after {count} iterations, switches={list(api_app.shared_ctx.switches.keys())}, changed={api_app.shared_ctx.switches_changed.is_set()}')
|
||||
raise RuntimeError('Timed out waiting for switches. Did you remember to use the `await_switches` fixture?')
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
import contextlib
|
||||
import pathlib
|
||||
from dataclasses import dataclass, field
|
||||
|
|
@ -16,7 +17,7 @@ from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_direc
|
|||
from wrolpi.dates import TZDateTime
|
||||
from wrolpi.db import get_db_curs, get_db_session
|
||||
from wrolpi.downloader import save_downloads_config
|
||||
from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, RefreshConflict, NoPrimaryFile
|
||||
from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, FileWorkerConflict, NoPrimaryFile
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
|
||||
from wrolpi.vars import PYTEST
|
||||
|
|
@ -164,8 +165,8 @@ class Tag(ModelHelper, Base):
|
|||
if (other := self.get_by_name(session, name)) and other.id != self.id:
|
||||
raise InvalidTag('Tag name already taken')
|
||||
|
||||
if flags.refreshing.is_set():
|
||||
raise RefreshConflict('Refusing to rename tag while file refresh is in progress')
|
||||
if flags.file_worker_busy.is_set():
|
||||
raise FileWorkerConflict('Refusing to rename tag while FileWorker is busy')
|
||||
|
||||
old_name = self.name
|
||||
self.name = name
|
||||
|
|
@ -607,7 +608,8 @@ def _delete_extra_tags_directory_paths(directory: pathlib.Path, links: List[path
|
|||
logger.warning(f'Refusing to delete extra Tags Directory directory which contains files: {directory}')
|
||||
else:
|
||||
logger.debug(f'Deleting extra Tags Directory directory: {directory}')
|
||||
directory.rmdir()
|
||||
from wrolpi.files.lib import delete_directory
|
||||
delete_directory(directory)
|
||||
|
||||
|
||||
def create_tags_directory(directory: pathlib.Path):
|
||||
|
|
|
|||
|
|
@ -781,7 +781,8 @@ async def test_batch_api_endpoints(async_client, test_session, test_download_man
|
|||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['retried_count'] == 1
|
||||
test_session.refresh(d1)
|
||||
assert d1.status == 'new'
|
||||
# Status may be 'new' or 'pending' depending on whether the download worker picked it up
|
||||
assert d1.status in ('new', 'pending')
|
||||
|
||||
# Test batch clear endpoint
|
||||
body = dict(download_ids=[d2_id])
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ def assert_events(expected: List[Dict], after=None):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_api_feed(test_session, async_client, example_pdf):
|
||||
async def test_events_api_feed(test_session, async_client, example_pdf, events_fixture):
|
||||
"""Events can be gotten from the API."""
|
||||
# refresh_files() creates a few events.
|
||||
request, response = await async_client.post('/api/files/refresh')
|
||||
|
|
@ -28,6 +28,15 @@ async def test_events_api_feed(test_session, async_client, example_pdf):
|
|||
assert response.status_code == HTTPStatus.OK, response.body
|
||||
assert (result := response.json.get('events'))
|
||||
|
||||
# Verify using the fixture
|
||||
events_fixture.assert_has_event('refresh_completed')
|
||||
events_fixture.assert_has_event('global_after_refresh_completed')
|
||||
events_fixture.assert_has_event('global_refresh_indexing_completed')
|
||||
events_fixture.assert_has_event('global_refresh_modeling_completed')
|
||||
events_fixture.assert_has_event('global_refresh_discovery_completed')
|
||||
events_fixture.assert_has_event('global_refresh_started')
|
||||
|
||||
# Also verify the API returns the same events
|
||||
expected = [
|
||||
dict(event='refresh_completed', subject='refresh'),
|
||||
dict(event='global_after_refresh_completed', subject='refresh'),
|
||||
|
|
@ -41,22 +50,29 @@ async def test_events_api_feed(test_session, async_client, example_pdf):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_history_limit(async_client):
|
||||
async def test_events_history_limit(async_client, events_fixture):
|
||||
"""EVENTS_HISTORY has a limited size."""
|
||||
for i in range(HISTORY_SIZE + 5):
|
||||
Events.send_ready()
|
||||
|
||||
assert len(get_events()) == HISTORY_SIZE
|
||||
# Fixture provides access to events, verify size limit
|
||||
assert len(events_fixture.events) == HISTORY_SIZE
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_events(async_client):
|
||||
async def test_get_events(async_client, events_fixture):
|
||||
"""Test that get_events() can filter by timestamp."""
|
||||
after = now()
|
||||
|
||||
Events.send_global_refresh_started()
|
||||
await asyncio.sleep(1)
|
||||
Events.send_ready()
|
||||
|
||||
# Fixture shows all events
|
||||
events_fixture.assert_has_event('ready')
|
||||
events_fixture.assert_has_event('global_refresh_started')
|
||||
|
||||
# get_events with after parameter filters by timestamp
|
||||
assert_events([dict(event='ready'), dict(event='global_refresh_started')], after=after)
|
||||
|
||||
after = after + timedelta(seconds=0.1)
|
||||
|
|
|
|||
|
|
@ -30,20 +30,20 @@ def test_flags(async_client, test_session, flags_lock):
|
|||
|
||||
def test_flags_with(async_client, flags_lock):
|
||||
"""A Flag can be used with `with` context."""
|
||||
assert flags.refreshing.is_set() is False, 'Flag should not be set by default'
|
||||
assert flags.file_worker_busy.is_set() is False, 'Flag should not be set by default'
|
||||
assert not any(v for k, v in flags.get_flags().items())
|
||||
|
||||
with flags.refreshing:
|
||||
assert flags.refreshing.is_set() is True, 'Flag should be set within context.'
|
||||
assert flags.get_flags()['refreshing']
|
||||
with flags.file_worker_busy:
|
||||
assert flags.file_worker_busy.is_set() is True, 'Flag should be set within context.'
|
||||
assert flags.get_flags()['file_worker_busy']
|
||||
|
||||
assert flags.refreshing.is_set() is False, 'Flag should not be restored after context.'
|
||||
assert flags.file_worker_busy.is_set() is False, 'Flag should not be restored after context.'
|
||||
assert not any(v for k, v in flags.get_flags().items())
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
# Can't do two `refreshing` as once.
|
||||
with flags.refreshing:
|
||||
with flags.refreshing:
|
||||
# Can't do two `file_worker_busy` as once.
|
||||
with flags.file_worker_busy:
|
||||
with flags.file_worker_busy:
|
||||
raise Exception('We should not get here!')
|
||||
|
||||
|
||||
|
|
@ -51,5 +51,5 @@ def test_flags_with(async_client, flags_lock):
|
|||
async def test_flag_wait_for(async_client, flags_lock):
|
||||
"""Wait for throws an error when waiting exceeds timeout."""
|
||||
with pytest.raises(TimeoutError):
|
||||
async with flags.refreshing.wait_for(timeout=1):
|
||||
async with flags.file_worker_busy.wait_for(timeout=1):
|
||||
await asyncio.sleep(2)
|
||||
|
|
|
|||
|
|
@ -54,17 +54,17 @@ async def test_perpetual_signal(await_switches):
|
|||
"""Test that perpetual_signal survives errors."""
|
||||
# Signal event is dispatched again.
|
||||
switch_handler_test2.activate_switch()
|
||||
await asyncio.sleep(1)
|
||||
await await_switches(timeout=2)
|
||||
assert count2.value == 1
|
||||
|
||||
# Error is logged.
|
||||
switch_handler_test2.activate_switch()
|
||||
await asyncio.sleep(1)
|
||||
await await_switches(timeout=2)
|
||||
assert count2.value == 2
|
||||
|
||||
# Switch is dispatched again.
|
||||
switch_handler_test2.activate_switch()
|
||||
await asyncio.sleep(1)
|
||||
await await_switches(timeout=2)
|
||||
assert count2.value == 3
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,17 +7,17 @@ import yaml
|
|||
from wrolpi import tags
|
||||
from wrolpi.common import is_hardlinked, walk
|
||||
from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, UsedTag
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import TagFile, Tag
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tags_file_group_json(async_client, test_session, make_files_structure, tag_factory, example_pdf):
|
||||
async def test_tags_file_group_json(async_client, test_session, make_files_structure, tag_factory, example_pdf,
|
||||
refresh_files):
|
||||
"""The tags of a file are returned in its JSON."""
|
||||
tag_one = await tag_factory()
|
||||
tag_two = await tag_factory()
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
file_group: FileGroup = test_session.query(FileGroup).one()
|
||||
|
||||
file_group.add_tag(test_session, tag_one.id)
|
||||
|
|
@ -54,12 +54,12 @@ async def test_tags_model(async_client, test_session, make_files_structure, tag_
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tags_file_group(async_client, test_session, make_files_structure, tag_factory, video_bytes,
|
||||
image_bytes_factory):
|
||||
image_bytes_factory, refresh_files):
|
||||
"""A FileGroup can be tagged with multiple Tags."""
|
||||
make_files_structure({
|
||||
'video.mp4': video_bytes, 'video.png': image_bytes_factory(),
|
||||
})
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
video_group: FileGroup = test_session.query(FileGroup).one()
|
||||
|
||||
tag_one = await tag_factory()
|
||||
|
|
@ -89,9 +89,9 @@ async def test_tags_file_group(async_client, test_session, make_files_structure,
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tags_config_file(test_session, test_directory, tag_factory, example_pdf, video_file, await_switches,
|
||||
test_tags_config):
|
||||
test_tags_config, refresh_files):
|
||||
"""Test that the config is updated when a FileGroup is tagged."""
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
pdf: FileGroup = FileGroup.get_by_path(test_session, example_pdf)
|
||||
video: FileGroup = FileGroup.get_by_path(test_session, video_file)
|
||||
tag1 = await tag_factory()
|
||||
|
|
@ -208,9 +208,9 @@ async def test_tags_crud(async_client, test_session, example_pdf, assert_tags_co
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_tagged_file(test_session, example_pdf, tag_factory):
|
||||
async def test_delete_tagged_file(test_session, example_pdf, tag_factory, refresh_files):
|
||||
"""You cannot delete a FileGroup if it is tagged."""
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
tag = await tag_factory()
|
||||
|
||||
pdf: FileGroup = test_session.query(FileGroup).one()
|
||||
|
|
@ -351,7 +351,8 @@ async def test_invalid_tag(async_client, test_session, test_directory):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tags_directory(test_session, test_directory, tag_factory, video_factory, example_pdf, await_switches):
|
||||
async def test_tags_directory(test_session, test_directory, tag_factory, video_factory, example_pdf, await_switches,
|
||||
refresh_files):
|
||||
"""Test that Tag Directory is synchronized with database."""
|
||||
readme = test_directory / 'tags/README.txt'
|
||||
|
||||
|
|
@ -454,7 +455,7 @@ async def test_tags_directory(test_session, test_directory, tag_factory, video_f
|
|||
f'First Aid lingers and contains: {[i.name for i in (test_directory / "tags/First Aid").iterdir()]}'
|
||||
|
||||
# Only three FileGroups were created. Tags Directory files are ignored during refresh.
|
||||
await files_lib.refresh_files()
|
||||
await refresh_files()
|
||||
assert test_session.query(FileGroup).count() == 3
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue