FileWorker: Async queue-based file synchronization system

Introduces FileWorker for background file operations with dual-queue
architecture (public/private). Handles refresh, move, and rename tasks
with progress tracking, job completion monitoring, and batch processing.
Compares DB FileGroups to filesystem state using FileGroupDiff to detect
new/modified/deleted files. Designed for memory efficiency on Raspberry Pi.
This commit is contained in:
Roland 2026-01-09 14:01:25 -07:00
parent 1d9b794fca
commit 173284699d
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
49 changed files with 4256 additions and 861 deletions

View file

@ -48,9 +48,9 @@ function FlagsMessages() {
let refreshingMessage;
let refreshRequiredMessage;
// Do not tell the maintainer to refresh the files if they are already refreshing.
if (flags.refreshing) {
// Actively refreshing.
// Do not tell the maintainer to refresh the files if the FileWorker is busy.
if (flags.file_worker_busy) {
// FileWorker is busy.
refreshingMessage = <Message icon>
<Icon name='circle notched' loading/>
<Message.Content>

View file

@ -1174,10 +1174,10 @@ export async function deleteFile(paths) {
}
export async function fetchFilesProgress() {
const response = await apiGet(`${API_URI}/files/refresh_progress`);
const response = await apiGet(`${API_URI}/files/worker_status`);
if (response.ok) {
const json = await response.json();
return json['progress'];
return json['status'];
}
}

View file

@ -542,50 +542,49 @@ export function FilesSearchView({
</>
}
function getPhaseLabel(status) {
switch (status) {
case 'counting':
return 'Counting';
case 'comparing':
case 'upserting':
case 'deleting':
return 'Refreshing';
case 'modeling':
return 'Modeling';
case 'indexing':
return 'Indexing';
case 'planning':
return 'Move: Planning';
case 'moving':
return 'Move: Moving';
case 'reverting':
return 'Reverting';
default:
return null;
}
}
export function FilesRefreshProgress() {
const {progress} = useFilesProgressInterval();
if (!progress) {
return;
if (!progress || progress.status === 'idle') {
return null;
}
const {
discovery,
refreshing,
modeling,
indexing,
cleanup,
indexed,
unindexed,
counted_files,
total_file_groups,
modeled
} = progress;
const {status, operation_total, operation_processed, operation_percent} = progress;
const phaseLabel = getPhaseLabel(status);
if (refreshing) {
// Default is Counting / Step 1. Move progress bar toward the middle to avoid clipping out of the screen.
let params = {value: Math.floor(counted_files / 2), total: counted_files, progress: 'ratio'};
let label = 'Refresh: Counting';
if (discovery) {
params['value'] = total_file_groups;
params['total'] = counted_files;
label = 'Refresh: Discovery'
} else if (modeling) {
params['value'] = modeled;
params['total'] = total_file_groups;
label = 'Refresh: Modeling';
} else if (indexing) {
params['value'] = unindexed;
params['total'] = unindexed + indexed;
label = 'Refresh: Indexing';
} else if (cleanup) {
params['value'] = 3;
params['total'] = 4;
label = 'Refresh: Cleanup';
}
return <Progress active color='violet' {...params}>{label}</Progress>
if (!phaseLabel) {
return null;
}
let label = `Refresh: ${phaseLabel}`;
if (operation_total > 0) {
label = `${label} (${operation_processed.toLocaleString()} / ${operation_total.toLocaleString()})`;
}
return <Progress active color='violet' percent={operation_percent || 0} progress>{label}</Progress>;
}
function FilesPage() {
@ -610,7 +609,7 @@ export function FilesRoute() {
}
const useRefresh = () => {
const refreshing = useStatusFlag('refreshing');
const refreshing = useStatusFlag('file_worker_busy');
const refreshingDirectory = useStatusFlag('refreshing_directory');
const wrolModeEnabled = useWROLMode();

View file

@ -167,7 +167,7 @@ export function NavBar() {
let processingLink;
if (status && status.flags) {
if (status.flags.refreshing) {
if (status.flags.file_worker_busy) {
processingLink = '/files';
} else if (status.flags.map_importing) {
processingLink = '/map/manage';

View file

@ -783,7 +783,7 @@ export const useFilesProgress = () => {
export const useFilesProgressInterval = () => {
const {progress, fetchFilesProgress} = useFilesProgress();
useRecurringTimeout(fetchFilesProgress, 1000 * 3);
useRecurringTimeout(fetchFilesProgress, 1000);
return {progress, fetchFilesProgress};
}

View file

@ -9,11 +9,11 @@ from contextlib import contextmanager
from sanic import Sanic
from sanic.signals import Event
from modules.archive.lib import import_domains_config
from modules.inventory import init_inventory
from modules.inventory.common import import_inventories_config
from modules.videos.lib import import_channels_config, get_videos_downloader_config
from wrolpi import flags
from modules.archive.lib import import_domains_config
from wrolpi import root_api # noqa
from wrolpi import tags
from wrolpi.api_utils import api_app, perpetual_signal
@ -24,6 +24,7 @@ from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initi
from wrolpi.dates import Seconds
from wrolpi.downloader import import_downloads_config, download_manager, get_download_manager_config
from wrolpi.errors import WROLModeEnabled
from wrolpi.files.worker import file_worker
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, INTERNET_SERVER
from wrolpi.version import get_version_string
@ -312,6 +313,12 @@ async def start_single_tasks(app: Sanic):
logger.debug(f'start_single_tasks done')
@perpetual_signal(sleep=1)
async def perpetual_file_worker_queue():
file_worker.transfer_queue()
await file_worker.process_queue()
@perpetual_signal(sleep=1)
async def perpetual_check_log_level():
"""Copies global log level into this Sanic worker's logger."""

View file

@ -185,7 +185,7 @@ async def test_archives_search_no_query(test_session, archive_factory, async_cli
@skip_circleci
@pytest.mark.asyncio
async def test_archive_upload(test_session, async_client, singlefile_contents_factory, make_multipart_form,
await_switches, events_history):
await_switches, events_fixture):
"""Test converting a SingleFile from the SingleFile browser extension to an Archive."""
singlefile_contents = singlefile_contents_factory(title='upload title', url='https://example.com/single-file-url')
forms = [
@ -204,8 +204,7 @@ async def test_archive_upload(test_session, async_client, singlefile_contents_fa
assert archive.screenshot_path.is_file(), 'Screenshot should have been created'
assert archive.file_group.title == 'upload title', 'Title was not correct'
assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted'
events = list(events_history)
assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.'
events_fixture.assert_has_event('upload_archive')
@pytest.mark.asyncio

View file

@ -20,7 +20,6 @@ from wrolpi.api_utils import CustomJSONEncoder
from wrolpi.collections import Collection
from wrolpi.common import get_wrolpi_config
from wrolpi.db import get_db_session
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.test.common import skip_circleci
@ -85,7 +84,7 @@ async def test_relationships(async_client, test_session, example_singlefile):
@pytest.mark.asyncio
async def test_archive_title(async_client, test_session, archive_factory, singlefile_contents_factory):
async def test_archive_title(async_client, test_session, archive_factory, singlefile_contents_factory, refresh_files):
"""An Archive's title can be fetched in multiple ways. This tests from most to least reliable."""
# Create some test files, delete all records for a fresh refresh.
archive_factory(
@ -98,7 +97,7 @@ async def test_archive_title(async_client, test_session, archive_factory, single
async def reset_and_get_archive():
test_session.query(FileGroup).delete()
test_session.commit()
await files_lib.refresh_files()
await refresh_files()
return test_session.query(Archive).one()
archive1: Archive = await reset_and_get_archive()
@ -175,7 +174,8 @@ async def test_archive_refresh_deleted_archive(async_client, test_session, archi
@pytest.mark.asyncio
async def test_fills_contents_with_refresh(async_client, test_session, archive_factory, singlefile_contents_factory):
async def test_fills_contents_with_refresh(async_client, test_session, archive_factory, singlefile_contents_factory,
refresh_files):
"""Refreshing archives fills in any missing contents."""
archive1 = archive_factory('example.com', 'https://example.com/one')
archive2 = archive_factory('example.com', 'https://example.com/one')
@ -208,7 +208,7 @@ async def test_fills_contents_with_refresh(async_client, test_session, archive_f
assert not archive4.file_group.d_text
# Fill the contents.
await files_lib.refresh_files()
await refresh_files()
# The archives will be renamed with their title.
archive1, archive2, archive3, archive4 = test_session.query(Archive).order_by(Archive.id)
assert not archive4.file_group.d_text
@ -291,7 +291,7 @@ def test_get_domains(test_session, archive_factory):
@skip_circleci
@pytest.mark.asyncio
async def test_new_archive(test_session, test_directory, fake_now):
async def test_new_archive(test_session, test_directory, fake_now, async_client):
singlefile, readability, screenshot = make_fake_archive_result()
fake_now(datetime(2000, 1, 1))
archive1 = await model_archive_result('https://example.com', singlefile, readability, screenshot)
@ -334,7 +334,7 @@ async def test_new_archive(test_session, test_directory, fake_now):
@pytest.mark.asyncio
async def test_get_title_from_html(test_directory, test_session, fake_now):
async def test_get_title_from_html(test_directory, test_session, fake_now, async_client):
fake_now(datetime(2000, 1, 1))
singlefile, readability, screenshot = make_fake_archive_result()
archive = await model_archive_result('https://example.com', singlefile, readability, screenshot)
@ -584,11 +584,11 @@ async def test_refresh_archives(test_session, test_directory, async_client, make
@pytest.mark.asyncio
async def test_refresh_archives_index(test_session, make_files_structure):
async def test_refresh_archives_index(test_session, make_files_structure, refresh_files):
"""Archives are indexed using ArchiveIndexer and archive_modeler."""
# The start of a typical singlefile html file.
singlefile_contents = '''<!DOCTYPE html> <html lang="en"><!--
Page saved with SingleFile
Page saved with SingleFile
url: https://example.com
saved date: Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)
--><head><meta charset="utf-8">
@ -609,7 +609,7 @@ async def test_refresh_archives_index(test_session, make_files_structure):
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
})
await files_lib.refresh_files()
await refresh_files()
archive: Archive = test_session.query(Archive).one()
assert archive.singlefile_path == singlefile
@ -625,7 +625,7 @@ async def test_refresh_archives_index(test_session, make_files_structure):
@pytest.mark.asyncio
async def test_archive_meta(async_client, test_session, make_files_structure):
async def test_archive_meta(async_client, test_session, make_files_structure, refresh_files):
# The start of a typical singlefile html file.
singlefile_contents = '''<!DOCTYPE html> <html lang="en">
<script data-vue-meta="ssr" type="application/ld+json">
@ -640,7 +640,7 @@ async def test_archive_meta(async_client, test_session, make_files_structure):
'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>',
})
await files_lib.refresh_files()
await refresh_files()
archive: Archive = test_session.query(Archive).one()
assert archive.singlefile_path == singlefile
@ -655,19 +655,19 @@ async def test_archive_meta(async_client, test_session, make_files_structure):
@pytest.mark.asyncio
async def test_refresh_archives_deleted_singlefile(async_client, test_session, make_files_structure,
singlefile_contents_factory):
singlefile_contents_factory, refresh_files):
"""Removing a Singlefile file from a FileGroup makes that group no longer an Archive."""
singlefile, readability = make_files_structure({
'2022-09-04-16-20-11_The Title.html': singlefile_contents_factory(),
'2022-09-04-16-20-11_The Title.readability.json': '{"url": "https://example.com"}',
})
await files_lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).one().model == 'archive'
assert test_session.query(Archive).count() == 1
# Remove singlefile, FileGroup is no longer an Archive.
singlefile.unlink()
await files_lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).one().model is None
assert test_session.query(Archive).count() == 0
@ -709,7 +709,7 @@ def test_archive_order(test_session, test_directory, archive_factory):
@pytest.mark.asyncio
async def test_archive_download_index(test_session, test_directory, image_bytes_factory):
async def test_archive_download_index(test_session, test_directory, image_bytes_factory, async_client):
"""An Archive is indexed when it is downloaded."""
singlefile = '<html><title>the singlefile</title></html>'
readability = dict(
@ -732,7 +732,7 @@ async def test_archive_download_index(test_session, test_directory, image_bytes_
'Did not store the screenshot'
def test_archive_history(test_session, archive_factory):
def test_archive_history(test_session, archive_factory, async_client):
"""Archive's can have a history of other archives.
Archive's with an empty URL are not associated."""

View file

@ -94,6 +94,7 @@ def video_cleanup():
''')
def claim_videos_for_channels(channel_ids: List[int]):
"""
Assign Videos to specific Channels based on directory paths.

View file

@ -11,7 +11,7 @@ from wrolpi.common import logger, \
get_media_directory, wrol_mode_check, background_task
from wrolpi.db import get_db_curs, get_db_session
from wrolpi.downloader import save_downloads_config, download_manager, Download
from wrolpi.errors import APIError, ValidationError, RefreshConflict
from wrolpi.errors import APIError, ValidationError, FileWorkerConflict
from wrolpi.events import Events
from wrolpi.tags import Tag
from .. import schema
@ -269,8 +269,8 @@ async def tag_channel(session: Session, tag_name: str | None, directory: pathlib
Move the Channel to the new directory, if provided."""
if directory and flags.refreshing.is_set():
raise RefreshConflict('Refusing to move channel while file refresh is in progress')
if directory and flags.file_worker_busy.is_set():
raise FileWorkerConflict('Refusing to move channel while FileWorker is busy')
channel = Channel.find_by_id(session, channel_id)

View file

@ -190,7 +190,7 @@ async def test_channel_empty_url_doesnt_conflict(async_client, test_session, tes
@pytest.mark.asyncio
async def test_channel_download_requires_refresh(
async_client, test_session, mock_video_extract_info, download_channel, video_download_manager,
video_factory, events_history, await_switches):
video_factory, events_fixture, await_switches, await_background_tasks):
"""A Channel cannot be downloaded until it has been refreshed.
Videos already downloaded are not downloaded again."""
@ -228,6 +228,8 @@ async def test_channel_download_requires_refresh(
await video_download_manager.do_downloads()
await video_download_manager.wait_for_all_downloads()
await await_switches()
# Wait for background tasks including channel refresh which processes the file worker queue
await await_background_tasks()
# Channel was refreshed before downloading videos.
assert_refreshed(True)
@ -235,8 +237,10 @@ async def test_channel_download_requires_refresh(
# Only the missing video was downloaded.
assert downloaded_urls == ['https://example.com/2']
# Should not send refresh events because downloads are automated.
assert list(events_history) == []
# Should not send user-facing events - only internal FileWorker events are allowed.
user_facing_events = ['user_notify_message', 'directory_refresh', 'files_refreshed']
for event_name in user_facing_events:
events_fixture.assert_no_event(event_name)
@pytest.mark.asyncio

View file

@ -13,7 +13,8 @@ from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_re
background_task, replace_file
from wrolpi.db import get_db_curs, get_db_session
from wrolpi.downloader import Download
from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
from wrolpi.files.lib import split_path_stem_and_suffix
from wrolpi.files.worker import file_worker
from wrolpi.files.models import FileGroup
from wrolpi.tags import Tag, TagFile
from wrolpi.vars import PYTEST, VIDEO_INFO_JSON_KEYS_TO_CLEAN
@ -900,9 +901,10 @@ class Channel(ModelHelper, Base):
# Perform info_json in background task. Channel will be marked as refreshed after this completes.
async def _():
# Refresh all files within this channel's directory first.
from modules.videos.common import update_view_counts_and_censored
await refresh_files([directory], send_events=send_events)
job_id = file_worker.queue_refresh([directory])
await file_worker.wait_for_job(job_id)
# Update view count second.
from modules.videos.common import update_view_counts_and_censored
await update_view_counts_and_censored(id_)
with get_db_session(commit=True) as session_:
channel_ = cls.find_by_id(session_, id_)

View file

@ -9,12 +9,11 @@ from modules.videos.video import lib as video_lib
from wrolpi.common import get_wrolpi_config
from wrolpi.db import get_db_curs
from wrolpi.downloader import Download
from wrolpi.files.lib import refresh_files
from wrolpi.files.models import FileGroup
@pytest.mark.asyncio
async def test_refresh_videos_index(async_client, test_session, test_directory, video_factory):
async def test_refresh_videos_index(async_client, test_session, test_directory, video_factory, refresh_files):
"""The video modeler indexes video data into the Video's FileGroup."""
video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True)
test_session.commit()
@ -54,12 +53,12 @@ async def test_refresh_videos(async_client, test_session, test_directory, simple
with_info_json=True, with_poster_ext='jpg')
test_session.commit()
video1.file_group.size = video1.file_group.data = None
video1.file_group.paths = list()
video1.file_group.files = []
# video2 is in the test directory.
video2 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True,
with_poster_ext='jpg')
video2.file_group.data = None
video2.file_group.paths = []
video2.file_group.files = []
test_session.commit()
assert not video1.file_group.size, 'video1 should not have size during creation'
@ -76,7 +75,7 @@ async def test_refresh_videos(async_client, test_session, test_directory, simple
with get_db_curs(commit=True) as curs:
stmt = "INSERT INTO file_group (mimetype, directory, primary_path, indexed, files, model)" \
" values ('video/mp4', %(directory)s, %(primary_path)s, true, %(files)s, 'video') RETURNING id"
params = {'directory': str(test_directory), 'primary_path': str(video_file), 'files': list()}
params = {'directory': str(test_directory), 'primary_path': str(video_file), 'files': '[]'}
curs.execute(stmt, params)
video4_id = curs.fetchall()[0][0]
stmt = "INSERT INTO video (file_group_id) values (%(video_id)s)"

View file

@ -11,7 +11,6 @@ from modules.videos.models import Channel, Video
from wrolpi.collections import Collection
from wrolpi.common import get_absolute_media_path, get_wrolpi_config
from wrolpi.downloader import Download, DownloadFrequency
from wrolpi.files import lib as files_lib
from wrolpi.vars import PROJECT_DIR
from .. import common
from ..common import convert_image, update_view_counts_and_censored, extract_video_duration, generate_video_poster, \
@ -411,11 +410,11 @@ async def test_ffprobe_json(async_client, video_file, corrupted_video_file):
@pytest.mark.asyncio
async def test_video_ffprobe_json(async_client, test_session, video_file):
async def test_video_ffprobe_json(async_client, test_session, video_file, refresh_files):
"""ffprobe data is extracted when a video is modeled."""
with mock.patch('modules.videos.lib.extract_video_duration') as mock_extract_video_duration:
mock_extract_video_duration.side_effect = Exception('duration should be from ffprobe json')
await files_lib.refresh_files()
await refresh_files()
video = test_session.query(Video).one()
assert video.ffprobe_json

View file

@ -10,7 +10,6 @@ from modules.videos.lib import parse_video_file_name, validate_video, get_statis
from modules.videos.models import Video, Channel
from wrolpi.common import get_wrolpi_config
from wrolpi.downloader import Download, RSSDownloader
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.vars import PROJECT_DIR
@ -142,7 +141,8 @@ async def test_get_statistics(test_session, video_factory, channel_factory):
@pytest.mark.asyncio
async def test_orphaned_files(async_client, test_session, make_files_structure, test_directory, video_factory):
async def test_orphaned_files(async_client, test_session, make_files_structure, test_directory, video_factory,
refresh_files):
# A Video without associated files is not orphaned.
vid1 = video_factory(title='vid1', with_video_file=True)
# The video files will be removed...
@ -159,7 +159,7 @@ async def test_orphaned_files(async_client, test_session, make_files_structure,
# Remove vid3 video. Poster is now orphaned.
vid3.video_path.unlink()
vid3_poster_path = vid3.poster_path
await files_lib.refresh_files()
await refresh_files()
test_session.commit()
# 6 files when two video files are deleted.

View file

@ -4,7 +4,6 @@ import pytest
from modules.videos.models import Video
from wrolpi.errors import FileGroupIsTagged
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
@ -47,7 +46,8 @@ async def test_delete_video_with_tag(async_client, test_session, video_factory,
@pytest.mark.asyncio
async def test_video_channel_refresh(async_client, test_session, test_directory, channel_factory, video_factory):
async def test_video_channel_refresh(async_client, test_session, test_directory, channel_factory, video_factory,
refresh_files):
"""A Video will be associated with a Channel when it's files are in that Channel's directory."""
# Create a video file in this channel's directory.
channel = channel_factory()
@ -60,7 +60,7 @@ async def test_video_channel_refresh(async_client, test_session, test_directory,
test_session.query(FileGroup).delete()
test_session.commit()
await files_lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).count() == 2
assert test_session.query(Video).count() == 2

View file

@ -22,7 +22,8 @@ from wrolpi.dates import Seconds
from wrolpi.events import Events
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.downloader import DownloadFrequency
from wrolpi.files.lib import refresh_files, split_file_name_words
from wrolpi.files.lib import split_file_name_words
from wrolpi.files.worker import file_worker
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST, DOCKERIZED
@ -550,8 +551,8 @@ async def remove_outdated_zim_files(path: pathlib.Path = None) -> int:
file.unlink()
deleted_count += 1
# Must refresh synchronously to clean up DB before returning.
await refresh_files(outdated)
# Refresh synchronously to clean up DB before returning.
await file_worker.refresh_sync(list(outdated))
await restart_kiwix()

View file

@ -10,15 +10,14 @@ from modules.zim.models import TagZimEntry, ZimSubscription
from wrolpi import tags, flags
from wrolpi.common import DownloadFileInfo, get_wrolpi_config
from wrolpi.downloader import Download, import_downloads_config, get_download_manager_config
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.test.common import skip_circleci
@pytest.mark.asyncio
async def test_get_zim(async_client, test_session, zim_path_factory):
async def test_get_zim(async_client, test_session, zim_path_factory, refresh_files):
zim_path_factory()
await files_lib.refresh_files()
await refresh_files()
assert lib.get_zim(test_session, 1)
@ -27,9 +26,9 @@ async def test_get_zim(async_client, test_session, zim_path_factory):
@pytest.mark.asyncio
async def test_zim_get_entry(async_client, test_session, zim_path_factory):
async def test_zim_get_entry(async_client, test_session, zim_path_factory, refresh_files):
zim_path_factory()
await files_lib.refresh_files()
await refresh_files()
entry: Entry = lib.get_entry('one', 1)
assert entry.path == 'one'
@ -389,13 +388,13 @@ async def test_zim_subscription_download_import(async_client, test_session):
@pytest.mark.asyncio
async def test_zim_modeler(async_client, test_session, zim_factory):
async def test_zim_modeler(async_client, test_session, zim_factory, refresh_files):
"""Zim modeler sets FileGroup title and a_text."""
zim = zim_factory('the_zim_title.zim')
assert zim.file_group.indexed is False
test_session.commit()
await files_lib.refresh_files()
await refresh_files()
fg: FileGroup = test_session.query(FileGroup).one()
assert fg.indexed is True

View file

@ -3,4 +3,7 @@ asyncio_mode = auto
testpaths =
wrolpi
modules
asyncio_default_fixture_loop_scope = function
asyncio_default_fixture_loop_scope = function
# Do not remove timeout - tests should not run forever
timeout = 20
timeout_method = signal

View file

@ -15,14 +15,15 @@ psutil==6.0.0
psycopg2-binary==2.9.9
pypdf==6.4.0
pytest-asyncio==0.24.0
pytest-xdist==3.5.0
pytest==8.3.3
python-docx==1.1.2
python-dotenv==1.0.1
python-magic~=0.4.27
pytz==2024.2
sanic-ext==23.6.0
sanic-testing==22.6.0 # 23.6.0 has error with "expandtabs"
sanic==23.6.0 # This is the latest version of Sanic that avoids this bug: https://github.com/sanic-org/sanic/issues/2921
sanic-ext==25.12.0
sanic-testing==24.6.0
sanic==25.12.0
selenium==4.9.1
srt==3.5.3
tqdm==4.67.1

View file

@ -201,7 +201,9 @@ def perpetual_signal(event: str = None, sleep: int | float = 1, run_while_testin
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
elapsed = int(time() - start)
logger.trace(f'perpetual_signal {event_} took {elapsed} seconds')
if not cancelled:
if not cancelled and not PYTEST:
# In PYTEST mode, don't self-dispatch - the test fixture handles this.
# This prevents "Task was destroyed but it is pending!" errors during teardown.
await asyncio.sleep(sleep)
await api_app.dispatch(event_)

View file

@ -14,7 +14,7 @@ from wrolpi import flags
from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
from wrolpi.db import get_db_session
import pathlib
from wrolpi.errors import RefreshConflict
from wrolpi.errors import FileWorkerConflict
from wrolpi.errors import ValidationError
from wrolpi.events import Events
from wrolpi.tags import Tag
@ -286,7 +286,7 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None:
ValidationError: If collection has no directory
"""
# Local import to avoid circular import: collections -> files -> collections
from wrolpi.files.lib import refresh_files
from wrolpi.files.worker import file_worker
with get_db_session() as session:
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
@ -303,7 +303,7 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None:
directory = collection.directory
# Refresh files asynchronously
asyncio.ensure_future(refresh_files([directory], send_events=send_events))
file_worker.queue_refresh([directory])
if send_events:
relative_dir = get_relative_to_media_directory(directory)
@ -350,7 +350,7 @@ async def tag_collection(
Raises:
UnknownCollection: If collection not found
ValidationError: If tagging requirements not met
RefreshConflict: If a file refresh is in progress and directory change is requested
FileWorkerConflict: If a file operation is in progress and directory change is requested
"""
collection = session.query(Collection).filter_by(id=collection_id).one_or_none()
@ -381,8 +381,8 @@ async def tag_collection(
logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, '
f'{old_directory} -> {target_directory}')
if need_to_move and flags.refreshing.is_set():
raise RefreshConflict('Refusing to move collection while file refresh is in progress')
if need_to_move and flags.file_worker_busy.is_set():
raise FileWorkerConflict('Refusing to move collection while FileWorker is busy')
session.flush()
@ -443,8 +443,8 @@ async def tag_collection(
logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, '
f'{old_directory} -> {target_directory}')
if need_to_move and flags.refreshing.is_set():
raise RefreshConflict('Refusing to move collection while file refresh is in progress')
if need_to_move and flags.file_worker_busy.is_set():
raise FileWorkerConflict('Refusing to move collection while FileWorker is busy')
# Apply the tag
collection.tag_id = tag.id

View file

@ -11,8 +11,9 @@ from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_re
from wrolpi.downloader import Download, save_downloads_config
from wrolpi.errors import ValidationError
from wrolpi.events import Events
from wrolpi.files.lib import move as move_files, refresh_files
from wrolpi.files.worker import file_worker
from wrolpi.files.models import FileGroup
from wrolpi.files.lib import delete_directory
from wrolpi.media_path import MediaPathType
from wrolpi.tags import Tag
from wrolpi.tags import save_tags_config
@ -942,7 +943,7 @@ class Collection(ModelHelper, Base):
self.directory = directory
with flags.refreshing:
with flags.file_worker_busy:
# Update download destinations before moving files.
change_download_destinations(old_directory, directory)
session.commit()
@ -956,14 +957,16 @@ class Collection(ModelHelper, Base):
try:
if not old_directory.exists():
# Old directory does not exist; refresh both
await refresh_files([old_directory, directory])
file_worker.queue_refresh([old_directory, directory])
if send_events:
Events.send_file_move_completed(f'Collection {repr(self.name)} was moved (directory missing)')
else:
files_to_move = list(old_directory.iterdir())
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'move_collection: moving {len(files_to_move)} items from {old_directory}')
await move_files(session, directory, *files_to_move)
if files_to_move:
job_id = file_worker.queue_move(directory, files_to_move)
await file_worker.wait_for_job(job_id)
if send_events:
Events.send_file_move_completed(f'Collection {repr(self.name)} was moved')
except Exception as e:
@ -978,7 +981,7 @@ class Collection(ModelHelper, Base):
finally:
session.commit()
if old_directory.exists() and not next(iter(old_directory.iterdir()), None):
old_directory.rmdir()
delete_directory(old_directory)
else:
# Files already moved, just send event
if send_events:

View file

@ -171,7 +171,7 @@ def test_set_tag_on_directory_less_domain_collection(test_session):
@pytest.mark.asyncio
async def test_tag_collection_removes_tag_and_updates_directory(test_session, test_directory, tag_factory,
await_switches):
await_switches, await_background_tasks):
"""When removing a tag via tag_collection, the directory should be updated if provided."""
from wrolpi.collections.lib import tag_collection
@ -207,7 +207,8 @@ async def test_tag_collection_removes_tag_and_updates_directory(test_session, te
directory=str(untagged_dir), # New directory
)
test_session.commit()
await await_switches()
await await_switches(timeout=2)
await await_background_tasks(timeout=2)
# Verify tag was removed
assert collection.tag_id is None

View file

@ -1801,14 +1801,16 @@ def background_task(coro) -> Task:
The task is stored in a global set of background tasks so the task will not be discarded by the garbage collector.
"""
# Get the coroutine name for better logging (used in task name for debugging stuck tasks)
coro_name = getattr(coro, '__qualname__', None) or getattr(coro, '__name__', None) or str(coro)
async def error_logger():
try:
await coro
except Exception as e:
logger_.error('Background task had error:', exc_info=e)
logger_.error(f'background_task: error in {coro_name}:', exc_info=e)
task = asyncio.create_task(error_logger())
task = asyncio.create_task(error_logger(), name=f'bg:{coro_name}')
add_background_task(task)
return task
@ -1822,13 +1824,66 @@ async def cancel_background_tasks():
await asyncio.gather(*BACKGROUND_TASKS)
async def await_background_tasks():
"""Awaits all background tasks, used only for testing."""
async def await_background_tasks(timeout: int = 15):
"""Awaits all background tasks, used only for testing.
Processes the FileWorker queue concurrently with background tasks to avoid
deadlocks when background tasks wait for file worker jobs to complete.
Args:
timeout: Maximum seconds to wait for background tasks (default 15s)
"""
if not PYTEST:
raise RuntimeError('await_background_tasks should only be used for testing')
for background_task in BACKGROUND_TASKS.copy():
await background_task
from wrolpi.files.worker import file_worker
import time
iteration = 0
max_iterations = 1000 # Safety limit to prevent infinite loops
start_time = time.time()
while BACKGROUND_TASKS or file_worker.public_queue.qsize() > 0 or not file_worker.private_queue.empty():
iteration += 1
elapsed = time.time() - start_time
if elapsed > timeout:
task_names = [t.get_name() if hasattr(t, 'get_name') else str(t) for t in BACKGROUND_TASKS]
logger_.error(f'await_background_tasks: TIMEOUT after {elapsed:.1f}s, {iteration} iterations')
logger_.error(f' BACKGROUND_TASKS={len(BACKGROUND_TASKS)}: {task_names}')
logger_.error(f' public_queue={file_worker.public_queue.qsize()}, private_queue_empty={file_worker.private_queue.empty()}')
# Cancel stuck tasks and clear to prevent cascading failures
for task in list(BACKGROUND_TASKS):
if not task.done():
task.cancel()
BACKGROUND_TASKS.clear()
break
if iteration > max_iterations:
task_names = [t.get_name() if hasattr(t, 'get_name') else str(t) for t in BACKGROUND_TASKS]
logger_.error(f'await_background_tasks: SAFETY LIMIT REACHED after {iteration} iterations')
logger_.error(f' BACKGROUND_TASKS={len(BACKGROUND_TASKS)}: {task_names}')
logger_.error(f' public_queue={file_worker.public_queue.qsize()}, private_queue_empty={file_worker.private_queue.empty()}')
break
# Process file worker queue
file_worker.transfer_queue()
while not file_worker.private_queue.empty():
await file_worker.process_queue()
if not BACKGROUND_TASKS:
continue
# Wait for any background task to complete (with timeout so we can keep processing queue)
done, pending = await asyncio.wait(
BACKGROUND_TASKS.copy(),
timeout=0.1,
return_when=asyncio.FIRST_COMPLETED
)
for task in done:
await task # Re-await to propagate exceptions
BACKGROUND_TASKS.discard(task)
def get_warn_once(message: str, logger__: logging.Logger, level=logging.ERROR):

View file

@ -11,6 +11,7 @@ import pathlib
import shutil
import tempfile
import threading
import time
import zipfile
from abc import ABC
from datetime import datetime
@ -36,7 +37,7 @@ from wrolpi import flags
from wrolpi.api_utils import api_app
from wrolpi.cmd import CommandResult
from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode, timer, TRACE_LEVEL
from wrolpi.common import logger, await_background_tasks as await_background_tasks_
from wrolpi.common import logger, await_background_tasks as await_background_tasks_, BACKGROUND_TASKS
from wrolpi.common import set_test_media_directory, Base, set_test_config
from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts
from wrolpi.dates import set_test_now
@ -111,6 +112,43 @@ def test_debug_logger(request):
yield
@pytest.fixture
def clear_background_tasks(request):
"""Clear BACKGROUND_TASKS before each test and fail if orphaned tasks remain after.
Tests must properly await all background tasks they create. Orphaned tasks indicate
a test is not properly cleaning up, which can cause cascading failures in other tests.
"""
# Clear before test - cancel any orphaned tasks from previous tests
if BACKGROUND_TASKS:
logger.warning(f'clear_background_tasks: clearing {len(BACKGROUND_TASKS)} orphaned tasks BEFORE test')
for task in list(BACKGROUND_TASKS):
task_name = task.get_name() if hasattr(task, 'get_name') else str(task)
logger.warning(f' orphaned task from previous test: {task_name}')
if not task.done():
task.cancel()
BACKGROUND_TASKS.clear()
yield
# After test - fail if orphaned tasks remain
if BACKGROUND_TASKS:
orphaned_tasks = []
for task in list(BACKGROUND_TASKS):
task_name = task.get_name() if hasattr(task, 'get_name') else str(task)
orphaned_tasks.append(f'{task_name} (done={task.done()})')
if not task.done():
task.cancel()
BACKGROUND_TASKS.clear()
# Fail the test
pytest.fail(
f'Test left {len(orphaned_tasks)} orphaned background task(s): {", ".join(orphaned_tasks)}. '
f'Tests must await all background tasks they create using await_background_tasks or '
f'the async_client fixture.'
)
@pytest.fixture
def test_directory() -> Generator[Path, Any, None]:
"""
@ -210,10 +248,11 @@ class AutoAwaitTestClient:
@pytest.fixture
async def async_client(test_directory, test_session) -> AsyncGenerator[AutoAwaitTestClient, Any]:
async def async_client(test_directory, test_session, clear_background_tasks) -> AsyncGenerator[AutoAwaitTestClient, Any]:
"""Get an Async Sanic Test Client with all default routes attached.
Depends on test_session to ensure the database mock is in place for session middleware.
Depends on clear_background_tasks to detect orphaned background tasks.
This client automatically awaits switches and background tasks after
POST, PUT, PATCH, and DELETE requests, and expires session objects.
@ -229,7 +268,6 @@ async def async_client(test_directory, test_session) -> AsyncGenerator[AutoAwait
yield wrapped_client
finally:
api_app.stop()
logger.debug('Destroying async_client')
@pytest.fixture
@ -253,11 +291,14 @@ async def await_background_tasks(async_client, test_session):
After background tasks complete, session objects may be stale, so we automatically
expire them to force a refresh on next access.
Args:
timeout: Optional timeout in seconds (default 15s from await_background_tasks_)
"""
await async_client.get('/api')
async def _await_and_expire():
await await_background_tasks_()
async def _await_and_expire(timeout: int = 15):
await await_background_tasks_(timeout=timeout)
test_session.expire_all()
return _await_and_expire
@ -584,10 +625,66 @@ def run_command_bad_result() -> CommandResult:
return CommandResult(1, False, b'run_command_bad_result stdout', b'run_command_bad_result stderr', 0, 123)
class EventsFixture:
"""Fixture for testing events with rich assertions.
Only returns events created after this fixture was instantiated,
making it safe for parallel test execution.
"""
def __init__(self, events_history: list):
self._history = events_history
# Track the starting index to only see new events during this test
self._start_index = len(events_history)
@property
def events(self) -> List[dict]:
"""Return events captured during this test (most recent first)."""
# Only return events added after fixture creation
new_events = list(self._history[self._start_index:])
return list(reversed(new_events))
def get_by_name(self, event_name: str) -> List[dict]:
"""Get all events with the given name."""
return [e for e in self.events if e['event'] == event_name]
def get_by_subject(self, subject: str) -> List[dict]:
"""Get all events with the given subject."""
return [e for e in self.events if e['subject'] == subject]
def assert_has_event(self, event_name: str, message: str = None) -> None:
"""Assert that an event with the given name exists."""
matching = self.get_by_name(event_name)
assert matching, f"No event found with name '{event_name}'. Events: {[e['event'] for e in self.events]}"
if message:
assert any(message in (e.get('message') or '') for e in matching), \
f"No '{event_name}' event contains message '{message}'"
def assert_no_event(self, event_name: str) -> None:
"""Assert that no event with the given name exists."""
matching = self.get_by_name(event_name)
assert not matching, f"Found unexpected event '{event_name}': {matching}"
def assert_events_contain(self, expected: List[dict]) -> None:
"""Assert that events contain all expected events (order matters, most recent first)."""
from wrolpi.test.common import assert_dict_contains
for exp, actual in zip_longest(expected, self.events):
assert actual is not None, f"Expected event {exp} but no more events found"
assert_dict_contains(actual, exp)
def assert_event_count(self, count: int) -> None:
"""Assert the total number of events."""
assert len(self.events) == count, f"Expected {count} events, got {len(self.events)}"
@pytest.fixture
def events_history(async_client):
"""Give each test its own Events history."""
yield api_app.shared_ctx.events_history
def events_fixture(async_client) -> EventsFixture:
"""Fixture for testing events with rich assertions.
Only sees events created after this fixture is instantiated,
making it safe for parallel test execution.
"""
return EventsFixture(api_app.shared_ctx.events_history)
FLAGS_LOCK = multiprocessing.Lock()
@ -900,3 +997,34 @@ def mock_downloader_download_file():
yield set_contents
async def await_file_worker(timeout: int = 30):
"""Wait for FileWorker to complete all pending tasks. Testing only."""
from wrolpi.files.worker import file_worker
start = time.time()
while time.time() - start < timeout:
file_worker.transfer_queue()
if file_worker.private_queue.qsize() > 0:
await file_worker.process_queue()
elif file_worker.public_queue.qsize() == 0:
break
await asyncio.sleep(0.01)
else:
raise RuntimeError('Timed out waiting for file_worker')
@pytest.fixture
def refresh_files(async_client):
"""Test helper that queues and waits for file refresh."""
async def _refresh(paths=None):
from wrolpi.common import get_media_directory
from wrolpi.files.worker import file_worker
paths = paths or [get_media_directory()]
file_worker.queue_refresh(paths)
await await_file_worker()
return _refresh

View file

@ -76,6 +76,10 @@ def attach_shared_contexts(app: Sanic):
app.shared_ctx.events_lock = multiprocessing.Lock()
app.shared_ctx.events_history = manager.list()
app.shared_ctx.file_worker_public_queue = manager.Queue()
app.shared_ctx.file_worker_status = manager.dict()
app.shared_ctx.file_worker_jobs = manager.dict()
reset_shared_contexts(app)
@ -157,7 +161,29 @@ def reset_shared_contexts(app: Sanic):
except queue.Empty:
break
# File Worker
while True:
# Clear out any pending file tasks.
try:
app.shared_ctx.file_worker_public_queue.get_nowait()
except queue.Empty:
break
app.shared_ctx.file_worker_status.clear()
app.shared_ctx.file_worker_jobs.clear()
app.shared_ctx.file_worker_status.update(dict(
status='idle',
task_type=None,
paths=[],
destination=None,
error=None,
operation_total=0,
operation_processed=0,
operation_percent=0,
))
# Events.
# Clear events_history in place to preserve references held by fixtures
del app.shared_ctx.events_history[:]
app.shared_ctx.single_tasks_started.clear()
app.shared_ctx.flags_initialized.clear()
app.shared_ctx.perpetual_tasks_started.clear()

View file

@ -171,9 +171,9 @@ class InvalidDirectory(APIError):
status_code = HTTPStatus.BAD_REQUEST
class RefreshConflict(APIError):
code = 'REFRESH_CONFLICT'
summary = 'Not possible during file refresh'
class FileWorkerConflict(APIError):
code = 'FILE_WORKER_CONFLICT'
summary = 'Not possible while FileWorker is busy'
status_code = HTTPStatus.CONFLICT

View file

@ -52,6 +52,10 @@ class Events:
def send_directory_refresh(message: str = None):
send_event('directory_refresh', message, subject='refresh')
@staticmethod
def send_files_refreshed(message: str = None):
send_event('files_refreshed', message, subject='refresh')
@staticmethod
def send_deleted(message: str = None):
send_event('deleted', message, subject='deleted')

View file

@ -13,6 +13,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_
background_task, walk, timer, TRACE_LEVEL, unique_by_predicate
from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict
from . import lib, schema
from .worker import file_worker
from ..api_utils import json_response, api_app
from ..events import Events
from ..schema import JSONErrorResponse
@ -76,14 +77,21 @@ async def delete_file(_: Request, body: schema.DeleteRequest):
)
@wrol_mode_check
async def refresh(request: Request):
paths = None
from wrolpi.files.worker import file_worker
if request.body:
media_directory = get_media_directory()
if not isinstance(request.json['paths'], list):
raise ValueError('Can only refresh a list')
paths = [media_directory / i for i in request.json['paths']]
await lib.refresh_files(paths)
# When user explicitly specifies files, don't expand to stem-mates.
# This allows refreshing specific files without discovering related files.
expand_stems = not all(p.is_file() for p in paths)
else:
paths = [get_media_directory()]
expand_stems = True
file_worker.queue_refresh(paths, expand_stems=expand_stems)
return response.empty()
@ -98,6 +106,17 @@ async def refresh_progress(request: Request):
))
@files_bp.get('/worker_status')
@openapi.definition(
summary='Get the FileWorker status with detailed progress'
)
async def worker_status(request: Request):
from wrolpi.files.worker import file_worker
return json_response(dict(
status=dict(file_worker.status),
))
@files_bp.post('/search')
@openapi.definition(
summary='Search Files',
@ -239,7 +258,7 @@ async def post_create_directory(_: Request, body: schema.Directory):
except FileExistsError:
raise FileConflict(f'{path} already exists')
background_task(lib.refresh_files([path, ]))
file_worker.queue_refresh([path])
return response.empty(HTTPStatus.CREATED)
@ -261,13 +280,7 @@ async def post_move(request: Request, body: schema.Move):
if source.is_mount():
raise FileConflict(f'Cannot move mounted directory!')
try:
await lib.move(request.ctx.session, destination, *sources)
Events.send_file_move_completed(f'Moving files to {body.destination} succeeded')
except Exception as e:
Events.send_file_move_failed(f'Moving files to {body.destination} failed!')
raise FileConflict(f'Failed to move {sources} to {destination}') from e
file_worker.queue_move(destination, sources)
return response.empty(HTTPStatus.NO_CONTENT)
@ -296,7 +309,7 @@ async def post_rename(request: Request, body: schema.Rename):
# Refreshing individual files causes the FileGroup's files list to be overwritten
# with only the specified file, losing track of associated files.
parent_dir = new_path.parent
background_task(lib.refresh_files([parent_dir]))
file_worker.queue_refresh([parent_dir])
return response.empty(HTTPStatus.NO_CONTENT)

View file

@ -10,7 +10,6 @@ import re
import shutil
import subprocess
import urllib.parse
from collections import OrderedDict
from itertools import zip_longest
from pathlib import Path
from typing import List, Tuple, Union, Dict, Generator, Iterable, Set
@ -23,11 +22,11 @@ from sqlalchemy.orm import Session
from wrolpi import flags
from wrolpi.cmd import which
from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \
partition, cancelable_wrapper, \
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, \
from wrolpi.common import get_media_directory, wrol_mode_check, logger, \
partition, \
get_files_and_directories, chunks_by_stem, walk, \
get_wrolpi_config, \
timer, chunks, unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory
unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
from wrolpi.db import get_db_session, get_db_curs, mogrify
from wrolpi.downloader import download_manager, Download
@ -50,10 +49,10 @@ except ImportError:
logger = logger.getChild(__name__)
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'search_files',
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics',
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
'move', 'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href']
'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href']
def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
@ -255,10 +254,11 @@ async def delete(*paths: Union[str, pathlib.Path]):
else:
path.unlink()
# Must refresh synchronously to clean up DB before returning.
# Refresh synchronously to clean up DB before returning.
# This ensures deleted FileGroups are removed before the caller continues
# (e.g., upload API recreating the file).
await refresh_files(paths)
from wrolpi.files.worker import file_worker
await file_worker.refresh_sync(list(paths))
def _mimetype_suffix_map(path: Path, mimetype: str):
@ -537,7 +537,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
idempotency = str(idempotency)
non_primary_files = set()
for chunk in chunks_by_stem(files, 100):
for chunk in chunks_by_stem(files, 500):
# Group all files by their shared full-stem. `chunk_by_stem` already sorted the files.
grouped = group_files_by_stem(chunk, pre_sorted=True)
# Convert the groups into file_groups. Extract common information into `file_group.data`.
@ -724,95 +724,6 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim
curs.execute(stmt)
@limit_concurrent(1) # Only one refresh at a time.
@wrol_mode_check
@cancelable_wrapper
async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = True):
"""Find, model, and index all files in the media directory."""
from wrolpi.api_utils import api_app
if isinstance(paths, str):
paths = [pathlib.Path(paths), ]
if isinstance(paths, pathlib.Path):
paths = [paths, ]
idempotency = now()
api_app.shared_ctx.refresh['idempotency'] = idempotency
refreshing_all_files = False
with flags.refreshing, timer('refresh_files', 'info'):
api_app.shared_ctx.refresh['counted_files'] = 0
if not paths:
refresh_logger.warning('Refreshing all files')
refreshing_all_files = True
else:
refresh_msg = ", ".join(list(map(str, paths)))
refresh_logger.warning(f'Refreshing {refresh_msg[:1000]}')
if send_events:
Events.send_global_refresh_started()
# Add all files in the media directory to the DB.
paths = paths or [get_media_directory()]
directories = list(filter(lambda i: i.is_dir(), paths))
found_directories = set()
if directories:
with flags.refresh_counting:
while directories:
directory = directories.pop()
try:
files, dirs = get_files_and_directories(directory)
except PermissionError as e:
refresh_logger.error(f'Error refreshing {directory}', exc_info=e)
continue
directories.extend(dirs)
found_directories |= set(dirs)
add_files_to_refresh_count(len(files))
# Sleep to catch cancel.
await asyncio.sleep(0)
refresh_logger.info(f'Counted {api_app.shared_ctx.refresh["counted_files"]} files')
with flags.refresh_discovery:
await refresh_discover_paths(paths, idempotency)
if send_events:
Events.send_global_refresh_discovery_completed()
# Model all files that have not been indexed.
with flags.refresh_modeling:
await apply_modelers()
if send_events:
Events.send_global_refresh_modeling_completed()
# Index the rest of the files that were not indexed by modelers.
with flags.refresh_indexing:
await apply_indexers()
if send_events:
Events.send_global_refresh_indexing_completed()
# Cleanup any outdated file data.
with flags.refresh_cleanup:
await apply_refresh_cleanup()
with get_db_session() as session:
parent_directories = {i[0] for i in session.query(Directory.path).filter(Directory.path.in_(paths))}
parent_directories |= set(filter(lambda i: i.is_dir(), paths))
upsert_directories(parent_directories, found_directories)
if send_events:
Events.send_global_after_refresh_completed()
refresh_logger.warning('Done refreshing Files')
if refreshing_all_files:
# Only set refresh_complete flag if all files have been refreshed.
flags.refresh_complete.set()
if send_events:
Events.send_refresh_completed()
api_app.shared_ctx.refresh['counted_files'] = 0
api_app.shared_ctx.refresh['idempotency'] = None
async def apply_indexers():
"""Indexes any Files that have not yet been indexed by Modelers, or by previous calls of this function."""
from wrolpi.files.models import FileGroup
@ -1058,8 +969,9 @@ def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List
results = session.query(FileGroup, Video) \
.filter(FileGroup.id.in_(ordered_ids)) \
.outerjoin(Video, Video.file_group_id == FileGroup.id)
# Order FileGroups by their location in ordered_ids.
file_groups: List[Tuple[FileGroup, Video]] = sorted(results, key=lambda i: ordered_ids.index(i[0].id))
# Order FileGroups by their location in ordered_ids. Use dict for O(1) lookups instead of O(n) list.index().
id_to_order = {id_: idx for idx, id_ in enumerate(ordered_ids)}
file_groups: List[Tuple[FileGroup, Video]] = sorted(results, key=lambda i: id_to_order.get(i[0].id, 0))
results = list()
for extra, (file_group, video) in zip_longest(extras, file_groups):
video: Video
@ -1320,14 +1232,14 @@ def get_refresh_progress() -> RefreshProgress:
progress = RefreshProgress(
counted_files=api_app.shared_ctx.refresh.get('counted_files', 0),
counting=flags.refresh_counting.is_set(),
discovery=flags.refresh_discovery.is_set(),
counting=flags.file_worker_counting.is_set(),
discovery=flags.file_worker_discovery.is_set(),
indexed=int(results['indexed'] or 0),
indexing=flags.refresh_indexing.is_set(),
indexing=flags.file_worker_indexing.is_set(),
modeled=int(results['modeled'] or 0),
modeling=flags.refresh_modeling.is_set(),
cleanup=flags.refresh_cleanup.is_set(),
refreshing=flags.refreshing.is_set(),
modeling=flags.file_worker_modeling.is_set(),
cleanup=flags.file_worker_cleanup.is_set(),
refreshing=flags.file_worker_busy.is_set(),
total_file_groups=int(results['total_file_groups'] or 0),
unindexed=int(results['unindexed'] or 0),
)
@ -1453,260 +1365,35 @@ def _bulk_update_file_groups_db(session: Session, chunk_plan: Dict[pathlib.Path,
curs.execute(sql)
def add_files_to_refresh_count(count: int):
from wrolpi.api_utils import api_app
count = api_app.shared_ctx.refresh.get('counted_files', 0) + count
api_app.shared_ctx.refresh['counted_files'] = count
async def _move(session: Session, destination: pathlib.Path, *sources: pathlib.Path) \
-> OrderedDict[pathlib.Path, pathlib.Path]:
media_directory = get_media_directory()
for source in sources:
if not str(source).startswith(str(media_directory)):
raise FileNotFoundError(f'{source} is not within the media directory')
if not str(destination).startswith(str(media_directory)):
raise FileNotFoundError(f'{destination} is not within the media directory')
destination_existed = destination.is_dir()
destination.mkdir(parents=True, exist_ok=True)
# The files that will be moved. [ (old_file, new_file), ... ]
plan: Dict[pathlib.Path, pathlib.Path] = dict()
# Directories that will need to be cleaned up.
old_directories = unique_by_predicate(i if i.is_dir() else i.parent for i in sources)
def add_file_group_to_plan(file_group_: FileGroup):
primary_path_ = file_group_.primary_path
if primary_path_ not in plan:
new_primary_path_ = destination / primary_path_.name
if new_primary_path_.exists():
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}')
plan[primary_path_] = new_primary_path_
def add_source_file_group_to_plan(file_group_: FileGroup, source_: pathlib.Path):
primary_path_ = file_group_.primary_path
if primary_path_ not in plan:
new_primary_path_ = destination / source_.name / primary_path_.relative_to(source)
if new_primary_path_.exists():
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path_}')
plan[primary_path_] = new_primary_path_
# Sources may be a list of files, we only want to issue a query for one file in each FileGroup.
sources = get_unique_files_by_stem(sources)
logger.info(f'move got {len(sources)} to move')
with flags.refresh_counting:
for source in sources:
if source.is_file():
# Get any FileGroups that share the source's stem.
files = glob_shared_stem(source)
file_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(files)).all()
if not file_groups:
# No FileGroups for this source file, create one.
try:
fg = FileGroup.from_paths(session, *files)
file_groups = [fg, ]
except NoPrimaryFile:
for file in files:
fg = FileGroup.from_paths(session, file)
add_file_group_to_plan(fg)
add_files_to_refresh_count(len(file_groups))
for fg in file_groups:
add_file_group_to_plan(fg)
# Sleep to catch cancel.
await asyncio.sleep(0)
elif source.is_dir():
# Use indexed directory column for efficient lookup
# Match exact directory or subdirectories
stmt = '''SELECT primary_path
FROM file_group
WHERE directory = :dir
OR directory LIKE :dir_pattern'''
source_str = str(source)
for (primary_path,) in session.execute(
stmt, dict(dir=source_str, dir_pattern=f'{source_str}/%')):
primary_path = pathlib.Path(primary_path)
if primary_path not in plan:
new_primary_path = destination / source.name / primary_path.relative_to(source)
if new_primary_path.exists():
raise FileExistsError(f'Cannot move file because it already exists: {new_primary_path}')
plan[primary_path] = new_primary_path
add_files_to_refresh_count(1)
# Sleep to catch cancel.
await asyncio.sleep(0)
# Move directories of this source.
for directory in (i for i in walk(source) if i.is_dir()):
new_directory = destination / source.name / directory.relative_to(source)
plan[directory] = new_directory
# Find any files in this directory that are not yet in the DB and add them to the plan.
files = {i for i in walk(source) if i.is_file()}
if missing_files := files - set(plan.keys()):
for paths in group_files_by_stem(missing_files):
try:
# Only create the FileGroup if a primary_path can be found.
get_primary_file(paths)
file_group = FileGroup.from_paths(session, *paths)
add_files_to_refresh_count(1)
add_source_file_group_to_plan(file_group, source)
except NoPrimaryFile:
# No primary path for these paths, create FileGroups for each.
add_files_to_refresh_count(1)
for file in paths:
fg = FileGroup.from_paths(session, file)
add_source_file_group_to_plan(fg, source)
# Sleep to catch cancel.
await asyncio.sleep(0)
else:
raise UnknownFile(f'Unknown path type, cannot move: {source}')
# Sort plan by the deepest files first.
plan: OrderedDict = OrderedDict(sorted(plan.items(), key=lambda i: (len(i[0].parents), i[0].name), reverse=True))
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'_move: plan created with {len(plan)} items to {destination}')
# Revert plan is built out as files are moved.
revert_plan = OrderedDict()
new_directories = set()
def do_plan(plan_: OrderedDict[pathlib.Path, pathlib.Path], is_revert: bool = False):
"""Apply the move plan to all the FileGroups.
@warning: Cannot be cancelled!"""
nonlocal new_directories
# Query existing directories ONCE before processing chunks (not per-chunk).
existing_directories = {i[0] for i in session.query(Directory.path)}
inserted_directories = set()
# Move FileGroups in groups.
for chunk in chunks(plan_.items(), MOVE_CHUNK_SIZE):
old_paths = [i for i, j in chunk]
old_files, old_directories_ = partition(lambda i: i.is_file(), old_paths)
file_groups_ = session.query(FileGroup).filter(FileGroup.primary_path.in_(old_files)).all()
if len(file_groups_) != len(old_files):
# We ensured there are FileGroups while building the plan, maybe user deleted a file?
raise RuntimeError('Could not get all FileGroups for move')
# Build chunk plan for bulk update
chunk_plan = {}
for file_group_ in file_groups_:
old_file = file_group_.primary_path
new_primary_path_ = plan_[old_file]
parent = new_primary_path_.parent
parent.mkdir(parents=True, exist_ok=True)
if parent not in new_directories:
new_directories.add(parent)
# Move physical files
_move_file_group_files(file_group_, new_primary_path_)
chunk_plan[old_file] = new_primary_path_
revert_plan[new_primary_path_] = old_file
for directory_ in old_directories_:
delete_directory(directory_)
# Find directories that need to be inserted (not existing and not already inserted).
missing_directories = new_directories - existing_directories - inserted_directories
if missing_directories:
session.add_all([Directory(path=d, name=d.name) for d in missing_directories])
inserted_directories.update(missing_directories)
# Batch update all FileGroups in this chunk with raw SQL
_bulk_update_file_groups_db(session, chunk_plan)
# Flush Directory inserts
session.flush()
with flags.refresh_discovery:
try:
do_plan(plan)
for source in sources:
if source.is_dir():
delete_directory(source)
logger.info(f'Move execution completed')
except Exception as e:
logger.error(f'_move failed, reverting {len(revert_plan)} items', exc_info=e)
new_directories = set()
# Move files back. Get copy because do_plan will change the revert plan.
do_plan(revert_plan.copy(), is_revert=True)
# Delete the directories that were created.
directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True)
for path in directories:
if path.is_dir():
delete_directory(path)
# Remove destination only if it did not exist at the start.
if destination.is_dir() and destination_existed is False:
delete_directory(destination)
raise
# Clean up any old Directory records.
for old_directory in old_directories:
# Delete directories that do no exist, or are empty.
if not old_directory.is_dir() or next(old_directory.iterdir(), None) is None:
directories = session.query(Directory).filter(Directory.path.like(str(old_directory)))
for directory in directories:
session.delete(directory)
return plan
def delete_directory(directory: pathlib.Path, recursive: bool = False):
"""Remove a directory, remove it's Directory record.
Will refuse to delete a directory if it contains Tagged Files."""
if recursive:
with get_db_session() as session:
# Use indexed directory column for efficient lookup
directory_str = str(directory)
tagged = session.query(FileGroup) \
.filter(or_(
FileGroup.directory == directory_str,
FileGroup.directory.like(f'{directory_str}/%')
)) \
.join(TagFile, TagFile.file_group_id == FileGroup.id) \
.limit(1).one_or_none()
if tagged:
raise FileGroupIsTagged(f'Cannot delete {tagged} because it is tagged')
shutil.rmtree(directory)
else:
directory.rmdir()
Will refuse to delete a directory if it contains Tagged Files.
This function is idempotent - safe to call even if the directory was already deleted."""
if directory.exists():
if recursive:
with get_db_session() as session:
# Use indexed directory column for efficient lookup
directory_str = str(directory)
tagged = session.query(FileGroup) \
.filter(or_(
FileGroup.directory == directory_str,
FileGroup.directory.like(f'{directory_str}/%')
)) \
.join(TagFile, TagFile.file_group_id == FileGroup.id) \
.limit(1).one_or_none()
if tagged:
raise FileGroupIsTagged(f'Cannot delete {tagged} because it is tagged')
shutil.rmtree(directory)
else:
directory.rmdir()
# Always clean up DB record even if directory was already deleted
with get_db_curs(commit=True) as curs:
stmt = 'DELETE FROM directory WHERE path=%s'
curs.execute(stmt, (str(directory),))
async def move(session: Session, destination: pathlib.Path, *sources: pathlib.Path) \
-> OrderedDict[pathlib.Path, pathlib.Path]:
"""Moves a file or directory (recursively), preserving applied Tags.
If the move fails at any point, the files will be returned to their previous locations.
Returns an OrderedDict, the key is the file's old location, the value is where the file was moved.
"""
with flags.refreshing:
from wrolpi.api_utils import api_app
api_app.shared_ctx.refresh['counted_files'] = 0
# Move the files, if this fails it will revert itself and raise an error.
with timer('moving files', 'info'):
plan = await _move(session, destination, *sources)
session.commit()
with flags.refresh_indexing:
await apply_indexers()
with flags.refresh_modeling:
await apply_modelers()
with flags.refresh_cleanup:
await apply_refresh_cleanup()
# Save tags now that files have been moved.
save_tags_config.activate_switch()
return plan
async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
"""Rename a file (and it's associated files). Preserve any tags.
@ -1742,6 +1429,8 @@ async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
async def rename_directory(session: Session, directory: pathlib.Path, new_name: str, send_events: bool = False) \
-> pathlib.Path:
"""Rename a directory. This is done by moving all files into the new directory, and removing the old directory."""
from wrolpi.files.worker import file_worker
new_directory = directory.with_name(new_name)
if new_directory.exists():
raise FileConflict(f'Cannot rename {directory} to {new_directory} because it already exists.')
@ -1758,7 +1447,8 @@ async def rename_directory(session: Session, directory: pathlib.Path, new_name:
# Regular directory rename - move all paths into the new directory.
paths = list(directory.iterdir())
try:
await move(session, new_directory, *paths)
job_id = file_worker.queue_move(new_directory, paths)
await file_worker.wait_for_job(job_id)
if send_events:
Events.send_file_move_completed(f'Directory has been renamed: {directory}')
except Exception:
@ -1766,7 +1456,7 @@ async def rename_directory(session: Session, directory: pathlib.Path, new_name:
Events.send_file_move_failed(f'Directory rename has failed: {directory}')
raise
# Update nested Collections (files already moved by move() above)
# Update nested Collections (files already moved by file_worker above)
directory_str = str(directory)
nested_collections = session.query(Collection).filter(
Collection.directory.like(f'{directory_str}/%')
@ -1776,8 +1466,9 @@ async def rename_directory(session: Session, directory: pathlib.Path, new_name:
new_coll_dir = new_directory / relative
await coll.move_collection(new_coll_dir, session, send_events=send_events, with_files=False)
# Remove the old directory.
delete_directory(directory)
# Remove the old directory (recursive=True to clean up any empty subdirectories
# that remained after files were moved out).
delete_directory(directory, recursive=True)
return new_directory

View file

@ -165,12 +165,12 @@ async def test_delete_wrol_mode(async_client, wrol_mode_fixture):
@pytest.mark.asyncio
async def test_files_search_recent(test_session, test_directory, async_client, video_file_factory):
async def test_files_search_recent(test_session, test_directory, async_client, video_file_factory, refresh_files):
"""File search can return the most recently viewed files."""
video_file_factory(test_directory / 'foo.mp4'), video_file_factory(test_directory / 'bar.mp4')
baz = (test_directory / 'baz.txt')
baz.write_text('baz contents')
await lib.refresh_files()
await refresh_files()
# baz is most recently viewed, foo has not been viewed.
bar, baz, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
@ -230,7 +230,7 @@ async def test_files_search(test_session, async_client, make_files_structure, as
@pytest.mark.asyncio
async def test_files_search_any_tag(async_client, test_session, make_files_structure, tag_factory):
async def test_files_search_any_tag(async_client, test_session, make_files_structure, tag_factory, refresh_files):
one, two = await tag_factory(), await tag_factory()
files = [
'foo.txt',
@ -238,7 +238,7 @@ async def test_files_search_any_tag(async_client, test_session, make_files_struc
'bar.txt',
]
make_files_structure(files)
await lib.refresh_files()
await refresh_files()
bar, foobar, foo = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
assert bar.primary_path.name == 'bar.txt' \
and foo.primary_path.name == 'foo.txt' \
@ -388,10 +388,10 @@ async def test_file_group_tag(async_client):
@pytest.mark.asyncio
async def test_search_directories(async_client, test_session, test_directory, make_files_structure,
assert_directories):
assert_directories, refresh_files):
"""Directories can be searched by name."""
make_files_structure(['foo/one.txt', 'foo/two.txt', 'bar/one.txt'])
await lib.refresh_files()
await refresh_files()
assert_directories({'foo', 'bar'})
# Create directory that was not refreshed.
(test_directory / 'baz').mkdir()
@ -745,11 +745,12 @@ async def test_post_upload_new_directory(test_session, async_client, test_direct
@pytest.mark.asyncio
async def test_directory_crud(test_session, async_client, test_directory, assert_directories, assert_files):
async def test_directory_crud(test_session, async_client, test_directory, assert_directories, assert_files,
refresh_files):
"""A directory can be created in a subdirectory. Errors are returned if there are conflicts."""
foo = test_directory / 'foo'
foo.mkdir()
await lib.refresh_files()
await refresh_files()
assert_directories({'foo', })
# Create a subdirectory.
@ -796,7 +797,7 @@ async def test_directory_crud(test_session, async_client, test_directory, assert
@pytest.mark.asyncio
async def test_move(test_session, test_directory, make_files_structure, async_client):
async def test_move(test_session, test_directory, make_files_structure, async_client, await_background_tasks):
"""Files can be moved up and down the media directory. Destination directories should already exist."""
baz, bar = make_files_structure({
'foo/bar.txt': 'bar',
@ -815,6 +816,7 @@ async def test_move(test_session, test_directory, make_files_structure, async_cl
content = dict(paths=['foo', 'baz.txt'], destination='qux')
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
await await_background_tasks()
assert not bar.exists()
assert (test_directory / 'qux/foo/bar.txt').is_file()
@ -822,6 +824,7 @@ async def test_move(test_session, test_directory, make_files_structure, async_cl
content = dict(paths=['qux/foo/bar.txt', ], destination='qux')
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
await await_background_tasks()
assert not bar.exists()
assert (test_directory / 'qux/bar.txt').is_file()
assert (test_directory / 'qux/foo').is_dir()
@ -830,30 +833,33 @@ async def test_move(test_session, test_directory, make_files_structure, async_cl
content = dict(paths=['qux/bar.txt', ], destination='')
request, response = await async_client.post('/api/files/move', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
await await_background_tasks()
assert (test_directory / 'bar.txt').is_file()
# mv bar.txt quux/quuz/bar.txt
content = dict(paths=['bar.txt', ], destination='quux/quuz')
request, response = await async_client.post('/api/files/move', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
await await_background_tasks()
assert (test_directory / 'quux/quuz/bar.txt').is_file()
# mv quux/quuz/bar.txt quux/bar.txt
content = dict(paths=['quux/quuz/bar.txt', ], destination='quux')
request, response = await async_client.post('/api/files/move', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
await await_background_tasks()
assert (test_directory / 'quux/bar.txt').is_file()
assert (test_directory / 'qux/foo').is_dir()
assert (test_directory / 'quux/quuz').is_dir()
@pytest.mark.asyncio
async def test_rename_file(test_session, test_directory, make_files_structure, async_client):
async def test_rename_file(test_session, test_directory, make_files_structure, async_client, refresh_files):
"""A FileGroup can be renamed. The title and search index is updated."""
foo_file, = make_files_structure({
'foo/bar/baz.txt': 'asdf',
})
await lib.refresh_files()
await refresh_files()
foo_fg = test_session.query(FileGroup).one()
assert foo_fg.a_text == 'baz txt'
assert foo_fg.d_text == 'asdf'
@ -871,7 +877,7 @@ async def test_rename_file(test_session, test_directory, make_files_structure, a
@pytest.mark.asyncio
async def test_rename_file_with_associated_files_via_api(test_session, test_directory, make_files_structure,
async_client, video_bytes, srt_text):
async_client, video_bytes, srt_text, refresh_files):
"""Renaming a FileGroup via API should rename all associated files.
Regression test: When renaming "example.mp4" to "example 2.mp4" and back via API,
@ -881,7 +887,7 @@ async def test_rename_file_with_associated_files_via_api(test_session, test_dire
'example.mp4': video_bytes,
'example.srt': srt_text,
})
await lib.refresh_files()
await refresh_files()
# Verify initial state
fg = test_session.query(FileGroup).one()
@ -932,11 +938,11 @@ async def test_rename_file_with_associated_files_via_api(test_session, test_dire
@pytest.mark.asyncio
async def test_rename_directory(test_session, test_directory, make_files_structure, async_client):
async def test_rename_directory(test_session, test_directory, make_files_structure, async_client, refresh_files):
make_files_structure({
'foo/bar/baz.txt': 'asdf',
})
await lib.refresh_files()
await refresh_files()
bar, foo = test_session.query(Directory).order_by(Directory.name).all()
assert foo.name == 'foo' and bar.name == 'bar'
@ -952,7 +958,8 @@ async def test_rename_directory(test_session, test_directory, make_files_structu
@pytest.mark.asyncio
async def test_rename_collection_directory(test_session, test_directory, make_files_structure, async_client):
async def test_rename_collection_directory(test_session, test_directory, make_files_structure, async_client,
refresh_files):
"""Renaming a Collection's directory updates the Collection.directory."""
from wrolpi.collections.models import Collection
@ -965,7 +972,7 @@ async def test_rename_collection_directory(test_session, test_directory, make_fi
test_session.commit()
collection_id = collection.id
await lib.refresh_files()
await refresh_files()
# Rename the directory
content = dict(path='my_collection', new_name='renamed_collection')
@ -980,7 +987,7 @@ async def test_rename_collection_directory(test_session, test_directory, make_fi
@pytest.mark.asyncio
async def test_rename_directory_with_nested_collections(test_session, test_directory, async_client):
async def test_rename_directory_with_nested_collections(test_session, test_directory, async_client, refresh_files):
"""Renaming a parent directory updates nested Collection directories and download destinations."""
from wrolpi.collections.models import Collection
from wrolpi.downloader import Download
@ -1009,7 +1016,7 @@ async def test_rename_directory_with_nested_collections(test_session, test_direc
test_session.commit()
coll_a_id, coll_b_id, download_id = coll_a.id, coll_b.id, download.id
await lib.refresh_files()
await refresh_files()
# Rename the parent directory
content = dict(path='videos', new_name='my_videos')
@ -1035,9 +1042,9 @@ async def test_rename_directory_with_nested_collections(test_session, test_direc
@pytest.mark.asyncio
async def test_delete_directory_recursive(test_session, test_directory, make_files_structure, async_client,
assert_files):
assert_files, refresh_files):
make_files_structure(['dir/foo', 'dir/bar', 'empty'])
await lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).count() == 3
content = {'paths': ['dir/', ]}
@ -1050,10 +1057,10 @@ async def test_delete_directory_recursive(test_session, test_directory, make_fil
@pytest.mark.asyncio
async def test_get_file(test_session, async_client, test_directory, make_files_structure):
async def test_get_file(test_session, async_client, test_directory, make_files_structure, refresh_files):
"""Can get info about a single file."""
make_files_structure({'foo/bar.txt': 'foo contents'})
await lib.refresh_files()
await refresh_files()
content = dict(file='foo/bar.txt')
request, response = await async_client.post('/api/files/file', content=json.dumps(content))
@ -1068,7 +1075,7 @@ async def test_get_file(test_session, async_client, test_directory, make_files_s
@pytest.mark.asyncio
async def test_ignore_directory(test_session, async_client, test_directory, make_files_structure, test_wrolpi_config,
await_switches):
await_switches, refresh_files):
"""A maintainer can ignore/un-ignore directories. The files in the directory should not be refreshed."""
# Remove any default ignored directories.
get_wrolpi_config().ignored_directories = []
@ -1082,7 +1089,7 @@ async def test_ignore_directory(test_session, async_client, test_directory, make
assert response.status_code == HTTPStatus.OK
assert len(get_wrolpi_config().ignored_directories) == 1
await lib.refresh_files()
await refresh_files()
files = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
assert {i.primary_path.name for i in files} == {'foo.txt', 'bar.txt', 'wrolpi.yaml'}
@ -1092,7 +1099,7 @@ async def test_ignore_directory(test_session, async_client, test_directory, make
assert response.status_code == HTTPStatus.OK
assert len(get_wrolpi_config().ignored_directories) == 0
await lib.refresh_files()
await refresh_files()
files = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
# Ignore configs.

View file

@ -5,13 +5,13 @@ from http import HTTPStatus
import pytest
from wrolpi.files.ebooks import EBook, EPUB_MIMETYPE
from wrolpi.files.lib import refresh_files, move
from wrolpi.files.models import FileGroup
from wrolpi.files.worker import file_worker
from wrolpi.test.common import assert_dict_contains
@pytest.mark.asyncio
async def test_index(async_client, test_session, test_directory, example_epub, example_mobi):
async def test_index(async_client, test_session, test_directory, example_epub, example_mobi, refresh_files):
"""Ebooks can be indexed.
Covers can be discovered."""
@ -57,7 +57,8 @@ async def test_index(async_client, test_session, test_directory, example_epub, e
@pytest.mark.asyncio
async def test_discover_local_cover(test_session, test_directory, example_epub, image_bytes_factory, await_switches):
async def test_discover_local_cover(test_session, test_directory, example_epub, image_bytes_factory, await_switches,
refresh_files):
cover_path = example_epub.with_suffix('.jpg')
cover_path.write_bytes(image_bytes_factory())
await refresh_files()
@ -69,7 +70,7 @@ async def test_discover_local_cover(test_session, test_directory, example_epub,
@pytest.mark.asyncio
async def test_extract_cover(test_session, test_directory, example_epub, await_switches):
async def test_extract_cover(test_session, test_directory, example_epub, await_switches, refresh_files):
"""First image is extracted from the Ebook and used as the cover."""
await refresh_files()
@ -115,7 +116,7 @@ async def test_search_ebooks(test_session, async_client, example_epub):
@pytest.mark.asyncio
async def test_discover_calibre_cover(test_session, async_client, test_directory, example_epub, example_mobi,
image_file):
image_file, refresh_files):
"""Calibre puts a cover near an ebook file, test if it can be found."""
# Create a Calibre metadata file.
metadata = test_directory / 'metadata.opf'
@ -139,7 +140,8 @@ async def test_discover_calibre_cover(test_session, async_client, test_directory
@pytest.mark.asyncio
async def test_move_ebook(async_client, test_session, test_directory, example_epub, image_file, tag_factory):
async def test_move_ebook(async_client, test_session, test_directory, example_epub, image_file, tag_factory,
refresh_files, await_background_tasks):
"""An ebook is re-indexed when moved."""
tag = await tag_factory()
shutil.move(image_file, example_epub.with_suffix('.jpg'))
@ -158,7 +160,9 @@ async def test_move_ebook(async_client, test_session, test_directory, example_ep
new_directory = test_directory / 'new'
new_directory.mkdir()
await move(test_session, new_directory, ebook.file_group.primary_path)
file_worker.queue_move(new_directory, [ebook.file_group.primary_path])
await await_background_tasks()
test_session.expire_all()
ebook: EBook = test_session.query(EBook).one()
assert ebook.file_group.primary_path == test_directory / 'new/ebook example.epub'

View file

@ -107,11 +107,12 @@ async def test_delete_file_link(async_client, test_session, test_directory):
@pytest.mark.asyncio
async def test_delete_tagged(await_switches, test_session, make_files_structure, tag_factory, video_bytes):
async def test_delete_tagged(await_switches, test_session, make_files_structure, tag_factory, video_bytes,
refresh_files):
"""Cannot delete a file that has been tagged."""
tag = await tag_factory()
make_files_structure({'foo/bar.txt': 'asdf', 'foo/bar.mp4': video_bytes})
await lib.refresh_files()
await refresh_files()
# Both files end up in a group.
bar = test_session.query(FileGroup).one()
bar.add_tag(test_session, tag.id)
@ -170,7 +171,7 @@ def test_split_path_stem_and_suffix(path, full, expected):
@pytest.mark.asyncio
async def test_refresh_files(async_client, test_session, make_files_structure, assert_file_groups):
async def test_refresh_files(async_client, test_session, make_files_structure, assert_file_groups, refresh_files):
"""All files in the media directory should be found when calling `refresh_files`"""
files = make_files_structure([
'foo.txt',
@ -179,7 +180,7 @@ async def test_refresh_files(async_client, test_session, make_files_structure, a
])
foo, bar, baz = files
await lib.refresh_files()
await refresh_files()
assert_file_groups([
{'primary_path': 'foo.txt', 'indexed': True},
{'primary_path': 'bar.txt', 'indexed': True},
@ -187,23 +188,23 @@ async def test_refresh_files(async_client, test_session, make_files_structure, a
baz.unlink()
await lib.refresh_files()
await refresh_files()
assert_file_groups([{'primary_path': 'foo.txt', 'indexed': True}, {'primary_path': 'bar.txt', 'indexed': True}])
foo.unlink()
await lib.refresh_files()
await refresh_files()
assert_file_groups([{'primary_path': 'bar.txt', 'indexed': True}])
@pytest.mark.asyncio
async def test_file_group_location(async_client, test_session, make_files_structure):
async def test_file_group_location(async_client, test_session, make_files_structure, refresh_files):
"""FileGroup can return the URL necessary to preview the FileGroup."""
make_files_structure({
'foo/one.txt': 'one',
'two.txt': 'two',
})
await lib.refresh_files()
await refresh_files()
one, two = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
assert one.location == '/files?folders=foo&preview=foo%2Fone.txt'
@ -212,10 +213,10 @@ async def test_file_group_location(async_client, test_session, make_files_struct
@pytest.mark.asyncio
async def test_refresh_bogus_files(async_client, test_session, make_files_structure, test_directory,
assert_file_groups, insert_file_group):
assert_file_groups, insert_file_group, refresh_files):
"""Bogus files are removed during a refresh."""
make_files_structure(['does exist.txt'])
await lib.refresh_files()
await refresh_files()
insert_file_group([test_directory / 'does not exist.txt'])
test_session.commit()
@ -225,17 +226,17 @@ async def test_refresh_bogus_files(async_client, test_session, make_files_struct
{'primary_path': 'does not exist.txt', 'indexed': True},
])
await lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).count() == 1, 'Bogus file was not removed'
assert_file_groups([{'primary_path': 'does exist.txt', 'indexed': True}])
@pytest.mark.asyncio
async def test_refresh_empty_media_directory(async_client, test_session, test_directory):
async def test_refresh_empty_media_directory(async_client, test_session, test_directory, refresh_files):
"""refresh_paths will refuse to refresh with an empty media directory."""
with pytest.raises(UnknownDirectory):
await lib.refresh_files()
await refresh_files()
@pytest.mark.asyncio
@ -392,54 +393,31 @@ async def test_file_group_tag(test_session, make_files_structure, test_directory
@pytest.mark.asyncio
async def test_refresh_a_text_no_indexer(async_client, test_session, make_files_structure):
async def test_refresh_a_text_no_indexer(async_client, test_session, make_files_structure, refresh_files):
"""File.a_text is filled even if the file does not match an Indexer."""
make_files_structure(['foo', 'bar-bar'])
await lib.refresh_files()
await refresh_files()
files = {i.a_text for i in test_session.query(FileGroup)}
assert files == {'bar bar bar-bar', 'foo'}
@pytest.mark.asyncio
async def test_refresh_many_files(async_client, test_session, make_files_structure):
async def test_refresh_many_files(async_client, test_session, make_files_structure, refresh_files):
"""Used to profile file refreshing"""
file_count = 10_000
make_files_structure([f'{uuid4()}.txt' for _ in range(file_count)])
with timer('first refresh'):
await lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).count() == file_count
with timer('second refresh'):
await lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).count() == file_count
@pytest.mark.asyncio
async def test_refresh_cancel(async_client, test_session, make_files_structure, test_directory):
"""Refresh tasks can be canceled."""
# Creat a lot of files so the refresh will take too long.
make_files_structure([f'{uuid4()}.txt' for _ in range(1_000)])
async def assert_cancel(task_):
# Time the time it takes to cancel.
before = datetime.now()
# Sleep so the refresh task has time to run.
await asyncio.sleep(0)
# Cancel the refresh (it will be sleeping soon).
task_.cancel()
with pytest.raises(asyncio.CancelledError):
await task_
assert (datetime.now() - before).total_seconds() < 1.2, 'Task took too long. Was the refresh canceled?'
task = asyncio.create_task(lib.refresh_files())
await assert_cancel(task)
@pytest.mark.asyncio
async def test_mime_type(async_client, test_session, make_files_structure, test_directory, assert_files):
async def test_mime_type(async_client, test_session, make_files_structure, test_directory, assert_files, refresh_files):
"""Files module uses the `file` command to get the mimetype of each file."""
from PIL import Image
@ -453,7 +431,7 @@ async def test_mime_type(async_client, test_session, make_files_structure, test_
Image.new('RGB', (25, 25), color='grey').save(bar)
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', baz)
await lib.refresh_files()
await refresh_files()
assert_files([
{'path': 'dir/foo text.txt'},
{'path': 'dir/bar.jpeg'},
@ -473,7 +451,7 @@ async def test_mime_type(async_client, test_session, make_files_structure, test_
@pytest.mark.asyncio
async def test_files_indexer(async_client, test_session, make_files_structure, test_directory):
async def test_files_indexer(async_client, test_session, make_files_structure, test_directory, refresh_files):
"""An Indexer is provided for each file based on it's mimetype or contents."""
source_files: List[str] = [
'a text file.txt',
@ -495,7 +473,7 @@ async def test_files_indexer(async_client, test_session, make_files_structure, t
# Enable slow feature for testing.
# TODO can this be sped up to always be included?
with mock.patch('modules.videos.lib.EXTRACT_SUBTITLES', True):
await lib.refresh_files()
await refresh_files()
text_file, zip_file, image_file, unknown_file, video_file \
= test_session.query(FileGroup).order_by(FileGroup.primary_path)
@ -525,11 +503,11 @@ async def test_files_indexer(async_client, test_session, make_files_structure, t
with mock.patch('modules.videos.models.Video.validate') as mock_validate:
mock_validate.side_effect = Exception('This should not be called twice')
await lib.refresh_files()
await refresh_files()
# Change the contents, the file should be re-indexed.
text_path.write_text('new text contents')
await lib.refresh_files()
await refresh_files()
files, total = lib.search_files('new', 10, 0)
assert total == 1
@ -549,14 +527,14 @@ def test_split_file_name_words(name, expected):
@pytest.mark.asyncio
async def test_large_text_indexer(async_client, test_session, make_files_structure):
async def test_large_text_indexer(async_client, test_session, make_files_structure, refresh_files):
"""
Large files have their indexes truncated.
"""
large, = make_files_structure({
'large_file.txt': 'foo ' * 1_000_000,
})
await lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).count() == 1
assert large.is_file() and large.stat().st_size == 4_000_000
@ -696,7 +674,7 @@ async def test_get_refresh_progress(async_client, test_session):
@pytest.mark.asyncio
async def test_refresh_files_no_groups(async_client, test_session, test_directory, make_files_structure,
zip_file_factory):
zip_file_factory, refresh_files):
"""Files that share a name, but cannot be grouped into a FileGroup have their own FileGroups."""
foo_txt, foo_zip = make_files_structure({
'foo.txt': 'text',
@ -704,7 +682,7 @@ async def test_refresh_files_no_groups(async_client, test_session, test_director
})
assert foo_txt.stat().st_size and foo_zip.stat().st_size
await lib.refresh_files()
await refresh_files()
# Two distinct FileGroups.
assert test_session.query(FileGroup).count() == 2
@ -714,7 +692,7 @@ async def test_refresh_files_no_groups(async_client, test_session, test_director
@pytest.mark.asyncio
async def test_refresh_directories(test_session, test_directory, assert_directories, await_switches):
async def test_refresh_directories(test_session, test_directory, assert_directories, await_switches, refresh_files):
"""
Directories are stored when they are discovered. They are removed when they can no longer be found.
"""
@ -725,21 +703,21 @@ async def test_refresh_directories(test_session, test_directory, assert_director
bar.mkdir()
baz.mkdir()
await lib.refresh_files()
await refresh_files()
assert_directories({'foo', 'bar', 'baz'})
# Deleted directory is removed.
foo.rmdir()
await lib.refresh_files()
await refresh_files()
assert_directories({'bar', 'baz'})
bar.rmdir()
await lib.refresh_files([bar])
await refresh_files([bar])
assert_directories({'baz', })
# A new directory can be refreshed directly.
foo.mkdir()
await lib.refresh_files([foo])
await refresh_files([foo])
assert_directories({'foo', 'baz'})
@ -784,115 +762,14 @@ async def test_file_group_merge(async_client, test_session, test_directory, make
assert vid.mimetype == 'video/mp4'
@pytest.mark.asyncio
async def test_move(async_client, test_session, test_directory, make_files_structure, video_bytes,
singlefile_contents_factory):
"""files.lib.move behaves likes posix mv"""
make_files_structure({
'foo/bar/video.mp4': video_bytes,
'foo/bar/baz/archive.html': (singlefile_text := singlefile_contents_factory()),
'foo/bytes.txt': b'text',
'foo/text.txt': 'text',
})
foo = test_directory / 'foo'
qux = test_directory / 'qux'
await lib.refresh_files()
# mv foo qux
plan = await lib.move(test_session, qux, foo)
plan = [(str(i.relative_to(test_directory)), str(j.relative_to(test_directory))) for i, j in plan.items()]
# The deepest files are moved first.
assert plan == [('foo/bar/baz/archive.html', 'qux/foo/bar/baz/archive.html'),
('foo/bar/video.mp4', 'qux/foo/bar/video.mp4'),
('foo/bar/baz', 'qux/foo/bar/baz'),
('foo/text.txt', 'qux/foo/text.txt'),
('foo/bytes.txt', 'qux/foo/bytes.txt'),
('foo/bar', 'qux/foo/bar'),
]
# Files were moved.
assert (qux / 'foo/bar/video.mp4').is_file() and (qux / 'foo/bar/video.mp4').read_bytes() == video_bytes
assert (qux / 'foo/bar/baz/archive.html').is_file() \
and (qux / 'foo/bar/baz/archive.html').read_text() == singlefile_text
assert (qux / 'foo/bytes.txt').is_file() and (qux / 'foo/bytes.txt').read_bytes() == b'text'
assert (qux / 'foo/text.txt').is_file() and (qux / 'foo/text.txt').read_text() == 'text'
# Directories were moved.
assert not foo.exists()
@pytest.mark.asyncio
async def test_move_files(async_client, test_session, test_directory, make_files_structure):
"""Files can be moved using files.lib.move."""
one, two = make_files_structure({
'foo/one.txt': 'one',
'two.txt': 'two',
})
dest = test_directory / 'dest'
plan = await lib.move(test_session, dest, one, two)
assert list(plan.items()) == [
(test_directory / 'foo/one.txt', test_directory / 'dest/one.txt'),
(test_directory / 'two.txt', test_directory / 'dest/two.txt'),
]
# one.txt is moved out of foo.
assert not (test_directory / 'foo/one.txt').exists(), 'one.txt is lingering in source'
assert not (test_directory / 'two.txt').exists(), 'two.txt is lingering in source'
assert (dest / 'one.txt').read_text() == 'one', 'one.txt was not moved to destination'
assert (dest / 'two.txt').read_text() == 'two', 'two.txt was not moved to destination'
@pytest.mark.asyncio
async def test_move_deep_directory(async_client, test_session, test_directory, make_files_structure):
"""Moving files/directories from one deep directory to another is supported."""
baz, foo, quuz_mp4, quuz_txt, quux = make_files_structure({
'foo/foo.txt': 'foo',
'foo/bar/baz.txt': 'baz',
'foo/qux/quux.txt': 'quux',
'foo/quuz.txt': 'quuz text', # this text file will be moved when it's MP4 is moved.
'foo/quuz.mp4': 'quuz mp4',
})
assert foo.name == 'foo.txt' \
and baz.name == 'baz.txt' \
and quux.name == 'quux.txt' \
and quuz_mp4.name == 'quuz.mp4' \
and quuz_txt.name == 'quuz.txt', \
'Test directory was not initiated correctly.'
bar, qux = baz.parent, quux.parent
dest = test_directory / 'deep/dest'
# mv foo/bar foo/qux foo/quuz.mp4 foo/quuz.txt deep/dest
plan = await lib.move(test_session, dest, bar, qux, quuz_mp4, quuz_txt)
assert list(plan.items()) == [
(test_directory / 'foo/qux/quux.txt', test_directory / 'deep/dest/qux/quux.txt'),
(test_directory / 'foo/bar/baz.txt', test_directory / 'deep/dest/bar/baz.txt'),
(test_directory / 'foo/quuz.txt', test_directory / 'deep/dest/quuz.txt'),
(test_directory / 'foo/quuz.mp4', test_directory / 'deep/dest/quuz.mp4'),
]
# Unrelated files and directories are untouched.
assert not (test_directory / 'deep/dest/foo').is_dir(), 'foo/ should not have been moved.'
assert (test_directory / 'foo').is_dir(), 'foo/ should not be deleted.'
assert (test_directory / 'foo/foo.txt').read_text() == 'foo', 'foo.txt should not be moved.'
# Files are moved, preserving the name of their directory.
assert (test_directory / 'deep/dest/bar').is_dir(), 'foo/bar/ was not moved'
assert (test_directory / 'deep/dest/bar/baz.txt').is_file(), 'foo/bar/baz.txt was not moved'
assert (test_directory / 'deep/dest/bar/baz.txt').read_text() == 'baz', 'foo/bar/baz.txt has wrong contents.'
assert (test_directory / 'deep/dest/qux/quux.txt').read_text() == 'quux', 'quux.txt has the wrong contents'
assert (test_directory / 'deep/dest/quuz.txt').read_text() == 'quuz text', 'quuz.txt has the wrong contents'
assert (test_directory / 'deep/dest/quuz.mp4').read_text() == 'quuz mp4', 'quuz.mp4 has the wrong contents'
# Old directory was removed.
assert not (test_directory / 'foo/bar').is_dir(), 'bar directory was not moved'
@pytest.mark.asyncio
async def test_move_directory(async_client, test_session, test_directory, make_files_structure,
assert_directories):
assert_directories, refresh_files):
"""A Directory record is deleted when it's directory is deleted."""
make_files_structure({
'foo/one.txt': 'one',
})
await lib.refresh_files()
await refresh_files()
assert_directories({'foo'})
bar = test_directory / 'bar'
@ -901,52 +778,15 @@ async def test_move_directory(async_client, test_session, test_directory, make_f
assert_directories({'bar'})
count = 0
@pytest.mark.asyncio
async def test_move_error(async_client, test_session, test_directory, make_files_structure, video_bytes,
singlefile_contents_factory):
"""Files are restored when a move fails."""
make_files_structure({
'foo/bar/video.mp4': video_bytes,
'foo/bar/baz/archive.html': (singlefile_text := singlefile_contents_factory()),
'foo/bytes.txt': b'text',
'foo/text.txt': 'text',
})
foo = test_directory / 'foo'
qux = test_directory / 'qux'
def mock_shutil_move(*args, **kwargs):
# Mock the `shutil.move` with this function which will unexpectedly fail on the 3rd call.
global count
count += 1
if count == 3:
raise FileNotFoundError('fake file move failure')
return shutil.move(*args, **kwargs)
with mock.patch('wrolpi.files.lib.shutil.move', mock_shutil_move), pytest.raises(FileNotFoundError):
await lib.move(test_session, qux, foo)
# The move failed, the files should be moved back.
# foo was not deleted.
assert foo.is_dir()
assert (foo / 'bar/baz/archive.html').is_file() and (foo / 'bar/baz/archive.html').read_text() == singlefile_text
assert (foo / 'bar/video.mp4').is_file() and (foo / 'bar/video.mp4').read_bytes() == video_bytes
assert (foo / 'bytes.txt').is_file() and (foo / 'bytes.txt').read_bytes() == b'text'
assert (foo / 'text.txt').is_file() and (foo / 'text.txt').read_text() == 'text'
# Destination did not exist on move, so it was deleted.
assert not qux.exists()
@pytest.mark.asyncio
async def test_file_group_move(async_client, test_session, make_files_structure, test_directory, video_bytes,
srt_text):
srt_text, refresh_files):
"""Test FileGroup's move method"""
video, srt = make_files_structure({
'video.mp4': video_bytes,
'video.srt': srt_text,
})
await lib.refresh_files()
await refresh_files()
file_group: FileGroup = test_session.query(FileGroup).one()
assert file_group.indexed is True
@ -962,14 +802,17 @@ async def test_file_group_move(async_client, test_session, make_files_structure,
@pytest.mark.asyncio
async def test_move_tagged(async_client, test_session, test_directory, make_files_structure, tag_factory):
async def test_move_tagged(async_client, test_session, test_directory, make_files_structure, tag_factory, refresh_files,
await_background_tasks):
"""A FileGroup's tag is preserved when moved or renamed."""
from wrolpi.files.worker import file_worker
tag = await tag_factory()
foo, bar, = make_files_structure({
'foo/foo.txt': 'foo',
'foo/bar.txt': 'bar',
})
await lib.refresh_files()
await refresh_files()
bar_file_group, foo_file_group = test_session.query(FileGroup).order_by(FileGroup.primary_path)
bar_file_group.title = 'custom title' # Should not be overwritten.
bar_file_group.add_tag(test_session, tag.id)
@ -979,7 +822,10 @@ async def test_move_tagged(async_client, test_session, test_directory, make_file
qux.mkdir()
# Move both files into qux. The Tag should also be moved.
await lib.move(test_session, qux, bar, foo)
file_worker.queue_move(qux, [bar, foo])
await await_background_tasks()
test_session.expire_all()
new_foo = qux / 'foo.txt'
new_bar = qux / 'bar.txt'
# Files were moved.
@ -1009,7 +855,7 @@ async def test_move_tagged(async_client, test_session, test_directory, make_file
@pytest.mark.asyncio
async def test_html_index(async_client, test_session, test_directory, make_files_structure):
async def test_html_index(async_client, test_session, test_directory, make_files_structure, refresh_files):
make_files_structure({
'archive.html': '''<html>
<title>The Title</title>
@ -1041,7 +887,7 @@ async def test_html_index(async_client, test_session, test_directory, make_files
'''
})
await lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).count() == 1, 'Too many files were refreshed.'
archive_file: FileGroup = test_session.query(FileGroup).one()
@ -1055,9 +901,9 @@ Outside element.'''
@pytest.mark.asyncio
async def test_doc_indexer(async_client, test_session, example_doc):
async def test_doc_indexer(async_client, test_session, example_doc, refresh_files):
"""The contents of a Microsoft doc files can be indexed."""
await lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).count() == 1
doc = test_session.query(FileGroup).one()
assert doc.title == 'example word.doc'
@ -1069,9 +915,9 @@ async def test_doc_indexer(async_client, test_session, example_doc):
@pytest.mark.asyncio
async def test_docx_indexer(async_client, test_session, example_docx):
async def test_docx_indexer(async_client, test_session, example_docx, refresh_files):
"""The contents of a Microsoft docx files can be indexed."""
await lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).count() == 1
doc = test_session.query(FileGroup).one()
assert doc.title == 'example word.docx'
@ -1080,9 +926,9 @@ async def test_docx_indexer(async_client, test_session, example_docx):
@pytest.mark.asyncio
async def test_file_search_date_range(async_client, test_session, example_pdf, example_doc):
async def test_file_search_date_range(async_client, test_session, example_pdf, example_doc, refresh_files):
"""Test searching FileGroups by their published datetime."""
await lib.refresh_files()
await refresh_files()
doc, pdf = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
files, total = lib.search_files('', 10, 0)
@ -1153,40 +999,6 @@ async def test_upsert_file_video_with_channel(async_client, test_session, test_d
assert video.info_json_path, 'Info json file not found near video file.'
@pytest.mark.asyncio
async def test_move_many_files(async_client, test_session, test_directory, make_files_structure):
from wrolpi.db import get_db_curs
make_files_structure([f'foo/{i}.txt' for i in range(10_000)])
await lib.refresh_files()
foo = test_directory / 'foo'
bar = test_directory / 'bar'
# Track get_db_curs calls to verify commit=True is passed for bulk updates.
# Without commit=True, raw SQL is rolled back in production.
get_db_curs_calls = []
original_get_db_curs = get_db_curs
def tracking_get_db_curs(*args, **kwargs):
get_db_curs_calls.append(kwargs)
return original_get_db_curs(*args, **kwargs)
with timer('test_move_many_files'), mock.patch('wrolpi.files.lib.get_db_curs', tracking_get_db_curs):
# mv foo bar
await lib.move(test_session, bar, foo)
assert not foo.is_dir()
assert bar.is_dir()
assert (bar / 'foo').is_dir()
assert (bar / 'foo/0.txt').is_file()
# Verify bulk update used commit=True (required for raw SQL to persist in production)
assert get_db_curs_calls, 'get_db_curs should have been called for bulk update'
assert all(call.get('commit') is True for call in get_db_curs_calls), \
'get_db_curs must be called with commit=True to persist raw SQL in production'
@only_macos
def test_get_real_path_name(test_directory):
(test_directory / 'foo.txt').touch()
@ -1443,7 +1255,7 @@ def test_sanitize_filename_surrogates_renames_file(test_directory):
@pytest.mark.asyncio
async def test_rename_file_with_associated_files_twice(async_client, test_session, test_directory, make_files_structure,
video_bytes, srt_text):
video_bytes, srt_text, refresh_files):
"""
Renaming a FileGroup twice should rename all associated files both times.
@ -1455,7 +1267,7 @@ async def test_rename_file_with_associated_files_twice(async_client, test_sessio
'example.mp4': video_bytes,
'example.srt': srt_text,
})
await lib.refresh_files()
await refresh_files()
# Verify initial state
file_group = test_session.query(FileGroup).one()
@ -1509,7 +1321,7 @@ async def test_rename_file_with_associated_files_twice(async_client, test_sessio
@pytest.mark.asyncio
async def test_rename_non_primary_file_renames_filegroup(async_client, test_session, test_directory,
make_files_structure, video_bytes, srt_text):
make_files_structure, video_bytes, srt_text, refresh_files):
"""
Renaming a non-primary file (like a subtitle) should rename the entire FileGroup.
@ -1521,7 +1333,7 @@ async def test_rename_non_primary_file_renames_filegroup(async_client, test_sess
'example.mp4': video_bytes,
'example.srt': srt_text,
})
await lib.refresh_files()
await refresh_files()
# Verify initial state
file_group = test_session.query(FileGroup).one()

View file

@ -1,7 +1,6 @@
import pytest
from wrolpi.errors import UnknownFile
from wrolpi.files import lib
from wrolpi.files.models import FileGroup
@ -46,10 +45,10 @@ async def test_file_group_move(async_client, test_session, test_directory, video
@pytest.mark.asyncio
async def test_add_tag_overload(async_client, test_session, make_files_structure, tag_factory):
async def test_add_tag_overload(async_client, test_session, make_files_structure, tag_factory, refresh_files):
"""FileGroup.add_tag and FileGroup.untag use `singeldispatch`."""
make_files_structure({'foo.txt': 'foo contents'})
await lib.refresh_files()
await refresh_files()
foo: FileGroup = test_session.query(FileGroup).one()
one, two = await tag_factory(), await tag_factory()

View file

@ -10,13 +10,13 @@ from wrolpi.files.models import FileGroup
@pytest.mark.asyncio
async def test_pdf_title(async_client, test_session, example_pdf):
async def test_pdf_title(async_client, test_session, example_pdf, refresh_files):
"""PDF modeler extracts a PDFs title and contents."""
await files_lib.refresh_files()
await refresh_files()
with mock.patch('wrolpi.files.pdfs.get_pdf_metadata') as mock_get_pdf_metadata:
mock_get_pdf_metadata.side_effect = Exception('Should not index twice')
await files_lib.refresh_files()
await refresh_files()
pdf: FileGroup = test_session.query(FileGroup).one()
assert pdf.indexed is True

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,532 @@
"""Tests for the FileWorker move task."""
import pytest
from wrolpi.files.lib import _move_file_group_files
from wrolpi.files.models import FileGroup, Directory
from wrolpi.files.worker import file_worker, FileTask, FileTaskType
@pytest.mark.asyncio
async def test_file_worker_queue_move(async_client, test_session, test_directory, make_files_structure):
"""FileWorker accepts and queues move tasks."""
file1, = make_files_structure(['source/file1.txt'])
dest = test_directory / 'destination'
dest.mkdir()
# Queue a move task
file_worker.queue_move(dest, [file1.parent])
# Verify task is in the public queue
assert file_worker.public_queue.qsize() == 1
@pytest.mark.asyncio
async def test_file_worker_move_single_file(async_client, test_session, test_directory, make_files_structure):
"""FileWorker moves a single file to destination."""
file1, = make_files_structure(['source/file1.txt'])
fg = FileGroup.from_paths(test_session, file1)
test_session.commit()
original_id = fg.id
dest = test_directory / 'destination'
dest.mkdir()
task = FileTask(FileTaskType.move, [file1], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Verify file was moved
assert (dest / 'file1.txt').exists()
assert not file1.exists()
# Verify FileGroup was updated
test_session.expire_all()
fg = test_session.query(FileGroup).filter(FileGroup.id == original_id).one()
assert str(dest) in str(fg.directory)
assert 'file1.txt' in str(fg.primary_path)
@pytest.mark.asyncio
async def test_file_worker_move_directory(async_client, test_session, test_directory, make_files_structure):
"""FileWorker moves an entire directory to destination."""
files = make_files_structure([
'source/file1.txt',
'source/file2.txt',
'source/subdir/file3.txt',
])
# Create FileGroups
for f in files:
FileGroup.from_paths(test_session, f)
test_session.commit()
source_dir = test_directory / 'source'
dest = test_directory / 'destination'
dest.mkdir()
task = FileTask(FileTaskType.move, [source_dir], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Verify files were moved
assert (dest / 'source' / 'file1.txt').exists()
assert (dest / 'source' / 'file2.txt').exists()
assert (dest / 'source' / 'subdir' / 'file3.txt').exists()
# Verify original files are gone
assert not (source_dir / 'file1.txt').exists()
# Verify FileGroups were updated
test_session.expire_all()
file_groups = test_session.query(FileGroup).all()
assert len(file_groups) == 3
for fg in file_groups:
assert str(dest) in str(fg.directory)
@pytest.mark.asyncio
async def test_file_worker_move_status_updates(async_client, test_session, test_directory, make_files_structure):
"""FileWorker updates status during move operation."""
files = make_files_structure([f'source/file{i}.txt' for i in range(5)])
for f in files:
FileGroup.from_paths(test_session, f)
test_session.commit()
dest = test_directory / 'destination'
dest.mkdir()
task = FileTask(FileTaskType.move, [test_directory / 'source'], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# After completion, status should be reset to idle
status = file_worker.status
assert status['status'] == 'idle'
assert status['destination'] is None
@pytest.mark.asyncio
async def test_file_worker_move_preserves_tags(
async_client, test_session, test_directory, make_files_structure, tag_factory, await_switches
):
"""FileWorker preserves tags when moving files."""
file1, = make_files_structure(['source/file1.txt'])
fg = FileGroup.from_paths(test_session, file1)
test_session.commit()
tag = await tag_factory()
fg.add_tag(test_session, tag.id)
test_session.commit()
await await_switches()
original_id = fg.id
dest = test_directory / 'destination'
dest.mkdir()
task = FileTask(FileTaskType.move, [file1], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Verify tag is preserved
test_session.expire_all()
fg = test_session.query(FileGroup).filter(FileGroup.id == original_id).one()
assert len(fg.tag_files) == 1
assert fg.tag_files[0].tag_id == tag.id
@pytest.mark.asyncio
async def test_file_worker_move_with_shared_stem_files(
async_client, test_session, test_directory, make_files_structure, video_file_factory
):
"""FileWorker moves all files sharing a stem together."""
docs_dir = test_directory / 'source'
docs_dir.mkdir(parents=True, exist_ok=True)
video_path = video_file_factory(docs_dir / 'video1.mp4')
info_path = docs_dir / 'video1.info.json'
info_path.write_text('{"title": "test"}')
fg = FileGroup.from_paths(test_session, video_path)
fg.files.append({'path': 'video1.info.json', 'mimetype': 'application/json'})
test_session.commit()
dest = test_directory / 'destination'
dest.mkdir()
task = FileTask(FileTaskType.move, [video_path], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Verify both files were moved
assert (dest / 'video1.mp4').exists()
assert (dest / 'video1.info.json').exists()
assert not video_path.exists()
assert not info_path.exists()
@pytest.mark.asyncio
async def test_file_worker_move_error_file_exists(
async_client, test_session, test_directory, make_files_structure
):
"""FileWorker handles error when destination file already exists."""
source_file, dest_file = make_files_structure([
'source/file1.txt',
'destination/file1.txt',
])
fg = FileGroup.from_paths(test_session, source_file)
test_session.commit()
dest = test_directory / 'destination'
task = FileTask(FileTaskType.move, [source_file], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Status should be reset to idle (error handled gracefully)
status = file_worker.status
assert status['status'] == 'idle'
# Source file should still exist (move failed)
assert source_file.exists()
@pytest.mark.asyncio
async def test_file_worker_move_invalid_source(async_client, test_session, test_directory):
"""FileWorker handles invalid source paths gracefully."""
nonexistent = test_directory / 'nonexistent'
dest = test_directory / 'destination'
dest.mkdir()
task = FileTask(FileTaskType.move, [nonexistent], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Status should be reset to idle
status = file_worker.status
assert status['status'] == 'idle'
@pytest.mark.asyncio
async def test_file_worker_move_bulk_files(async_client, test_session, test_directory, make_files_structure):
"""FileWorker handles bulk moves efficiently with chunking."""
# Create 150+ files to test chunking (MOVE_CHUNK_SIZE = 100)
file_count = 150
files = make_files_structure([f'source/file{i}.txt' for i in range(file_count)])
for f in files:
FileGroup.from_paths(test_session, f)
test_session.commit()
dest = test_directory / 'destination'
dest.mkdir()
task = FileTask(FileTaskType.move, [test_directory / 'source'], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Verify all files moved
test_session.expire_all()
moved_count = test_session.query(FileGroup).filter(
FileGroup.directory.like(f'{dest}%')
).count()
assert moved_count == file_count
# Verify physical files exist
assert (dest / 'source' / 'file0.txt').exists()
assert (dest / 'source' / f'file{file_count - 1}.txt').exists()
@pytest.mark.asyncio
async def test_file_worker_move_creates_destination_directories(
async_client, test_session, test_directory, make_files_structure
):
"""FileWorker creates destination directories as needed."""
file1, = make_files_structure(['source/file1.txt'])
fg = FileGroup.from_paths(test_session, file1)
test_session.commit()
# Destination doesn't exist yet
dest = test_directory / 'new' / 'nested' / 'destination'
assert not dest.exists()
task = FileTask(FileTaskType.move, [file1], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Verify destination was created and file moved
assert dest.exists()
assert (dest / 'file1.txt').exists()
@pytest.mark.asyncio
async def test_file_worker_move_multiple_sources(
async_client, test_session, test_directory, make_files_structure
):
"""FileWorker moves multiple source files to one destination."""
file1, file2 = make_files_structure([
'source1/file1.txt',
'source2/file2.txt',
])
fg1 = FileGroup.from_paths(test_session, file1)
fg2 = FileGroup.from_paths(test_session, file2)
test_session.commit()
dest = test_directory / 'destination'
dest.mkdir()
task = FileTask(FileTaskType.move, [file1, file2], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Verify both files moved
assert (dest / 'file1.txt').exists()
assert (dest / 'file2.txt').exists()
assert not file1.exists()
assert not file2.exists()
@pytest.mark.asyncio
async def test_file_worker_move_already_moved_files(
async_client, test_session, test_directory, make_files_structure
):
"""FileWorker handles files that were already moved on filesystem.
When a user manually moves files and then tells the worker to move them,
the worker should detect the files are already at the destination and
simply update the database records.
"""
import shutil
# Create file and FileGroup at original location
file1, = make_files_structure(['source/file1.txt'])
fg = FileGroup.from_paths(test_session, file1)
test_session.commit()
original_id = fg.id
# User manually moves file on filesystem (outside of WROLPi)
dest = test_directory / 'destination'
dest.mkdir()
shutil.move(str(file1), str(dest / 'file1.txt'))
# Verify: source gone, dest exists
assert not file1.exists()
assert (dest / 'file1.txt').exists()
# Tell worker to move (from old location to new)
task = FileTask(FileTaskType.move, [file1], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Verify FileGroup was updated to new location
test_session.expire_all()
fg = test_session.query(FileGroup).filter(FileGroup.id == original_id).one()
assert str(dest) in str(fg.directory)
assert 'file1.txt' in str(fg.primary_path)
# Status should be idle (success)
assert file_worker.status['status'] == 'idle'
# ============================================================================
# Tests for missing scenarios from files.lib.move
# ============================================================================
@pytest.mark.asyncio
async def test_file_worker_move_rollback_on_failure(
async_client, test_session, test_directory, make_files_structure, monkeypatch
):
"""FileWorker rolls back moved files when an error occurs mid-move."""
files = make_files_structure([
'source/file1.txt',
'source/file2.txt',
'source/file3.txt',
])
for f in files:
FileGroup.from_paths(test_session, f)
test_session.commit()
dest = test_directory / 'destination'
dest.mkdir()
# Patch _move_file_group_files to fail after first file
call_count = 0
original_move = _move_file_group_files
def failing_move(fg, new_path):
nonlocal call_count
call_count += 1
if call_count > 1:
raise IOError("Simulated disk failure")
return original_move(fg, new_path)
monkeypatch.setattr('wrolpi.files.worker._move_file_group_files', failing_move)
task = FileTask(FileTaskType.move, [test_directory / 'source'], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# All source files should still exist (rolled back)
assert files[0].exists(), f"file1.txt should exist after rollback"
assert files[1].exists(), f"file2.txt should exist after rollback"
assert files[2].exists(), f"file3.txt should exist after rollback"
# Destination should be empty or not exist
dest_source = dest / 'source'
assert not dest_source.exists() or not list(dest_source.iterdir()), \
"Destination should be empty after rollback"
@pytest.mark.asyncio
async def test_file_worker_move_unindexed_files(
async_client, test_session, test_directory, make_files_structure
):
"""FileWorker creates FileGroups for files not yet in database."""
# Create files but don't add them all to DB
files = make_files_structure([
'source/indexed.txt',
'source/unindexed.txt',
])
# Only index one file
FileGroup.from_paths(test_session, files[0])
test_session.commit()
dest = test_directory / 'destination'
dest.mkdir()
task = FileTask(FileTaskType.move, [test_directory / 'source'], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Both files should be moved
assert (dest / 'source' / 'indexed.txt').exists()
assert (dest / 'source' / 'unindexed.txt').exists()
# Both should have FileGroups now
test_session.expire_all()
file_groups = test_session.query(FileGroup).all()
assert len(file_groups) == 2
@pytest.mark.asyncio
async def test_file_worker_move_preserves_existing_destination_on_failure(
async_client, test_session, test_directory, make_files_structure, monkeypatch
):
"""FileWorker preserves existing destination directory on failure."""
source_file, existing_file = make_files_structure([
'source/file1.txt',
'destination/existing.txt', # Pre-existing file
])
FileGroup.from_paths(test_session, source_file)
test_session.commit()
dest = test_directory / 'destination'
# Patch to fail
def failing_move(fg, new_path):
raise IOError("Simulated failure")
monkeypatch.setattr('wrolpi.files.worker._move_file_group_files', failing_move)
task = FileTask(FileTaskType.move, [source_file], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Destination should still exist with original file
assert dest.exists(), "Existing destination should be preserved"
assert existing_file.exists(), "Existing file should be preserved"
@pytest.mark.asyncio
async def test_file_worker_move_removes_created_destination_on_failure(
async_client, test_session, test_directory, make_files_structure, monkeypatch
):
"""FileWorker removes destination directory it created when move fails."""
source_file, = make_files_structure(['source/file1.txt'])
FileGroup.from_paths(test_session, source_file)
test_session.commit()
dest = test_directory / 'new_destination'
assert not dest.exists() # Doesn't exist yet
# Patch to fail
def failing_move(fg, new_path):
raise IOError("Simulated failure")
monkeypatch.setattr('wrolpi.files.worker._move_file_group_files', failing_move)
task = FileTask(FileTaskType.move, [source_file], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Destination should NOT exist (was created by move and should be cleaned up)
assert not dest.exists(), "Created destination should be removed on failure"
@pytest.mark.asyncio
async def test_file_worker_move_cleans_directory_records(
async_client, test_session, test_directory, make_files_structure
):
"""FileWorker removes Directory records for directories that no longer exist.
Directories that still exist (even if empty) keep their DB records.
"""
files = make_files_structure([
'source/subdir/file1.txt',
])
# Create directory records
source_dir = test_directory / 'source'
subdir = source_dir / 'subdir'
test_session.add(Directory(path=str(source_dir), name='source'))
test_session.add(Directory(path=str(subdir), name='subdir'))
FileGroup.from_paths(test_session, files[0])
test_session.commit()
dest = test_directory / 'destination'
dest.mkdir()
task = FileTask(FileTaskType.move, [source_dir], destination=dest)
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
test_session.expire_all()
# subdir no longer exists - its DB record should be deleted
subdir_record = test_session.query(Directory).filter(
Directory.path == str(subdir)
).one_or_none()
assert subdir_record is None, "subdir Directory record should be deleted (directory no longer exists)"
assert not subdir.exists(), "subdir should not exist on disk"
# source still exists (empty) - its DB record should remain
source_record = test_session.query(Directory).filter(
Directory.path == str(source_dir)
).one_or_none()
assert source_record is not None, "source Directory record should remain (directory still exists)"
assert source_dir.exists(), "source should still exist on disk (empty)"

1797
wrolpi/files/worker.py Normal file

File diff suppressed because it is too large Load diff

View file

@ -10,7 +10,7 @@ from wrolpi.common import logger, Base
__all__ = [
'get_flags',
'refreshing',
'file_worker_busy',
'refresh_complete',
]
@ -106,8 +106,6 @@ class Flag:
# Set by `check_db_is_up`.
db_up = Flag('db_up')
# The global refresh is running.
refreshing = Flag('refreshing')
# Outdated Zims need to be removed.
outdated_zims = Flag('outdated_zims', store_db=True)
# Map Importing
@ -115,12 +113,14 @@ map_importing = Flag('map_importing')
# Used to disable or enable downloading when Internet is down.
have_internet = Flag('have_internet')
# Steps of refreshing.
refresh_counting = Flag('refresh_counting')
refresh_discovery = Flag('refresh_discovery')
refresh_modeling = Flag('refresh_modeling')
refresh_indexing = Flag('refresh_indexing')
refresh_cleanup = Flag('refresh_cleanup')
# The FileWorker is busy (refreshing, moving, tagging, or renaming files).
file_worker_busy = Flag('file_worker_busy')
# Steps of FileWorker operations.
file_worker_counting = Flag('file_worker_counting')
file_worker_discovery = Flag('file_worker_discovery')
file_worker_modeling = Flag('file_worker_modeling')
file_worker_indexing = Flag('file_worker_indexing')
file_worker_cleanup = Flag('file_worker_cleanup')
# The global refresh has been performed. This is False on a fresh instance of WROLPi.
refresh_complete = Flag('refresh_complete', store_db=True)
@ -130,16 +130,16 @@ def get_flags() -> dict:
"""Return a list of all Flags which are set."""
flags = dict(
db_up=db_up.is_set(),
file_worker_busy=file_worker_busy.is_set(),
file_worker_cleanup=file_worker_cleanup.is_set(),
file_worker_counting=file_worker_counting.is_set(),
file_worker_discovery=file_worker_discovery.is_set(),
file_worker_indexing=file_worker_indexing.is_set(),
file_worker_modeling=file_worker_modeling.is_set(),
have_internet=have_internet.is_set(),
map_importing=map_importing.is_set(),
outdated_zims=outdated_zims.is_set(),
refresh_cleanup=refresh_cleanup.is_set(),
refresh_complete=refresh_complete.is_set(),
refresh_counting=refresh_counting.is_set(),
refresh_discovery=refresh_discovery.is_set(),
refresh_indexing=refresh_indexing.is_set(),
refresh_modeling=refresh_modeling.is_set(),
refreshing=refreshing.is_set(),
have_internet=have_internet.is_set(),
)
return flags

View file

@ -510,7 +510,7 @@ async def feed(request: Request, query: schema.EventsRequest):
# available.
start = now()
after = None if query.after == 'None' else dates.strpdate(query.after)
after = None if query.after in (None, 'None') else dates.strpdate(query.after)
events = get_events(after)
return json_response(dict(events=events, now=start))

View file

@ -84,16 +84,16 @@ class SettingsRequest:
@dataclass
class FlagsStatusResponse:
db_up: bool
file_worker_busy: bool
file_worker_cleanup: bool
file_worker_counting: bool
file_worker_discovery: bool
file_worker_indexing: bool
file_worker_modeling: bool
have_internet: bool
map_importing: bool
outdated_zims: bool
refresh_cleanup: bool
refresh_complete: bool
refresh_counting: bool
refresh_discovery: bool
refresh_indexing: bool
refresh_modeling: bool
refreshing: bool
have_internet: bool
@dataclass

View file

@ -85,7 +85,7 @@ def register_switch_handler(switch_name: str):
DEBUG_LOGGED = False
@perpetual_signal(sleep=0.1, run_while_testing=True)
@perpetual_signal(sleep=0.1, run_while_testing=False)
async def switch_worker():
"""Watches `api_app.shared_ctx.switches` for new activated switches, handles each one at a time."""
global DEBUG_LOGGED
@ -99,34 +99,36 @@ async def switch_worker():
switch_name = None
try:
# Get new switch and its context. Handle each switch one at a time.
switches_changed.wait(timeout=1)
# During testing, don't block the event loop - check for pending switches directly.
# In production, use blocking wait which is more efficient for long-running processes.
if PYTEST:
# Non-blocking: return immediately if no switches pending.
# The perpetual_signal decorator will call again in 0.1s.
if not api_app.shared_ctx.switches:
return
else:
switches_changed.wait(timeout=1)
switches: dict = api_app.shared_ctx.switches
with api_app.shared_ctx.switches_lock:
switch_name, context = switches.popitem()
switches_keys = list(switches.keys())
# Call handler with the stored context, await coroutine, if any.
handler = SWITCH_HANDLERS[switch_name]
if logger.isEnabledFor(logging.DEBUG):
logger.debug(f'switch_worker handling {switch_name} of {switches_keys}')
else:
logger.info(f'switch_worker handling {switch_name} of {len(switches_keys)}')
coro = handler(**context)
if inspect.iscoroutine(coro):
# Handler is async.
await coro
logger.debug(f'switch_worker completed {switch_name}')
except TimeoutError:
# `switches_changed` is not set.
pass
except StopIteration:
# No switches to handle.
pass
except KeyError:
except KeyError as e:
# `switches` is empty, or `switch_name` not in `SWITCH_HANDLER`
if switch_name:
logger.critical(f'No switch handler defined for: {switch_name}')
finally:
# Clear only after task is complete to avoid
# Clear only after task is complete to avoid race conditions
switches_changed.clear()
@ -137,13 +139,21 @@ async def await_switches(timeout: int = 10):
count = 0
while count < (timeout * 10):
count += 1
if api_app.shared_ctx.switches:
await asyncio.sleep(0.1)
continue
if api_app.shared_ctx.switches_changed.is_set():
switches_dict = dict(api_app.shared_ctx.switches) if api_app.shared_ctx.switches else {}
switches_changed = api_app.shared_ctx.switches_changed.is_set()
if switches_dict or switches_changed:
# Call switch_worker directly instead of using Sanic signal dispatch.
# Signal dispatch causes issues with ASGI testing in Sanic 25.x.
try:
await switch_worker()
except Exception as e:
# Log errors but continue processing, matching perpetual_signal behavior
logger.error(f'switch_worker had error: {e}')
await asyncio.sleep(0.1)
continue
# All switches handled.
break
else:
logger.error(f'await_switches: TIMEOUT after {count} iterations, switches={list(api_app.shared_ctx.switches.keys())}, changed={api_app.shared_ctx.switches_changed.is_set()}')
raise RuntimeError('Timed out waiting for switches. Did you remember to use the `await_switches` fixture?')

View file

@ -1,3 +1,4 @@
import contextlib
import pathlib
from dataclasses import dataclass, field
@ -16,7 +17,7 @@ from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_direc
from wrolpi.dates import TZDateTime
from wrolpi.db import get_db_curs, get_db_session
from wrolpi.downloader import save_downloads_config
from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, RefreshConflict, NoPrimaryFile
from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, FileWorkerConflict, NoPrimaryFile
from wrolpi.events import Events
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.vars import PYTEST
@ -164,8 +165,8 @@ class Tag(ModelHelper, Base):
if (other := self.get_by_name(session, name)) and other.id != self.id:
raise InvalidTag('Tag name already taken')
if flags.refreshing.is_set():
raise RefreshConflict('Refusing to rename tag while file refresh is in progress')
if flags.file_worker_busy.is_set():
raise FileWorkerConflict('Refusing to rename tag while FileWorker is busy')
old_name = self.name
self.name = name
@ -607,7 +608,8 @@ def _delete_extra_tags_directory_paths(directory: pathlib.Path, links: List[path
logger.warning(f'Refusing to delete extra Tags Directory directory which contains files: {directory}')
else:
logger.debug(f'Deleting extra Tags Directory directory: {directory}')
directory.rmdir()
from wrolpi.files.lib import delete_directory
delete_directory(directory)
def create_tags_directory(directory: pathlib.Path):

View file

@ -781,7 +781,8 @@ async def test_batch_api_endpoints(async_client, test_session, test_download_man
assert response.status_code == HTTPStatus.OK
assert response.json['retried_count'] == 1
test_session.refresh(d1)
assert d1.status == 'new'
# Status may be 'new' or 'pending' depending on whether the download worker picked it up
assert d1.status in ('new', 'pending')
# Test batch clear endpoint
body = dict(download_ids=[d2_id])

View file

@ -18,7 +18,7 @@ def assert_events(expected: List[Dict], after=None):
@pytest.mark.asyncio
async def test_events_api_feed(test_session, async_client, example_pdf):
async def test_events_api_feed(test_session, async_client, example_pdf, events_fixture):
"""Events can be gotten from the API."""
# refresh_files() creates a few events.
request, response = await async_client.post('/api/files/refresh')
@ -28,6 +28,15 @@ async def test_events_api_feed(test_session, async_client, example_pdf):
assert response.status_code == HTTPStatus.OK, response.body
assert (result := response.json.get('events'))
# Verify using the fixture
events_fixture.assert_has_event('refresh_completed')
events_fixture.assert_has_event('global_after_refresh_completed')
events_fixture.assert_has_event('global_refresh_indexing_completed')
events_fixture.assert_has_event('global_refresh_modeling_completed')
events_fixture.assert_has_event('global_refresh_discovery_completed')
events_fixture.assert_has_event('global_refresh_started')
# Also verify the API returns the same events
expected = [
dict(event='refresh_completed', subject='refresh'),
dict(event='global_after_refresh_completed', subject='refresh'),
@ -41,22 +50,29 @@ async def test_events_api_feed(test_session, async_client, example_pdf):
@pytest.mark.asyncio
async def test_events_history_limit(async_client):
async def test_events_history_limit(async_client, events_fixture):
"""EVENTS_HISTORY has a limited size."""
for i in range(HISTORY_SIZE + 5):
Events.send_ready()
assert len(get_events()) == HISTORY_SIZE
# Fixture provides access to events, verify size limit
assert len(events_fixture.events) == HISTORY_SIZE
@pytest.mark.asyncio
async def test_get_events(async_client):
async def test_get_events(async_client, events_fixture):
"""Test that get_events() can filter by timestamp."""
after = now()
Events.send_global_refresh_started()
await asyncio.sleep(1)
Events.send_ready()
# Fixture shows all events
events_fixture.assert_has_event('ready')
events_fixture.assert_has_event('global_refresh_started')
# get_events with after parameter filters by timestamp
assert_events([dict(event='ready'), dict(event='global_refresh_started')], after=after)
after = after + timedelta(seconds=0.1)

View file

@ -30,20 +30,20 @@ def test_flags(async_client, test_session, flags_lock):
def test_flags_with(async_client, flags_lock):
"""A Flag can be used with `with` context."""
assert flags.refreshing.is_set() is False, 'Flag should not be set by default'
assert flags.file_worker_busy.is_set() is False, 'Flag should not be set by default'
assert not any(v for k, v in flags.get_flags().items())
with flags.refreshing:
assert flags.refreshing.is_set() is True, 'Flag should be set within context.'
assert flags.get_flags()['refreshing']
with flags.file_worker_busy:
assert flags.file_worker_busy.is_set() is True, 'Flag should be set within context.'
assert flags.get_flags()['file_worker_busy']
assert flags.refreshing.is_set() is False, 'Flag should not be restored after context.'
assert flags.file_worker_busy.is_set() is False, 'Flag should not be restored after context.'
assert not any(v for k, v in flags.get_flags().items())
with pytest.raises(ValueError):
# Can't do two `refreshing` as once.
with flags.refreshing:
with flags.refreshing:
# Can't do two `file_worker_busy` as once.
with flags.file_worker_busy:
with flags.file_worker_busy:
raise Exception('We should not get here!')
@ -51,5 +51,5 @@ def test_flags_with(async_client, flags_lock):
async def test_flag_wait_for(async_client, flags_lock):
"""Wait for throws an error when waiting exceeds timeout."""
with pytest.raises(TimeoutError):
async with flags.refreshing.wait_for(timeout=1):
async with flags.file_worker_busy.wait_for(timeout=1):
await asyncio.sleep(2)

View file

@ -54,17 +54,17 @@ async def test_perpetual_signal(await_switches):
"""Test that perpetual_signal survives errors."""
# Signal event is dispatched again.
switch_handler_test2.activate_switch()
await asyncio.sleep(1)
await await_switches(timeout=2)
assert count2.value == 1
# Error is logged.
switch_handler_test2.activate_switch()
await asyncio.sleep(1)
await await_switches(timeout=2)
assert count2.value == 2
# Switch is dispatched again.
switch_handler_test2.activate_switch()
await asyncio.sleep(1)
await await_switches(timeout=2)
assert count2.value == 3

View file

@ -7,17 +7,17 @@ import yaml
from wrolpi import tags
from wrolpi.common import is_hardlinked, walk
from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, UsedTag
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.tags import TagFile, Tag
@pytest.mark.asyncio
async def test_tags_file_group_json(async_client, test_session, make_files_structure, tag_factory, example_pdf):
async def test_tags_file_group_json(async_client, test_session, make_files_structure, tag_factory, example_pdf,
refresh_files):
"""The tags of a file are returned in its JSON."""
tag_one = await tag_factory()
tag_two = await tag_factory()
await files_lib.refresh_files()
await refresh_files()
file_group: FileGroup = test_session.query(FileGroup).one()
file_group.add_tag(test_session, tag_one.id)
@ -54,12 +54,12 @@ async def test_tags_model(async_client, test_session, make_files_structure, tag_
@pytest.mark.asyncio
async def test_tags_file_group(async_client, test_session, make_files_structure, tag_factory, video_bytes,
image_bytes_factory):
image_bytes_factory, refresh_files):
"""A FileGroup can be tagged with multiple Tags."""
make_files_structure({
'video.mp4': video_bytes, 'video.png': image_bytes_factory(),
})
await files_lib.refresh_files()
await refresh_files()
video_group: FileGroup = test_session.query(FileGroup).one()
tag_one = await tag_factory()
@ -89,9 +89,9 @@ async def test_tags_file_group(async_client, test_session, make_files_structure,
@pytest.mark.asyncio
async def test_tags_config_file(test_session, test_directory, tag_factory, example_pdf, video_file, await_switches,
test_tags_config):
test_tags_config, refresh_files):
"""Test that the config is updated when a FileGroup is tagged."""
await files_lib.refresh_files()
await refresh_files()
pdf: FileGroup = FileGroup.get_by_path(test_session, example_pdf)
video: FileGroup = FileGroup.get_by_path(test_session, video_file)
tag1 = await tag_factory()
@ -208,9 +208,9 @@ async def test_tags_crud(async_client, test_session, example_pdf, assert_tags_co
@pytest.mark.asyncio
async def test_delete_tagged_file(test_session, example_pdf, tag_factory):
async def test_delete_tagged_file(test_session, example_pdf, tag_factory, refresh_files):
"""You cannot delete a FileGroup if it is tagged."""
await files_lib.refresh_files()
await refresh_files()
tag = await tag_factory()
pdf: FileGroup = test_session.query(FileGroup).one()
@ -351,7 +351,8 @@ async def test_invalid_tag(async_client, test_session, test_directory):
@pytest.mark.asyncio
async def test_tags_directory(test_session, test_directory, tag_factory, video_factory, example_pdf, await_switches):
async def test_tags_directory(test_session, test_directory, tag_factory, video_factory, example_pdf, await_switches,
refresh_files):
"""Test that Tag Directory is synchronized with database."""
readme = test_directory / 'tags/README.txt'
@ -454,7 +455,7 @@ async def test_tags_directory(test_session, test_directory, tag_factory, video_f
f'First Aid lingers and contains: {[i.name for i in (test_directory / "tags/First Aid").iterdir()]}'
# Only three FileGroups were created. Tags Directory files are ignored during refresh.
await files_lib.refresh_files()
await refresh_files()
assert test_session.query(FileGroup).count() == 3