From 173284699d54540d4b722a3c3814173c8b74ffd1 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 9 Jan 2026 14:01:25 -0700 Subject: [PATCH] FileWorker: Async queue-based file synchronization system Introduces FileWorker for background file operations with dual-queue architecture (public/private). Handles refresh, move, and rename tasks with progress tracking, job completion monitoring, and batch processing. Compares DB FileGroups to filesystem state using FileGroupDiff to detect new/modified/deleted files. Designed for memory efficiency on Raspberry Pi. --- app/src/DashboardPage.js | 6 +- app/src/api.js | 4 +- app/src/components/Files.js | 75 +- app/src/components/Nav.js | 2 +- app/src/hooks/customHooks.js | 2 +- main.py | 9 +- modules/archive/test/test_api.py | 5 +- modules/archive/test/test_lib.py | 34 +- modules/videos/__init__.py | 1 + modules/videos/channel/lib.py | 6 +- modules/videos/channel/test/test_api.py | 10 +- modules/videos/models.py | 8 +- modules/videos/test/test_api.py | 9 +- modules/videos/test/test_common.py | 5 +- modules/videos/test/test_lib.py | 6 +- modules/videos/test/test_video.py | 6 +- modules/zim/lib.py | 7 +- modules/zim/test/test_lib.py | 13 +- pytest.ini | 5 +- requirements.txt | 7 +- wrolpi/api_utils.py | 4 +- wrolpi/collections/lib.py | 16 +- wrolpi/collections/models.py | 13 +- .../test/test_collection_tagging.py | 5 +- wrolpi/common.py | 67 +- wrolpi/conftest.py | 144 +- wrolpi/contexts.py | 26 + wrolpi/errors.py | 6 +- wrolpi/events.py | 4 + wrolpi/files/api.py | 37 +- wrolpi/files/lib.py | 405 +--- wrolpi/files/test/test_api.py | 59 +- wrolpi/files/test/test_ebooks.py | 18 +- wrolpi/files/test/test_lib.py | 306 +-- wrolpi/files/test/test_models.py | 5 +- wrolpi/files/test/test_pdfs.py | 6 +- wrolpi/files/test/test_worker.py | 1277 ++++++++++++ wrolpi/files/test/test_worker_move.py | 532 +++++ wrolpi/files/worker.py | 1797 +++++++++++++++++ wrolpi/flags.py | 32 +- wrolpi/root_api.py | 2 +- wrolpi/schema.py | 14 +- wrolpi/switches.py | 38 +- wrolpi/tags.py | 10 +- wrolpi/test/test_downloader.py | 3 +- wrolpi/test/test_events.py | 24 +- wrolpi/test/test_flags.py | 18 +- wrolpi/test/test_switches.py | 6 +- wrolpi/test/test_tags.py | 23 +- 49 files changed, 4256 insertions(+), 861 deletions(-) create mode 100644 wrolpi/files/test/test_worker.py create mode 100644 wrolpi/files/test/test_worker_move.py create mode 100644 wrolpi/files/worker.py diff --git a/app/src/DashboardPage.js b/app/src/DashboardPage.js index 29331b88..fceb4e3a 100644 --- a/app/src/DashboardPage.js +++ b/app/src/DashboardPage.js @@ -48,9 +48,9 @@ function FlagsMessages() { let refreshingMessage; let refreshRequiredMessage; - // Do not tell the maintainer to refresh the files if they are already refreshing. - if (flags.refreshing) { - // Actively refreshing. + // Do not tell the maintainer to refresh the files if the FileWorker is busy. + if (flags.file_worker_busy) { + // FileWorker is busy. refreshingMessage = diff --git a/app/src/api.js b/app/src/api.js index a0b98618..9ce68ec3 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -1174,10 +1174,10 @@ export async function deleteFile(paths) { } export async function fetchFilesProgress() { - const response = await apiGet(`${API_URI}/files/refresh_progress`); + const response = await apiGet(`${API_URI}/files/worker_status`); if (response.ok) { const json = await response.json(); - return json['progress']; + return json['status']; } } diff --git a/app/src/components/Files.js b/app/src/components/Files.js index 115413af..aa5c10a6 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -542,50 +542,49 @@ export function FilesSearchView({ } +function getPhaseLabel(status) { + switch (status) { + case 'counting': + return 'Counting'; + case 'comparing': + case 'upserting': + case 'deleting': + return 'Refreshing'; + case 'modeling': + return 'Modeling'; + case 'indexing': + return 'Indexing'; + case 'planning': + return 'Move: Planning'; + case 'moving': + return 'Move: Moving'; + case 'reverting': + return 'Reverting'; + default: + return null; + } +} + export function FilesRefreshProgress() { const {progress} = useFilesProgressInterval(); - if (!progress) { - return; + if (!progress || progress.status === 'idle') { + return null; } - const { - discovery, - refreshing, - modeling, - indexing, - cleanup, - indexed, - unindexed, - counted_files, - total_file_groups, - modeled - } = progress; + const {status, operation_total, operation_processed, operation_percent} = progress; + const phaseLabel = getPhaseLabel(status); - if (refreshing) { - // Default is Counting / Step 1. Move progress bar toward the middle to avoid clipping out of the screen. - let params = {value: Math.floor(counted_files / 2), total: counted_files, progress: 'ratio'}; - let label = 'Refresh: Counting'; - - if (discovery) { - params['value'] = total_file_groups; - params['total'] = counted_files; - label = 'Refresh: Discovery' - } else if (modeling) { - params['value'] = modeled; - params['total'] = total_file_groups; - label = 'Refresh: Modeling'; - } else if (indexing) { - params['value'] = unindexed; - params['total'] = unindexed + indexed; - label = 'Refresh: Indexing'; - } else if (cleanup) { - params['value'] = 3; - params['total'] = 4; - label = 'Refresh: Cleanup'; - } - return {label} + if (!phaseLabel) { + return null; } + + let label = `Refresh: ${phaseLabel}`; + if (operation_total > 0) { + label = `${label} (${operation_processed.toLocaleString()} / ${operation_total.toLocaleString()})`; + } + + return {label}; } function FilesPage() { @@ -610,7 +609,7 @@ export function FilesRoute() { } const useRefresh = () => { - const refreshing = useStatusFlag('refreshing'); + const refreshing = useStatusFlag('file_worker_busy'); const refreshingDirectory = useStatusFlag('refreshing_directory'); const wrolModeEnabled = useWROLMode(); diff --git a/app/src/components/Nav.js b/app/src/components/Nav.js index 1f3c88e1..c815f7a8 100644 --- a/app/src/components/Nav.js +++ b/app/src/components/Nav.js @@ -167,7 +167,7 @@ export function NavBar() { let processingLink; if (status && status.flags) { - if (status.flags.refreshing) { + if (status.flags.file_worker_busy) { processingLink = '/files'; } else if (status.flags.map_importing) { processingLink = '/map/manage'; diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index c7f60985..7f9ffb4c 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -783,7 +783,7 @@ export const useFilesProgress = () => { export const useFilesProgressInterval = () => { const {progress, fetchFilesProgress} = useFilesProgress(); - useRecurringTimeout(fetchFilesProgress, 1000 * 3); + useRecurringTimeout(fetchFilesProgress, 1000); return {progress, fetchFilesProgress}; } diff --git a/main.py b/main.py index 1ba51edd..71286854 100755 --- a/main.py +++ b/main.py @@ -9,11 +9,11 @@ from contextlib import contextmanager from sanic import Sanic from sanic.signals import Event +from modules.archive.lib import import_domains_config from modules.inventory import init_inventory from modules.inventory.common import import_inventories_config from modules.videos.lib import import_channels_config, get_videos_downloader_config from wrolpi import flags -from modules.archive.lib import import_domains_config from wrolpi import root_api # noqa from wrolpi import tags from wrolpi.api_utils import api_app, perpetual_signal @@ -24,6 +24,7 @@ from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initi from wrolpi.dates import Seconds from wrolpi.downloader import import_downloads_config, download_manager, get_download_manager_config from wrolpi.errors import WROLModeEnabled +from wrolpi.files.worker import file_worker from wrolpi.vars import PROJECT_DIR, DOCKERIZED, INTERNET_SERVER from wrolpi.version import get_version_string @@ -312,6 +313,12 @@ async def start_single_tasks(app: Sanic): logger.debug(f'start_single_tasks done') +@perpetual_signal(sleep=1) +async def perpetual_file_worker_queue(): + file_worker.transfer_queue() + await file_worker.process_queue() + + @perpetual_signal(sleep=1) async def perpetual_check_log_level(): """Copies global log level into this Sanic worker's logger.""" diff --git a/modules/archive/test/test_api.py b/modules/archive/test/test_api.py index d393509d..bed82095 100644 --- a/modules/archive/test/test_api.py +++ b/modules/archive/test/test_api.py @@ -185,7 +185,7 @@ async def test_archives_search_no_query(test_session, archive_factory, async_cli @skip_circleci @pytest.mark.asyncio async def test_archive_upload(test_session, async_client, singlefile_contents_factory, make_multipart_form, - await_switches, events_history): + await_switches, events_fixture): """Test converting a SingleFile from the SingleFile browser extension to an Archive.""" singlefile_contents = singlefile_contents_factory(title='upload title', url='https://example.com/single-file-url') forms = [ @@ -204,8 +204,7 @@ async def test_archive_upload(test_session, async_client, singlefile_contents_fa assert archive.screenshot_path.is_file(), 'Screenshot should have been created' assert archive.file_group.title == 'upload title', 'Title was not correct' assert archive.file_group.url == 'https://example.com/single-file-url', 'SingleFile URL should be trusted' - events = list(events_history) - assert events and events[0]['event'] == 'upload_archive', 'Event should have been sent.' + events_fixture.assert_has_event('upload_archive') @pytest.mark.asyncio diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index af7493e2..077e23bd 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -20,7 +20,6 @@ from wrolpi.api_utils import CustomJSONEncoder from wrolpi.collections import Collection from wrolpi.common import get_wrolpi_config from wrolpi.db import get_db_session -from wrolpi.files import lib as files_lib from wrolpi.files.models import FileGroup from wrolpi.test.common import skip_circleci @@ -85,7 +84,7 @@ async def test_relationships(async_client, test_session, example_singlefile): @pytest.mark.asyncio -async def test_archive_title(async_client, test_session, archive_factory, singlefile_contents_factory): +async def test_archive_title(async_client, test_session, archive_factory, singlefile_contents_factory, refresh_files): """An Archive's title can be fetched in multiple ways. This tests from most to least reliable.""" # Create some test files, delete all records for a fresh refresh. archive_factory( @@ -98,7 +97,7 @@ async def test_archive_title(async_client, test_session, archive_factory, single async def reset_and_get_archive(): test_session.query(FileGroup).delete() test_session.commit() - await files_lib.refresh_files() + await refresh_files() return test_session.query(Archive).one() archive1: Archive = await reset_and_get_archive() @@ -175,7 +174,8 @@ async def test_archive_refresh_deleted_archive(async_client, test_session, archi @pytest.mark.asyncio -async def test_fills_contents_with_refresh(async_client, test_session, archive_factory, singlefile_contents_factory): +async def test_fills_contents_with_refresh(async_client, test_session, archive_factory, singlefile_contents_factory, + refresh_files): """Refreshing archives fills in any missing contents.""" archive1 = archive_factory('example.com', 'https://example.com/one') archive2 = archive_factory('example.com', 'https://example.com/one') @@ -208,7 +208,7 @@ async def test_fills_contents_with_refresh(async_client, test_session, archive_f assert not archive4.file_group.d_text # Fill the contents. - await files_lib.refresh_files() + await refresh_files() # The archives will be renamed with their title. archive1, archive2, archive3, archive4 = test_session.query(Archive).order_by(Archive.id) assert not archive4.file_group.d_text @@ -291,7 +291,7 @@ def test_get_domains(test_session, archive_factory): @skip_circleci @pytest.mark.asyncio -async def test_new_archive(test_session, test_directory, fake_now): +async def test_new_archive(test_session, test_directory, fake_now, async_client): singlefile, readability, screenshot = make_fake_archive_result() fake_now(datetime(2000, 1, 1)) archive1 = await model_archive_result('https://example.com', singlefile, readability, screenshot) @@ -334,7 +334,7 @@ async def test_new_archive(test_session, test_directory, fake_now): @pytest.mark.asyncio -async def test_get_title_from_html(test_directory, test_session, fake_now): +async def test_get_title_from_html(test_directory, test_session, fake_now, async_client): fake_now(datetime(2000, 1, 1)) singlefile, readability, screenshot = make_fake_archive_result() archive = await model_archive_result('https://example.com', singlefile, readability, screenshot) @@ -584,11 +584,11 @@ async def test_refresh_archives(test_session, test_directory, async_client, make @pytest.mark.asyncio -async def test_refresh_archives_index(test_session, make_files_structure): +async def test_refresh_archives_index(test_session, make_files_structure, refresh_files): """Archives are indexed using ArchiveIndexer and archive_modeler.""" # The start of a typical singlefile html file. singlefile_contents = ''' @@ -609,7 +609,7 @@ async def test_refresh_archives_index(test_session, make_files_structure): 'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '', }) - await files_lib.refresh_files() + await refresh_files() archive: Archive = test_session.query(Archive).one() assert archive.singlefile_path == singlefile @@ -625,7 +625,7 @@ async def test_refresh_archives_index(test_session, make_files_structure): @pytest.mark.asyncio -async def test_archive_meta(async_client, test_session, make_files_structure): +async def test_archive_meta(async_client, test_session, make_files_structure, refresh_files): # The start of a typical singlefile html file. singlefile_contents = '''