From a3321cf2fa9029209b452b59c6932c751496210b Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 10 Apr 2023 16:28:10 -0600 Subject: [PATCH] Adding Counting step to file refreshing. --- app/src/components/Files.js | 26 ++++++++++++++++++++------ wrolpi/files/lib.py | 19 +++++++++++++++---- wrolpi/flags.py | 1 + 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/app/src/components/Files.js b/app/src/components/Files.js index 92a8635e..a62796b5 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -487,15 +487,29 @@ export function FilesRefreshProgress() { return; } - const {refreshing, modeling, indexing, cleanup, indexed, unindexed, total_files, total_file_groups, modeled} - = progress; + const { + discovery, + refreshing, + modeling, + indexing, + cleanup, + indexed, + unindexed, + counted_files, + total_file_groups, + modeled + } = progress; if (refreshing) { - // Default is Discovery / Step 1. - let params = {value: total_file_groups, total: total_files, progress: 'ratio'}; - let label = 'Refresh: Discovery'; + // Default is Counting / Step 1. Move progress bar toward the middle to avoid clipping out of the screen. + let params = {value: Math.floor(counted_files / 5), total: counted_files, progress: 'ratio'}; + let label = 'Refresh: Counting'; - if (modeling) { + if (discovery) { + params['value'] = total_file_groups; + params['total'] = counted_files; + label = 'Refresh: Discovery' + } else if (modeling) { params['value'] = modeled; params['total'] = total_file_groups; label = 'Refresh: Modeling'; diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index f9a83568..8c85960e 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -461,10 +461,20 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr # Add all files in the media directory to the DB. paths = paths or [get_media_directory()] + + directories = list(filter(lambda i: i.is_dir(), paths)) + if directories: + with flags.refresh_counting: + while directories: + directory = directories.pop() + files, dirs = get_files_and_directories(directory) + directories.extend(dirs) + REFRESH['counted_files'] = REFRESH.get('counted_files', 0) + len(files) + # Sleep to catch cancel. + await asyncio.sleep(0) + refresh_logger.info(f'Counted {REFRESH["counted_files"]} files') + with flags.refresh_discovery: - from wrolpi.count_files import count_files - total_count = sum(count_files(i) for i in paths) - REFRESH['total_files'] = total_count await refresh_discover_paths(paths, idempotency) if send_events: Events.send_global_refresh_discovery_completed() @@ -853,13 +863,14 @@ def get_refresh_progress(): status = dict( cleanup=flags.cleanup.is_set(), + counting=flags.refresh_counting.is_set(), discovery=flags.refresh_discovery.is_set(), indexed=results['indexed'], indexing=flags.refresh_indexing.is_set(), modeled=results['modeled'], modeling=flags.refresh_modeling.is_set(), refreshing=flags.refreshing.is_set(), - total_files=REFRESH.get('total_files', 0), + counted_files=REFRESH.get('counted_files', 0), total_file_groups=results['total_file_groups'], unindexed=results['unindexed'], ) diff --git a/wrolpi/flags.py b/wrolpi/flags.py index 92c7b3bb..1f90121f 100644 --- a/wrolpi/flags.py +++ b/wrolpi/flags.py @@ -111,6 +111,7 @@ db_up = Flag('db_up') refreshing = Flag('refreshing') # Steps of refreshing. +refresh_counting = Flag('refresh_counting') refresh_discovery = Flag('refresh_discovery') refresh_modeling = Flag('refresh_modeling') refresh_indexing = Flag('refresh_indexing')