diff --git a/app/src/components/Files.js b/app/src/components/Files.js index f54453eb..a2da4338 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -487,11 +487,12 @@ export function FilesRefreshProgress() { return; } - const {refreshing, modeling, indexing, cleanup, indexed, unindexed, total_files, modeled} = progress; + const {refreshing, modeling, indexing, cleanup, indexed, unindexed, total_files, total_file_groups, modeled} + = progress; if (refreshing) { // Default is Discovery / Step 1. - let params = {value: 0, total: 3, progress: 'ratio'}; + let params = {value: total_file_groups, total: total_files, progress: 'ratio'}; let label = 'Refresh: Discovery'; if (modeling) { diff --git a/wrolpi/count_files.py b/wrolpi/count_files.py new file mode 100755 index 00000000..d98c1805 --- /dev/null +++ b/wrolpi/count_files.py @@ -0,0 +1,30 @@ +#! /usr/bin/env python +import argparse +import os +import pathlib +import sys + + +def count_files(directory: pathlib.Path) -> int: + count = 0 + for path in directory.iterdir(): + if path.is_dir(): + count += count_files(path) + else: + count += 1 + return count + + +if __name__ == '__main__': + default_cwd = pathlib.Path(os.getcwd()) + parser = argparse.ArgumentParser() + parser.add_argument('directory', nargs='?', default=default_cwd, type=pathlib.Path) + + args = parser.parse_args() + + if not args.directory.is_dir(): + print('Directory does not exist', file=sys.stderr) + sys.exit(1) + + total_count = count_files(args.directory) + print(total_count) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index f35a4a80..f9a83568 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -3,6 +3,7 @@ import datetime import functools import glob import json +import multiprocessing import os import pathlib import re @@ -432,6 +433,9 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim curs.execute(stmt) +REFRESH = multiprocessing.Manager().dict() + + @limit_concurrent(1) # Only one refresh at a time. @wrol_mode_check @cancelable_wrapper @@ -458,6 +462,9 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr # Add all files in the media directory to the DB. paths = paths or [get_media_directory()] with flags.refresh_discovery: + from wrolpi.count_files import count_files + total_count = sum(count_files(i) for i in paths) + REFRESH['total_files'] = total_count await refresh_discover_paths(paths, idempotency) if send_events: Events.send_global_refresh_discovery_completed() @@ -836,7 +843,7 @@ def get_refresh_progress(): with get_db_curs() as curs: curs.execute(''' SELECT - COUNT(id) AS "total_files", + COUNT(id) AS "total_file_groups", COUNT(id) FILTER (WHERE indexed IS TRUE) AS "indexed", COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed", COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled" @@ -852,7 +859,8 @@ def get_refresh_progress(): modeled=results['modeled'], modeling=flags.refresh_modeling.is_set(), refreshing=flags.refreshing.is_set(), - total_files=results['total_files'], + total_files=REFRESH.get('total_files', 0), + total_file_groups=results['total_file_groups'], unindexed=results['unindexed'], )