Counting files before refresh to display more helpful progress.

This commit is contained in:
Roland 2023-04-10 15:48:25 -06:00
parent d7e284f8de
commit 1016e32846
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
3 changed files with 43 additions and 4 deletions

View file

@ -487,11 +487,12 @@ export function FilesRefreshProgress() {
return;
}
const {refreshing, modeling, indexing, cleanup, indexed, unindexed, total_files, modeled} = progress;
const {refreshing, modeling, indexing, cleanup, indexed, unindexed, total_files, total_file_groups, modeled}
= progress;
if (refreshing) {
// Default is Discovery / Step 1.
let params = {value: 0, total: 3, progress: 'ratio'};
let params = {value: total_file_groups, total: total_files, progress: 'ratio'};
let label = 'Refresh: Discovery';
if (modeling) {

30
wrolpi/count_files.py Executable file
View file

@ -0,0 +1,30 @@
#! /usr/bin/env python
import argparse
import os
import pathlib
import sys
def count_files(directory: pathlib.Path) -> int:
count = 0
for path in directory.iterdir():
if path.is_dir():
count += count_files(path)
else:
count += 1
return count
if __name__ == '__main__':
default_cwd = pathlib.Path(os.getcwd())
parser = argparse.ArgumentParser()
parser.add_argument('directory', nargs='?', default=default_cwd, type=pathlib.Path)
args = parser.parse_args()
if not args.directory.is_dir():
print('Directory does not exist', file=sys.stderr)
sys.exit(1)
total_count = count_files(args.directory)
print(total_count)

View file

@ -3,6 +3,7 @@ import datetime
import functools
import glob
import json
import multiprocessing
import os
import pathlib
import re
@ -432,6 +433,9 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim
curs.execute(stmt)
REFRESH = multiprocessing.Manager().dict()
@limit_concurrent(1) # Only one refresh at a time.
@wrol_mode_check
@cancelable_wrapper
@ -458,6 +462,9 @@ async def refresh_files(paths: List[pathlib.Path] = None, send_events: bool = Tr
# Add all files in the media directory to the DB.
paths = paths or [get_media_directory()]
with flags.refresh_discovery:
from wrolpi.count_files import count_files
total_count = sum(count_files(i) for i in paths)
REFRESH['total_files'] = total_count
await refresh_discover_paths(paths, idempotency)
if send_events:
Events.send_global_refresh_discovery_completed()
@ -836,7 +843,7 @@ def get_refresh_progress():
with get_db_curs() as curs:
curs.execute('''
SELECT
COUNT(id) AS "total_files",
COUNT(id) AS "total_file_groups",
COUNT(id) FILTER (WHERE indexed IS TRUE) AS "indexed",
COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled"
@ -852,7 +859,8 @@ def get_refresh_progress():
modeled=results['modeled'],
modeling=flags.refresh_modeling.is_set(),
refreshing=flags.refreshing.is_set(),
total_files=results['total_files'],
total_files=REFRESH.get('total_files', 0),
total_file_groups=results['total_file_groups'],
unindexed=results['unindexed'],
)