diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 49367058..2c69cca3 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -1920,11 +1920,13 @@ export function getDistinctColor(hexColors) { function isDistinct(newColor, existingColors, threshold) { const hslNew = newColor; - for (let color of existingColors) { - let hslColor = hexToHSL(color); - if (Math.abs(hslNew.h - hslColor.h) < threshold && - Math.abs(hslNew.s * 100 - hslColor.s * 100) < threshold && - Math.abs(hslNew.l - hslColor.l * 100) < threshold) { + for (const color of existingColors) { + const hslColor = hexToHSL(color); + const h = Math.abs(hslNew.h - hslColor.h); + const s = Math.abs(hslNew.s * 100 - hslColor.s * 100); + const l = Math.abs(hslNew.l - hslColor.l * 100); + if (h < threshold && s < threshold && l < threshold) { + console.debug(`trying distinct color h=${h} s=${s} l=${l} with threshold=${threshold}`); return false; } } diff --git a/wrolpi/common.py b/wrolpi/common.py index 003874b7..f7fb91af 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -1123,6 +1123,17 @@ def get_relative_to_media_directory(path: Union[str, Path]) -> Path: return absolute.relative_to(get_media_directory()) +def get_paths_in_parent_directory(paths: List[Union[str, Path]], parent_directory: pathlib.Path) -> List[Path]: + """Return only those of the provided paths that are children of the provided parent directory.""" + paths = [i if isinstance(i, Path) else Path(i) for i in paths] + return [path for path in paths if parent_directory in path.resolve().parents and path != parent_directory] + + +def get_paths_in_media_directory(paths: List[Union[str, Path]]) -> List[Path]: + """Return only those of the provided paths that are children of the media directory.""" + return get_paths_in_parent_directory(paths, get_media_directory()) + + def minimize_dict(d: dict, keys: Iterable) -> Optional[dict]: """Return a new dictionary that contains only the keys provided.""" if d is None: diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index c31b592a..5cfc9ec9 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -26,7 +26,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co partition, cancelable_wrapper, \ get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, \ get_wrolpi_config, \ - timer, chunks, unique_by_predicate + timer, chunks, unique_by_predicate, get_paths_in_media_directory from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session from wrolpi.downloader import download_manager @@ -723,8 +723,7 @@ def upsert_directories(parent_directories, directories): directories = list(directories) + list(parent_directories) # Only insert directories that are children of `media_directory` and exist. - media_directory = get_media_directory() - directories = [i for i in directories if i.is_dir() and i != media_directory] + directories = get_paths_in_media_directory([i for i in directories if i.is_dir()]) if directories: # Insert any directories that were created, update any directories which previously existed. @@ -1579,6 +1578,10 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> if url and download_manager.is_skipped(url): download_manager.remove_from_skip_list(url) + upsert_directories(file.parents, []) + + session.commit() + if tag_names: with get_db_session(commit=True) as session: file_group = FileGroup.find_by_id(file_group_id, session) @@ -1587,6 +1590,5 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> tag = Tag.get_by_name(tag_name) file_group.add_tag(tag.id) - session.commit() file_group.flush() return file_group diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index 1a257faf..c8bc6dc1 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -503,6 +503,11 @@ def test_post_upload_directory(test_session, test_client, test_directory, make_f assert (test_directory / 'uploads/foo/bar.txt').read_text() == 'foo' assert test_session.query(FileGroup).count() == 1 + assert test_session.query(Directory).count() == 2 + assert {i for i, in test_session.query(Directory.path)} == { + test_directory / 'uploads', + test_directory / 'uploads/foo', + } @pytest.mark.asyncio diff --git a/wrolpi/files/test/test_downloader.py b/wrolpi/files/test/test_downloader.py new file mode 100644 index 00000000..e69de29b diff --git a/wrolpi/scripts/migrate_map_db.sh b/wrolpi/scripts/migrate_map_db.sh index 48c4ded8..b387c590 100755 --- a/wrolpi/scripts/migrate_map_db.sh +++ b/wrolpi/scripts/migrate_map_db.sh @@ -43,6 +43,7 @@ systemctl daemon-reload systemctl restart postgresql@15-map.service # Dump old DB, then restore in new DB. +>&2 echo "Dumping map database, this may take several hours if your map is large..." sudo -iu postgres pg_dump -Fd gis --port=5432 -j3 -f /tmp/map.dump && # 5432 is old DB. sudo -iu postgres pg_restore --port=5433 -j3 -d gis /tmp/map.dump && # 5433 is new DB. sudo -iu postgres dropdb --port=5432 gis diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 2b6b7c45..4838d376 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -1110,3 +1110,39 @@ async def test_initialize_config_files(async_client, test_session, test_director test_session.delete(test_inventory) assert common.create_empty_config_files() == ['inventories.yaml'], 'Inventory config should have been created.' assert get_inventories_config().is_valid() + + +def test_get_paths_in_parent_directory(test_directory, make_files_structure): + make_files_structure([ + 'foo/bar.txt', + 'foo/bar/baz.txt', + 'qux.txt', + ]) + + paths = list(common.walk(test_directory)) + + assert common.get_paths_in_parent_directory(paths, test_directory / 'foo') == [ + test_directory / 'foo/bar.txt', + test_directory / 'foo/bar', + test_directory / 'foo/bar/baz.txt', + ] + + assert common.get_paths_in_parent_directory(paths, test_directory / 'foo/bar') == [ + test_directory / 'foo/bar/baz.txt', + ] + + assert common.get_paths_in_parent_directory(paths, test_directory) == [ + test_directory / 'qux.txt', + test_directory / 'foo', + test_directory / 'foo/bar.txt', + test_directory / 'foo/bar', + test_directory / 'foo/bar/baz.txt', + ] + + assert common.get_paths_in_media_directory(paths) == [ + test_directory / 'qux.txt', + test_directory / 'foo', + test_directory / 'foo/bar.txt', + test_directory / 'foo/bar', + test_directory / 'foo/bar/baz.txt', + ]