Adding directories of uploaded files, if they are missing.

This commit is contained in:
Roland 2025-01-06 16:46:53 -07:00
parent a9fbf7b384
commit a363b18734
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
7 changed files with 66 additions and 9 deletions

View file

@ -1920,11 +1920,13 @@ export function getDistinctColor(hexColors) {
function isDistinct(newColor, existingColors, threshold) {
const hslNew = newColor;
for (let color of existingColors) {
let hslColor = hexToHSL(color);
if (Math.abs(hslNew.h - hslColor.h) < threshold &&
Math.abs(hslNew.s * 100 - hslColor.s * 100) < threshold &&
Math.abs(hslNew.l - hslColor.l * 100) < threshold) {
for (const color of existingColors) {
const hslColor = hexToHSL(color);
const h = Math.abs(hslNew.h - hslColor.h);
const s = Math.abs(hslNew.s * 100 - hslColor.s * 100);
const l = Math.abs(hslNew.l - hslColor.l * 100);
if (h < threshold && s < threshold && l < threshold) {
console.debug(`trying distinct color h=${h} s=${s} l=${l} with threshold=${threshold}`);
return false;
}
}

View file

@ -1123,6 +1123,17 @@ def get_relative_to_media_directory(path: Union[str, Path]) -> Path:
return absolute.relative_to(get_media_directory())
def get_paths_in_parent_directory(paths: List[Union[str, Path]], parent_directory: pathlib.Path) -> List[Path]:
"""Return only those of the provided paths that are children of the provided parent directory."""
paths = [i if isinstance(i, Path) else Path(i) for i in paths]
return [path for path in paths if parent_directory in path.resolve().parents and path != parent_directory]
def get_paths_in_media_directory(paths: List[Union[str, Path]]) -> List[Path]:
"""Return only those of the provided paths that are children of the media directory."""
return get_paths_in_parent_directory(paths, get_media_directory())
def minimize_dict(d: dict, keys: Iterable) -> Optional[dict]:
"""Return a new dictionary that contains only the keys provided."""
if d is None:

View file

@ -26,7 +26,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co
partition, cancelable_wrapper, \
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, \
get_wrolpi_config, \
timer, chunks, unique_by_predicate
timer, chunks, unique_by_predicate, get_paths_in_media_directory
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
from wrolpi.downloader import download_manager
@ -723,8 +723,7 @@ def upsert_directories(parent_directories, directories):
directories = list(directories) + list(parent_directories)
# Only insert directories that are children of `media_directory` and exist.
media_directory = get_media_directory()
directories = [i for i in directories if i.is_dir() and i != media_directory]
directories = get_paths_in_media_directory([i for i in directories if i.is_dir()])
if directories:
# Insert any directories that were created, update any directories which previously existed.
@ -1579,6 +1578,10 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) ->
if url and download_manager.is_skipped(url):
download_manager.remove_from_skip_list(url)
upsert_directories(file.parents, [])
session.commit()
if tag_names:
with get_db_session(commit=True) as session:
file_group = FileGroup.find_by_id(file_group_id, session)
@ -1587,6 +1590,5 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) ->
tag = Tag.get_by_name(tag_name)
file_group.add_tag(tag.id)
session.commit()
file_group.flush()
return file_group

View file

@ -503,6 +503,11 @@ def test_post_upload_directory(test_session, test_client, test_directory, make_f
assert (test_directory / 'uploads/foo/bar.txt').read_text() == 'foo'
assert test_session.query(FileGroup).count() == 1
assert test_session.query(Directory).count() == 2
assert {i for i, in test_session.query(Directory.path)} == {
test_directory / 'uploads',
test_directory / 'uploads/foo',
}
@pytest.mark.asyncio

View file

View file

@ -43,6 +43,7 @@ systemctl daemon-reload
systemctl restart postgresql@15-map.service
# Dump old DB, then restore in new DB.
>&2 echo "Dumping map database, this may take several hours if your map is large..."
sudo -iu postgres pg_dump -Fd gis --port=5432 -j3 -f /tmp/map.dump && # 5432 is old DB.
sudo -iu postgres pg_restore --port=5433 -j3 -d gis /tmp/map.dump && # 5433 is new DB.
sudo -iu postgres dropdb --port=5432 gis

View file

@ -1110,3 +1110,39 @@ async def test_initialize_config_files(async_client, test_session, test_director
test_session.delete(test_inventory)
assert common.create_empty_config_files() == ['inventories.yaml'], 'Inventory config should have been created.'
assert get_inventories_config().is_valid()
def test_get_paths_in_parent_directory(test_directory, make_files_structure):
make_files_structure([
'foo/bar.txt',
'foo/bar/baz.txt',
'qux.txt',
])
paths = list(common.walk(test_directory))
assert common.get_paths_in_parent_directory(paths, test_directory / 'foo') == [
test_directory / 'foo/bar.txt',
test_directory / 'foo/bar',
test_directory / 'foo/bar/baz.txt',
]
assert common.get_paths_in_parent_directory(paths, test_directory / 'foo/bar') == [
test_directory / 'foo/bar/baz.txt',
]
assert common.get_paths_in_parent_directory(paths, test_directory) == [
test_directory / 'qux.txt',
test_directory / 'foo',
test_directory / 'foo/bar.txt',
test_directory / 'foo/bar',
test_directory / 'foo/bar/baz.txt',
]
assert common.get_paths_in_media_directory(paths) == [
test_directory / 'qux.txt',
test_directory / 'foo',
test_directory / 'foo/bar.txt',
test_directory / 'foo/bar',
test_directory / 'foo/bar/baz.txt',
]