From c01d62cd28901121dbd51332e7ebd7f0432f40a2 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 29 Mar 2023 19:30:33 -0600 Subject: [PATCH] Tags config is imported on startup. --- main.py | 7 +++ modules/videos/test/test_api.py | 6 +-- wrolpi/common.py | 12 +++-- wrolpi/conftest.py | 1 + wrolpi/files/models.py | 4 +- wrolpi/tags.py | 90 +++++++++++++++++++++++++++++++-- wrolpi/test/test_tags.py | 50 ++++++++++++++++-- 7 files changed, 153 insertions(+), 17 deletions(-) diff --git a/main.py b/main.py index eee285db..101fb545 100755 --- a/main.py +++ b/main.py @@ -221,6 +221,13 @@ async def start_workers(app: Sanic): download_manager.start_workers() +@api_app.before_server_start +@limit_concurrent(1) +async def main_import_tags_config(app: Sanic): + from wrolpi import tags + tags.import_tags_config() + + @api_app.after_server_start async def bandwidth_worker(app: Sanic): from wrolpi import status diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py index 09147fb7..be5c90c3 100644 --- a/modules/videos/test/test_api.py +++ b/modules/videos/test/test_api.py @@ -203,9 +203,9 @@ def test_search_videos_file(test_client, test_session, test_directory, video_wit def test_search_videos(test_client, test_session, video_factory, assert_video_search, simple_channel, tag_factory): """Search the Video table. This does not need to use a join with the File table.""" - vid1 = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1') - vid2 = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2', channel_id=simple_channel.id) - vid3 = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3') + vid1: Video = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1') + vid2: Video = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2', channel_id=simple_channel.id) + vid3: Video = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3') tag1, tag2 = tag_factory(), tag_factory() vid1.add_tag(tag1) vid3.add_tag(tag2) diff --git a/wrolpi/common.py b/wrolpi/common.py index 249853d4..4a9fe0bf 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -195,6 +195,13 @@ class ConfigFile: self._config = self.default_config.copy() return + self.width = self.width or 90 + self.initialize() + + def __repr__(self): + return f'<{self.__class__.__name__} file={self.get_file()}>' + + def initialize(self): config_file = self.get_file() self._config = Manager().dict() # Use the default settings to initialize the config. @@ -204,11 +211,6 @@ class ConfigFile: with config_file.open('rt') as fh: self._config.update(yaml.load(fh, Loader=yaml.Loader)) - self.width = self.width or 90 - - def __repr__(self): - return f'<{self.__class__.__name__} file={self.get_file()}>' - def save(self): """ Write this config to its file. diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 0288e78e..1293ef6e 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -18,6 +18,7 @@ from uuid import uuid1, uuid4 import pytest import sqlalchemy +import yaml from PIL import Image from sanic_testing.testing import SanicTestClient from sqlalchemy.engine import Engine, create_engine diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index a1d28bbb..2108d930 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -124,11 +124,11 @@ class FileGroup(ModelHelper, Base): @optional_session def add_tag(self, tag: Tag, session: Session = None) -> TagFile: - return tag.add_tag(self, session) + return tag.add_file_group_tag(self, session) @optional_session def remove_tag(self, tag: Tag, session: Session = None): - tag.remove_tag(self, session) + tag.remove_file_group_tag(self, session) def append_files(self, *paths: pathlib.Path): """Add all `paths` to this FileGroup.files.""" diff --git a/wrolpi/tags.py b/wrolpi/tags.py index 81fd7aff..ec0a3d0d 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -1,11 +1,12 @@ import contextlib -from typing import List +from typing import List, Dict from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import relationship, Session -from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_directory, background_task, run_after +from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_directory, background_task, run_after, \ + limit_concurrent from wrolpi.db import optional_session from wrolpi.errors import UnknownTag, UsedTag, InvalidTag from wrolpi.vars import PYTEST @@ -43,7 +44,7 @@ class Tag(ModelHelper, Base): ) @optional_session - def add_tag(self, file_group, session: Session = None) -> TagFile: + def add_file_group_tag(self, file_group, session: Session = None) -> TagFile: """Add a TagFile for the provided FileGroup and this Tag. @warning: Commits the session to keep the config in sync.""" @@ -62,7 +63,7 @@ class Tag(ModelHelper, Base): return tag_file @optional_session - def remove_tag(self, file_group, session: Session = None): + def remove_file_group_tag(self, file_group, session: Session = None): """Remove the record of a Tag applied to the FileGroup. @warning: Commits the session to keep config in sync.""" @@ -107,6 +108,14 @@ class TagsConfig(ConfigFile): value = sorted(value, key=lambda i: (i[0].lower(), i[1])) self.update({'tag_files': value}) + @property + def tags(self) -> dict: + return self._config['tags'] + + @tags.setter + def tags(self, value: dict): + self.update({'tags': value}) + def save_tags(self, session: Session): media_directory = get_media_directory() @@ -211,3 +220,76 @@ def delete_tag(tag_id: int, session: Session = None): session.delete(tag) session.commit() + + +@optional_session +def import_tags_config(session: Session = None): + """Reads the Tags and TagFiles from the config file, upserts them in the DB.""" + if PYTEST and not TEST_TAGS_CONFIG: + logger.warning('Refusing to import tags without test tags config. ' + 'Use `test_tags_config` fixture if you would like to call this.') + return + + config = get_tags_config() + if not (path := config.get_file()).is_file(): + logger.warning(f'Refusing to import tags config because it does not exist: {path}') + return + + logger.info('Importing tags config') + + try: + if config.tags: + # Tags have been saved to config, import them + tags_by_name: Dict[str, Tag] = {i.name: i for i in session.query(Tag)} + new_tags = list() + for name, attrs in config.tags.items(): + tag = tags_by_name.get(name) + if not tag: + # Maintainer added a Tag to the config manually, or DB was wiped. + tag = Tag(name=name, color=attrs['color']) + new_tags.append(tag) + tag.color = attrs['color'] + + if new_tags: + session.add_all(new_tags) + + session.commit() + + if config.tag_files: + from wrolpi.files.models import FileGroup + + need_commit = False + + # Get all Tags again because new ones may exist. + tags_by_name: Dict[str, Tag] = {i.name: i for i in session.query(Tag)} + + media_directory = get_media_directory() + primary_paths = [str(media_directory / i[1]) for i in config.tag_files] + file_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(primary_paths)) + file_groups_by_primary_path = {i.primary_path: i for i in file_groups} + file_group_ids = [i.id for i in file_groups] + # Get all TagFiles referencing the FileGroups. + tag_files = session.query(TagFile).filter(TagFile.file_group_id.in_(file_group_ids)) + tag_files = [(i.tag_id, i.file_group_id) for i in tag_files] + + for tag_name, primary_path in config.tag_files: + tag: Tag = tags_by_name.get(tag_name) + # Paths are absolute in the DB, relative in config. + absolute_path = media_directory / primary_path + file_group: FileGroup = file_groups_by_primary_path.get(absolute_path) + if tag and file_group: + if (tag.id, file_group.id) not in tag_files: + # This FileGroup has not been tagged with the Tag, add it. + tag.add_file_group_tag(file_group, session) + need_commit = True + elif not file_group: + logger.warning(f'Cannot find FileGroup for {repr(str(primary_path))}') + elif not tag: + logger.warning(f'Cannot find Tag for {repr(str(primary_path))}') + + if need_commit: + session.commit() + except Exception as e: + logger.error(f'Failed to import tags config', exc_info=e) + if PYTEST: + raise diff --git a/wrolpi/test/test_tags.py b/wrolpi/test/test_tags.py index f59e2734..f589be90 100644 --- a/wrolpi/test/test_tags.py +++ b/wrolpi/test/test_tags.py @@ -2,6 +2,7 @@ import json from http import HTTPStatus import pytest +import yaml from wrolpi import tags from wrolpi.files import lib as files_lib @@ -17,7 +18,7 @@ async def test_tags_file_group_json(test_session, make_files_structure, tag_fact await files_lib.refresh_files() file_group: FileGroup = test_session.query(FileGroup).one() - tag_one.add_tag(file_group) + tag_one.add_file_group_tag(file_group) assert file_group.__json__()['tags'] == ['one'] file_group.add_tag(tag_two) @@ -37,13 +38,13 @@ async def test_tags_file_group(test_session, make_files_structure, tag_factory): tag_one = tag_factory() assert len(tag_one.tag_files) == 0 - tag_one.add_tag(video_group) + tag_one.add_file_group_tag(video_group) test_session.commit() assert len(tag_one.tag_files) == 1 assert sorted([i.tag.name for i in video_group.tag_files]) == ['one', ] tag_two = tag_factory() - tag_two.add_tag(video_group) + tag_two.add_file_group_tag(video_group) test_session.commit() assert len(tag_one.tag_files) == 1 assert len(tag_two.tag_files) == 1 @@ -166,3 +167,46 @@ async def test_delete_tagged_file(test_session, example_pdf, tag_factory): pdf.delete() test_session.commit() + + +def test_import_empty_tags_config(test_session, test_tags_config): + """An empty config can be imported.""" + tags.import_tags_config() + + +@pytest.mark.asyncio() +async def test_import_tags_config(test_session, test_directory, test_tags_config, example_singlefile): + with test_tags_config.open('wt') as fh: + data = dict( + tags={'Foo': {'color': '#123456'}}, + tag_files=[ + ('Foo', str(example_singlefile.relative_to(test_directory))), + ('Foo', 'does not exist'), + ], + ) + yaml.dump(data, fh) + + assert test_session.query(tags.Tag).count() == 0, 'Expected no Tags.' + assert test_session.query(tags.TagFile).count() == 0, 'Expected no FileGroups to be tagged.' + + # Re-initialize config with the data we just saved. + tags.get_tags_config().initialize() + + FileGroup.from_paths(test_session, example_singlefile) + test_session.commit() + + tags.import_tags_config(test_session) + + # 'Foo' Tag was created. + assert test_session.query(tags.Tag).count() == 1 + + file_group: FileGroup = test_session.query(FileGroup).one() + assert file_group.tag_files, 'File was not tagged during import.' + + # Importing again does not change count. + await files_lib.refresh_files() + tags.import_tags_config(test_session) + assert test_session.query(tags.Tag).count() == 1 + assert test_session.query(tags.TagFile).count() == 1 + # Example file, and the config file. + assert test_session.query(FileGroup).count() == 2