Tags config is imported on startup.

main.py

@@ -221,6 +221,13 @@ async def start_workers(app: Sanic):
     download_manager.start_workers()
 
 
+@api_app.before_server_start
+@limit_concurrent(1)
+async def main_import_tags_config(app: Sanic):
+    from wrolpi import tags
+    tags.import_tags_config()
+
+
 @api_app.after_server_start
 async def bandwidth_worker(app: Sanic):
     from wrolpi import status

modules/videos/test/test_api.py

@@ -203,9 +203,9 @@ def test_search_videos_file(test_client, test_session, test_directory, video_wit
 
 def test_search_videos(test_client, test_session, video_factory, assert_video_search, simple_channel, tag_factory):
     """Search the Video table.  This does not need to use a join with the File table."""
-    vid1 = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
-    vid2 = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2', channel_id=simple_channel.id)
-    vid3 = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
+    vid1: Video = video_factory(upload_date='2022-09-16', with_video_file=True, title='vid1')
+    vid2: Video = video_factory(upload_date='2022-09-17', with_video_file=True, title='vid2', channel_id=simple_channel.id)
+    vid3: Video = video_factory(upload_date='2022-09-18', with_video_file=True, title='vid3')
     tag1, tag2 = tag_factory(), tag_factory()
     vid1.add_tag(tag1)
     vid3.add_tag(tag2)

wrolpi/common.py

@@ -195,6 +195,13 @@ class ConfigFile:
             self._config = self.default_config.copy()
             return
 
+        self.width = self.width or 90
+        self.initialize()
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__} file={self.get_file()}>'
+
+    def initialize(self):
         config_file = self.get_file()
         self._config = Manager().dict()
         # Use the default settings to initialize the config.
@@ -204,11 +211,6 @@ class ConfigFile:
             with config_file.open('rt') as fh:
                 self._config.update(yaml.load(fh, Loader=yaml.Loader))
 
-        self.width = self.width or 90
-
-    def __repr__(self):
-        return f'<{self.__class__.__name__} file={self.get_file()}>'
-
     def save(self):
         """
         Write this config to its file.

wrolpi/conftest.py

@@ -18,6 +18,7 @@ from uuid import uuid1, uuid4
 
 import pytest
 import sqlalchemy
+import yaml
 from PIL import Image
 from sanic_testing.testing import SanicTestClient
 from sqlalchemy.engine import Engine, create_engine

wrolpi/files/models.py

@@ -124,11 +124,11 @@ class FileGroup(ModelHelper, Base):
 
     @optional_session
     def add_tag(self, tag: Tag, session: Session = None) -> TagFile:
-        return tag.add_tag(self, session)
+        return tag.add_file_group_tag(self, session)
 
     @optional_session
     def remove_tag(self, tag: Tag, session: Session = None):
-        tag.remove_tag(self, session)
+        tag.remove_file_group_tag(self, session)
 
     def append_files(self, *paths: pathlib.Path):
         """Add all `paths` to this FileGroup.files."""

wrolpi/tags.py

@@ -1,11 +1,12 @@
 import contextlib
-from typing import List
+from typing import List, Dict
 
 from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import relationship, Session
 
-from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_directory, background_task, run_after
+from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_directory, background_task, run_after, \
+    limit_concurrent
 from wrolpi.db import optional_session
 from wrolpi.errors import UnknownTag, UsedTag, InvalidTag
 from wrolpi.vars import PYTEST
@@ -43,7 +44,7 @@ class Tag(ModelHelper, Base):
         )
 
     @optional_session
-    def add_tag(self, file_group, session: Session = None) -> TagFile:
+    def add_file_group_tag(self, file_group, session: Session = None) -> TagFile:
         """Add a TagFile for the provided FileGroup and this Tag.
 
         @warning: Commits the session to keep the config in sync."""
@@ -62,7 +63,7 @@ class Tag(ModelHelper, Base):
         return tag_file
 
     @optional_session
-    def remove_tag(self, file_group, session: Session = None):
+    def remove_file_group_tag(self, file_group, session: Session = None):
         """Remove the record of a Tag applied to the FileGroup.
 
         @warning: Commits the session to keep config in sync."""
@@ -107,6 +108,14 @@ class TagsConfig(ConfigFile):
         value = sorted(value, key=lambda i: (i[0].lower(), i[1]))
         self.update({'tag_files': value})
 
+    @property
+    def tags(self) -> dict:
+        return self._config['tags']
+
+    @tags.setter
+    def tags(self, value: dict):
+        self.update({'tags': value})
+
     def save_tags(self, session: Session):
         media_directory = get_media_directory()
 
@@ -211,3 +220,76 @@ def delete_tag(tag_id: int, session: Session = None):
 
     session.delete(tag)
     session.commit()
+
+
+@optional_session
+def import_tags_config(session: Session = None):
+    """Reads the Tags and TagFiles from the config file, upserts them in the DB."""
+    if PYTEST and not TEST_TAGS_CONFIG:
+        logger.warning('Refusing to import tags without test tags config.  '
+                       'Use `test_tags_config` fixture if you would like to call this.')
+        return
+
+    config = get_tags_config()
+    if not (path := config.get_file()).is_file():
+        logger.warning(f'Refusing to import tags config because it does not exist: {path}')
+        return
+
+    logger.info('Importing tags config')
+
+    try:
+        if config.tags:
+            # Tags have been saved to config, import them
+            tags_by_name: Dict[str, Tag] = {i.name: i for i in session.query(Tag)}
+            new_tags = list()
+            for name, attrs in config.tags.items():
+                tag = tags_by_name.get(name)
+                if not tag:
+                    # Maintainer added a Tag to the config manually, or DB was wiped.
+                    tag = Tag(name=name, color=attrs['color'])
+                    new_tags.append(tag)
+                tag.color = attrs['color']
+
+            if new_tags:
+                session.add_all(new_tags)
+
+            session.commit()
+
+        if config.tag_files:
+            from wrolpi.files.models import FileGroup
+
+            need_commit = False
+
+            # Get all Tags again because new ones may exist.
+            tags_by_name: Dict[str, Tag] = {i.name: i for i in session.query(Tag)}
+
+            media_directory = get_media_directory()
+            primary_paths = [str(media_directory / i[1]) for i in config.tag_files]
+            file_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(primary_paths))
+            file_groups_by_primary_path = {i.primary_path: i for i in file_groups}
+            file_group_ids = [i.id for i in file_groups]
+            # Get all TagFiles referencing the FileGroups.
+            tag_files = session.query(TagFile).filter(TagFile.file_group_id.in_(file_group_ids))
+            tag_files = [(i.tag_id, i.file_group_id) for i in tag_files]
+
+            for tag_name, primary_path in config.tag_files:
+                tag: Tag = tags_by_name.get(tag_name)
+                # Paths are absolute in the DB, relative in config.
+                absolute_path = media_directory / primary_path
+                file_group: FileGroup = file_groups_by_primary_path.get(absolute_path)
+                if tag and file_group:
+                    if (tag.id, file_group.id) not in tag_files:
+                        # This FileGroup has not been tagged with the Tag, add it.
+                        tag.add_file_group_tag(file_group, session)
+                        need_commit = True
+                elif not file_group:
+                    logger.warning(f'Cannot find FileGroup for {repr(str(primary_path))}')
+                elif not tag:
+                    logger.warning(f'Cannot find Tag for {repr(str(primary_path))}')
+
+            if need_commit:
+                session.commit()
+    except Exception as e:
+        logger.error(f'Failed to import tags config', exc_info=e)
+        if PYTEST:
+            raise

wrolpi/test/test_tags.py

@@ -2,6 +2,7 @@ import json
 from http import HTTPStatus
 
 import pytest
+import yaml
 
 from wrolpi import tags
 from wrolpi.files import lib as files_lib
@@ -17,7 +18,7 @@ async def test_tags_file_group_json(test_session, make_files_structure, tag_fact
     await files_lib.refresh_files()
     file_group: FileGroup = test_session.query(FileGroup).one()
 
-    tag_one.add_tag(file_group)
+    tag_one.add_file_group_tag(file_group)
     assert file_group.__json__()['tags'] == ['one']
 
     file_group.add_tag(tag_two)
@@ -37,13 +38,13 @@ async def test_tags_file_group(test_session, make_files_structure, tag_factory):
     tag_one = tag_factory()
     assert len(tag_one.tag_files) == 0
 
-    tag_one.add_tag(video_group)
+    tag_one.add_file_group_tag(video_group)
     test_session.commit()
     assert len(tag_one.tag_files) == 1
     assert sorted([i.tag.name for i in video_group.tag_files]) == ['one', ]
 
     tag_two = tag_factory()
-    tag_two.add_tag(video_group)
+    tag_two.add_file_group_tag(video_group)
     test_session.commit()
     assert len(tag_one.tag_files) == 1
     assert len(tag_two.tag_files) == 1
@@ -166,3 +167,46 @@ async def test_delete_tagged_file(test_session, example_pdf, tag_factory):
 
     pdf.delete()
     test_session.commit()
+
+
+def test_import_empty_tags_config(test_session, test_tags_config):
+    """An empty config can be imported."""
+    tags.import_tags_config()
+
+
+@pytest.mark.asyncio()
+async def test_import_tags_config(test_session, test_directory, test_tags_config, example_singlefile):
+    with test_tags_config.open('wt') as fh:
+        data = dict(
+            tags={'Foo': {'color': '#123456'}},
+            tag_files=[
+                ('Foo', str(example_singlefile.relative_to(test_directory))),
+                ('Foo', 'does not exist'),
+            ],
+        )
+        yaml.dump(data, fh)
+
+    assert test_session.query(tags.Tag).count() == 0, 'Expected no Tags.'
+    assert test_session.query(tags.TagFile).count() == 0, 'Expected no FileGroups to be tagged.'
+
+    # Re-initialize config with the data we just saved.
+    tags.get_tags_config().initialize()
+
+    FileGroup.from_paths(test_session, example_singlefile)
+    test_session.commit()
+
+    tags.import_tags_config(test_session)
+
+    # 'Foo' Tag was created.
+    assert test_session.query(tags.Tag).count() == 1
+
+    file_group: FileGroup = test_session.query(FileGroup).one()
+    assert file_group.tag_files, 'File was not tagged during import.'
+
+    # Importing again does not change count.
+    await files_lib.refresh_files()
+    tags.import_tags_config(test_session)
+    assert test_session.query(tags.Tag).count() == 1
+    assert test_session.query(tags.TagFile).count() == 1
+    # Example file, and the config file.
+    assert test_session.query(FileGroup).count() == 2