Models like Archive/Video/Doc can be deleted, even if they are tagged. User is shown a warning and must confirm deletion of tagged files.

modules/archive/api.py

@@ -46,15 +46,21 @@ async def get_archive(request: Request, file_group_id: int):
 
 @archive_bp.delete('/<file_group_ids:int>', name='archive_delete_one')
 @archive_bp.delete('/<file_group_ids:[0-9,]+>', name='archive_delete_many')
-@openapi.description('Delete Archives by FileGroup IDs')
+@openapi.description('Delete Archives by FileGroup IDs.  Pass ?force=true to delete even tagged FileGroups.')
 @openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
 @wrol_mode_check
-async def delete_archive(_: Request, file_group_ids: str):
+async def delete_archive(request: Request, file_group_ids: str):
     try:
         file_group_ids = [int(i) for i in str(file_group_ids).split(',')]
     except ValueError:
         raise ValidationError('Could not parse file group ids')
-    lib.delete_archives_by_file_group_ids(*file_group_ids)
+    force = request.args.get('force', '').lower() == 'true'
+    tagged = lib.delete_archives_by_file_group_ids(*file_group_ids, force=force)
+    if tagged:
+        return json_response(
+            {'code': 'FILE_GROUP_IS_TAGGED', 'file_groups': tagged},
+            HTTPStatus.CONFLICT,
+        )
     return response.empty()
 
 

modules/archive/lib.py

@@ -27,10 +27,10 @@ from wrolpi.dates import now, Seconds
 from wrolpi.db import get_db_session, get_db_curs
 from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
 from wrolpi.events import Events
-from wrolpi.files.lib import handle_file_group_search_results
+from wrolpi.files.lib import handle_file_group_search_results, get_tagged_file_groups_by_ids
 from wrolpi.files.models import FileGroup
 from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
-from wrolpi.tags import tag_append_sub_select_where
+from wrolpi.tags import tag_append_sub_select_where, save_tags_config
 from wrolpi.vars import PYTEST, DOCKERIZED
 
 logger = logger.getChild(__name__)
@@ -1125,15 +1125,28 @@ def get_archive_by_file_group_id(session: Session, file_group_id: int, skip_view
     return archive
 
 
-def delete_archives_by_file_group_ids(*file_group_ids: List[int]):
-    """Delete Archives by their FileGroup IDs and all of their files."""
+def delete_archives_by_file_group_ids(*file_group_ids: int, force: bool = False) -> Optional[List[dict]]:
+    """Delete Archives by their FileGroup IDs and all of their files.
+
+    If any FileGroup is tagged and force is False, returns the list of serialized tagged FileGroups
+    without deleting anything.  If force is True, deletes even tagged files and re-saves the tags
+    config so `tags.yaml` reflects the removed TagFiles.
+    """
     with get_db_session(commit=True) as session:
         archives: List[Archive] = list(session.query(Archive).filter(Archive.file_group_id.in_(file_group_ids)))
         if not archives:
             raise UnknownArchive(f'Unknown Archives with FileGroup IDs: {", ".join(map(str, file_group_ids))}')
 
+        tagged = get_tagged_file_groups_by_ids(session, [a.file_group_id for a in archives])
+        if tagged and not force:
+            return tagged
+
         for archive in archives:
-            archive.delete()
+            archive.delete(force=force)
+
+    if tagged:
+        save_tags_config.activate_switch()
+    return None
 
 
 def archive_strftime(dt: datetime) -> str:

modules/archive/models.py

@@ -140,8 +140,8 @@ class Archive(Base, ModelHelper):
 
         return self.file_group.published_datetime > other.file_group.published_datetime
 
-    def delete(self):
-        self.file_group.delete()
+    def delete(self, force: bool = False):
+        self.file_group.delete(force=force)
 
         session = Session.object_session(self)
         collection = self.collection

modules/archive/test/test_api.py

@@ -510,3 +510,68 @@ async def test_archive_browsers_endpoint_dockerized(async_client):
         assert response.json['browsers'] == []
         assert response.json['available'] is False
         assert 'message' in response.json
+
+
+@pytest.mark.asyncio
+async def test_delete_archive_untagged(async_client, test_session, archive_factory):
+    """An untagged Archive can be deleted via DELETE /api/archive/<id> without force."""
+    archive = archive_factory('example.com', 'https://example.com/1')
+    test_session.commit()
+    paths = list(archive.my_paths())
+    assert paths and all(p.is_file() for p in paths)
+
+    request, response = await async_client.delete(f'/api/archive/{archive.file_group_id}')
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    assert all(not p.exists() for p in paths)
+    assert test_session.query(Archive).count() == 0
+
+
+@pytest.mark.asyncio
+async def test_delete_archive_tagged_requires_force(async_client, test_session, archive_factory, tag_factory,
+                                                   await_switches):
+    """A tagged Archive returns 409 with FILE_GROUP_IS_TAGGED until force=true is set."""
+    tag = await tag_factory()
+    archive = archive_factory('example.com', 'https://example.com/1', tag_names=[tag.name])
+    test_session.commit()
+    paths = list(archive.my_paths())
+
+    request, response = await async_client.delete(f'/api/archive/{archive.file_group_id}')
+    assert response.status_code == HTTPStatus.CONFLICT
+    assert response.json['code'] == 'FILE_GROUP_IS_TAGGED'
+    assert len(response.json['file_groups']) == 1
+    returned = response.json['file_groups'][0]
+    assert returned['id'] == archive.file_group_id
+    assert tag.name in returned['tags']
+    # Nothing was deleted.
+    assert all(p.is_file() for p in paths)
+    assert test_session.query(Archive).count() == 1
+
+    # With force=true, the Archive is deleted.
+    request, response = await async_client.delete(f'/api/archive/{archive.file_group_id}?force=true')
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    assert all(not p.exists() for p in paths)
+    assert test_session.query(Archive).count() == 0
+    await await_switches()
+
+
+@pytest.mark.asyncio
+async def test_delete_archive_atomic_when_some_tagged(async_client, test_session, archive_factory, tag_factory):
+    """When deleting multiple Archives and any are tagged, NONE are deleted until force=true."""
+    tag = await tag_factory()
+    untagged = archive_factory('example.com', 'https://example.com/1')
+    tagged = archive_factory('example.com', 'https://example.com/2', tag_names=[tag.name])
+    test_session.commit()
+    untagged_paths = list(untagged.my_paths())
+    tagged_paths = list(tagged.my_paths())
+
+    ids = f'{untagged.file_group_id},{tagged.file_group_id}'
+    request, response = await async_client.delete(f'/api/archive/{ids}')
+    assert response.status_code == HTTPStatus.CONFLICT
+    assert response.json['code'] == 'FILE_GROUP_IS_TAGGED'
+    # Only the tagged FileGroup is in the response.
+    returned_ids = {fg['id'] for fg in response.json['file_groups']}
+    assert returned_ids == {tagged.file_group_id}
+    # Nothing was deleted.
+    assert all(p.is_file() for p in untagged_paths)
+    assert all(p.is_file() for p in tagged_paths)
+    assert test_session.query(Archive).count() == 2

modules/docs/api.py

@@ -36,15 +36,21 @@ async def get_doc(request: Request, file_group_id: int):
 
 @docs_bp.delete('/<file_group_ids:int>', name='doc_delete_one')
 @docs_bp.delete('/<file_group_ids:[0-9,]+>', name='doc_delete_many')
-@openapi.description('Delete Docs by FileGroup IDs')
+@openapi.description('Delete Docs by FileGroup IDs.  Pass ?force=true to delete even tagged FileGroups.')
 @openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
 @wrol_mode_check
-async def delete_docs(_: Request, file_group_ids: str):
+async def delete_docs(request: Request, file_group_ids: str):
     try:
         file_group_ids = [int(i) for i in str(file_group_ids).split(',')]
     except ValueError:
         raise ValidationError('Could not parse file group ids')
-    _delete_docs(*file_group_ids)
+    force = request.args.get('force', '').lower() == 'true'
+    tagged = _delete_docs(*file_group_ids, force=force)
+    if tagged:
+        return json_response(
+            {'code': 'FILE_GROUP_IS_TAGGED', 'file_groups': tagged},
+            HTTPStatus.CONFLICT,
+        )
     return response.empty()
 
 

modules/docs/lib.py

@@ -11,7 +11,8 @@ from wrolpi.collections.models import Collection
 from wrolpi.collections.types import collection_type_registry
 from wrolpi.db import get_db_session
 from wrolpi.errors import ValidationError
-from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype
+from wrolpi.files.lib import split_path_stem_and_suffix, get_mimetype, get_tagged_file_groups_by_ids
+from wrolpi.tags import save_tags_config
 
 logger = logging.getLogger(__name__)
 
@@ -346,13 +347,33 @@ def _get_doc(session, file_group_id: int):
     return doc
 
 
-def _delete_docs(*file_group_ids):
+def _delete_docs(*file_group_ids, force: bool = False) -> Optional[List[dict]]:
+    """Delete Docs by their FileGroup IDs and all of their files.
+
+    If any FileGroup is tagged and force is False, returns the list of serialized tagged FileGroups
+    without deleting anything.  If force is True, deletes even tagged files and re-saves the tags
+    config so `tags.yaml` reflects the removed TagFiles.
+    """
     from .models import Doc
     with get_db_session(commit=True) as session:
-        for file_group_id in file_group_ids:
-            doc = session.query(Doc).filter_by(file_group_id=file_group_id).one_or_none()
-            if doc:
-                session.delete(doc)
+        docs = list(session.query(Doc).filter(Doc.file_group_id.in_(file_group_ids)))
+        if not docs:
+            raise ValidationError(
+                f'Could not find Docs to delete with file_group_ids: {", ".join(map(str, file_group_ids))}'
+            )
+
+        tagged = get_tagged_file_groups_by_ids(session, [d.file_group_id for d in docs])
+        if tagged and not force:
+            return tagged
+
+        # Delete the FileGroup, which removes the files on disk and cascades the Doc row via
+        # the ON DELETE CASCADE foreign key on Doc.file_group_id.
+        for doc in docs:
+            doc.file_group.delete(force=force)
+
+    if tagged:
+        save_tags_config.activate_switch()
+    return None
 
 
 def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None,

modules/docs/test/test_api.py

@@ -77,14 +77,74 @@ async def test_docs_search_by_tag(async_client, test_session, doc_factory, tag_f
 
 @pytest.mark.asyncio
 async def test_delete_doc(async_client, test_session, test_directory, example_epub, refresh_files):
-    """Docs can be deleted."""
+    """Docs can be deleted, including their on-disk files and FileGroup row."""
+    from wrolpi.files.models import FileGroup
     await refresh_files()
 
     doc = test_session.query(Doc).first()
     assert doc
     file_group_id = doc.file_group_id
+    paths = list(doc.file_group.my_paths())
+    assert paths and all(p.is_file() for p in paths)
 
     request, response = await async_client.delete(f'/api/docs/{file_group_id}')
     assert response.status_code == HTTPStatus.NO_CONTENT
 
+    # Doc, FileGroup, and on-disk files all removed.
     assert test_session.query(Doc).filter_by(file_group_id=file_group_id).one_or_none() is None
+    assert test_session.query(FileGroup).filter_by(id=file_group_id).one_or_none() is None
+    assert all(not p.exists() for p in paths)
+
+
+@pytest.mark.asyncio
+async def test_delete_doc_tagged_requires_force(async_client, test_session, doc_factory, tag_factory,
+                                                await_switches):
+    """DELETE /api/docs/<id> returns 409 for tagged docs until ?force=true is passed."""
+    from wrolpi.files.models import FileGroup
+    tag = await tag_factory()
+    doc = doc_factory()
+    doc.file_group.add_tag(test_session, tag.name)
+    test_session.commit()
+    file_group_id = doc.file_group_id
+    paths = list(doc.file_group.my_paths())
+
+    request, response = await async_client.delete(f'/api/docs/{file_group_id}')
+    assert response.status_code == HTTPStatus.CONFLICT
+    assert response.json['code'] == 'FILE_GROUP_IS_TAGGED'
+    assert len(response.json['file_groups']) == 1
+    assert response.json['file_groups'][0]['id'] == file_group_id
+    assert tag.name in response.json['file_groups'][0]['tags']
+    # Nothing was deleted.
+    assert all(p.is_file() for p in paths)
+    assert test_session.query(Doc).count() == 1
+
+    # With force=true, the Doc is deleted.
+    request, response = await async_client.delete(f'/api/docs/{file_group_id}?force=true')
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    assert all(not p.exists() for p in paths)
+    assert test_session.query(Doc).count() == 0
+    assert test_session.query(FileGroup).filter_by(id=file_group_id).one_or_none() is None
+    await await_switches()
+
+
+@pytest.mark.asyncio
+async def test_delete_doc_atomic_when_some_tagged(async_client, test_session, doc_factory, tag_factory):
+    """When deleting multiple Docs and any are tagged, NONE are deleted until ?force=true."""
+    untagged = doc_factory()
+    tagged = doc_factory()
+    tag = await tag_factory()
+    tagged.file_group.add_tag(test_session, tag.name)
+    test_session.commit()
+    untagged_paths = list(untagged.file_group.my_paths())
+    tagged_paths = list(tagged.file_group.my_paths())
+
+    ids = f'{untagged.file_group_id},{tagged.file_group_id}'
+    request, response = await async_client.delete(f'/api/docs/{ids}')
+    assert response.status_code == HTTPStatus.CONFLICT
+    assert response.json['code'] == 'FILE_GROUP_IS_TAGGED'
+    returned_ids = {fg['id'] for fg in response.json['file_groups']}
+    assert returned_ids == {tagged.file_group_id}
+    # Nothing was deleted.
+    assert all(p.is_file() for p in untagged_paths)
+    assert all(p.is_file() for p in tagged_paths)
+    assert test_session.query(Doc).count() == 2

modules/videos/models.py

@@ -90,10 +90,10 @@ class Video(ModelHelper, Base):
         )
         return d
 
-    def delete(self, add_to_skip_list: bool = True):
+    def delete(self, add_to_skip_list: bool = True, force: bool = False):
         """Remove all files and File records related to this video.  Delete this Video record.
         Add it to it's Channel's skip list."""
-        self.file_group.delete(add_to_skip_list=add_to_skip_list)
+        self.file_group.delete(add_to_skip_list=add_to_skip_list, force=force)
         session = Session.object_session(self)
         session.delete(self)
 

modules/videos/test/test_video.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timezone
+from http import HTTPStatus
 
 import pytest
 
@@ -45,6 +46,59 @@ async def test_delete_video_with_tag(async_client, test_session, video_factory,
     assert test_session.query(FileGroup).count() == 1
 
 
+@pytest.mark.asyncio
+async def test_delete_video_api_tagged_requires_force(async_client, test_session, video_factory, tag_factory,
+                                                     await_switches):
+    """DELETE /api/videos/<id> returns 409 for tagged videos until ?force=true is passed."""
+    video = video_factory(with_video_file=True, with_poster_ext='png')
+    tag = await tag_factory()
+    video.add_tag(test_session, tag.name)
+    test_session.commit()
+    file_group_id = video.file_group_id
+    paths = list(video.file_group.my_paths())
+
+    request, response = await async_client.delete(f'/api/videos/{file_group_id}')
+    assert response.status_code == HTTPStatus.CONFLICT
+    assert response.json['code'] == 'FILE_GROUP_IS_TAGGED'
+    assert len(response.json['file_groups']) == 1
+    assert response.json['file_groups'][0]['id'] == file_group_id
+    assert tag.name in response.json['file_groups'][0]['tags']
+    # Nothing was deleted.
+    assert all(p.is_file() for p in paths)
+    assert test_session.query(Video).count() == 1
+
+    # With force=true, the Video is deleted.
+    request, response = await async_client.delete(f'/api/videos/{file_group_id}?force=true')
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    assert all(not p.exists() for p in paths)
+    assert test_session.query(Video).count() == 0
+    assert test_session.query(FileGroup).count() == 0
+    await await_switches()
+
+
+@pytest.mark.asyncio
+async def test_delete_video_api_atomic_when_some_tagged(async_client, test_session, video_factory, tag_factory):
+    """When deleting multiple Videos and any are tagged, NONE are deleted until ?force=true."""
+    untagged = video_factory(with_video_file=True)
+    tagged = video_factory(with_video_file=True)
+    tag = await tag_factory()
+    tagged.add_tag(test_session, tag.name)
+    test_session.commit()
+    untagged_paths = list(untagged.file_group.my_paths())
+    tagged_paths = list(tagged.file_group.my_paths())
+
+    ids = f'{untagged.file_group_id},{tagged.file_group_id}'
+    request, response = await async_client.delete(f'/api/videos/{ids}')
+    assert response.status_code == HTTPStatus.CONFLICT
+    assert response.json['code'] == 'FILE_GROUP_IS_TAGGED'
+    returned_ids = {fg['id'] for fg in response.json['file_groups']}
+    assert returned_ids == {tagged.file_group_id}
+    # Nothing was deleted.
+    assert all(p.is_file() for p in untagged_paths)
+    assert all(p.is_file() for p in tagged_paths)
+    assert test_session.query(Video).count() == 2
+
+
 @pytest.mark.asyncio
 async def test_video_channel_refresh(async_client, test_session, test_directory, channel_factory, video_factory,
                                      refresh_files):

modules/videos/video/api.py

@@ -75,7 +75,7 @@ async def search_videos(_: Request, body: schema.VideoSearchRequest):
 
 @video_bp.delete('/<file_group_ids:[0-9,]+>', name='Video Delete Many')
 @video_bp.delete('/<file_group_ids:int>', name='Video Delete One')
-@openapi.description('Delete videos.')
+@openapi.description('Delete videos.  Pass ?force=true to delete even tagged FileGroups.')
 @openapi.response(HTTPStatus.NO_CONTENT)
 @openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
 @wrol_mode_check
@@ -86,7 +86,13 @@ async def video_delete(request: Request, file_group_ids: str):
         raise ValidationError('Unable to parse file group ids')
 
     session = request.ctx.session
-    lib.delete_videos(session, *file_group_ids)
+    force = request.args.get('force', '').lower() == 'true'
+    tagged = lib.delete_videos(session, *file_group_ids, force=force)
+    if tagged:
+        return json_response(
+            {'code': 'FILE_GROUP_IS_TAGGED', 'file_groups': tagged},
+            HTTPStatus.CONFLICT,
+        )
 
     save_channels_config.activate_switch()
     Events.send_deleted(f'Deleted {len(file_group_ids)} videos')

modules/videos/video/lib.py

@@ -12,9 +12,9 @@ from wrolpi.common import logger, limit_concurrent, wrol_mode_check
 from wrolpi.dates import now
 from wrolpi.db import get_db_session
 from wrolpi.downloader import download_manager
-from wrolpi.files.lib import handle_file_group_search_results
+from wrolpi.files.lib import handle_file_group_search_results, get_tagged_file_groups_by_ids
 from wrolpi.files.models import FileGroup
-from wrolpi.tags import tag_append_sub_select_where
+from wrolpi.tags import tag_append_sub_select_where, save_tags_config
 from wrolpi.vars import VIDEO_COMMENTS_FETCH_COUNT, YTDLP_CACHE_DIR
 from ..cookies import cookies_unlocked, cookies_for_download
 from ..errors import UnknownVideo
@@ -162,19 +162,33 @@ def search_videos(
     return results, total
 
 
-def delete_videos(session: Session, *file_group_ids: int):
+def delete_videos(session: Session, *file_group_ids: int, force: bool = False) -> Optional[List[dict]]:
+    """Delete Videos by their FileGroup IDs and all of their files.
+
+    If any FileGroup is tagged and force is False, returns the list of serialized tagged FileGroups
+    without deleting anything.  If force is True, deletes even tagged files and re-saves the tags
+    config so `tags.yaml` reflects the removed TagFiles.
+    """
     videos = list(session.query(Video).filter(Video.file_group_id.in_(file_group_ids)))
     if not videos:
         raise UnknownVideo('Could not find videos to delete')
 
+    tagged = get_tagged_file_groups_by_ids(session, [v.file_group_id for v in videos])
+    if tagged and not force:
+        return tagged
+
     logger.warning(f'Deleting {len(videos)} videos')
     # Get all URLs, skip them once (so config isn't saved multiple times).
     urls = list(filter(None, [i.file_group.url for i in videos]))
     download_manager.add_to_skip_list(*urls)
     for video in videos:
-        video.delete()
+        video.delete(force=force)
     session.commit()
 
+    if tagged:
+        save_tags_config.activate_switch()
+    return None
+
 
 def download_video_info_json(url: str) -> dict:
     """Download video info JSON, using encrypted cookies for authentication if available."""