files: SQL FileGroup compare, stem column, drop unused discovery path (#713)

* files: SQL FileGroup compare, stem column, drop unused discovery path

Refresh walked the tree twice (find, then stat every unchanged group),
and a two-directory refresh scanned the whole media directory.  Compare
now diffs in SQL on a TEMP table that already has mtime, accepts multiple
roots, and groups by a stored stem that is versioned so a suffix-rule
change cannot delete-and-reinsert tagged files.

Deletes refresh_discover_paths and /api/files/refresh_progress, which
nothing in production called.  Video/doc/zim modelers use the same
(no domain row OR indexed != True) gate.

* files: keep stem current on reorganize; NUL-safe find parse

Reorganize wrote the new primary_path but not stem, so the next refresh
treated the FileGroup as deleted+new and dropped tags.  GNU find now
emits NUL-delimited records so a tab or newline in a filename cannot
abort a global refresh.  Stem backfill skips the table scan once the
algorithm version matches, and runs off the event loop.  An empty roots
list is a global refresh, not invalid SQL.

modules/archive/__init__.py

@@ -369,7 +369,10 @@ async def archive_modeler(progress_callback: Callable[[int], None] = None):
         with get_db_session(commit=True) as session:
             results = session.query(FileGroup, Archive) \
                 .filter(
-                # Get all groups that contain an HTML file that have not been indexed.
+                # Archive shares text/html with generic HTML.  Unlike video/doc/zim, do NOT
+                # select Archive.id IS NULL: that would re-probe every leftover HTML file
+                # every refresh.  Non-SingleFile HTML stays indexed=0 so apply_indexers
+                # can claim it.
                 FileGroup.indexed != True,
                 FileGroup.mimetype == 'text/html',
             ).filter(not_(FileGroup.id.in_(list(invalid_archives)))) \

modules/docs/__init__.py

@@ -60,7 +60,7 @@ async def doc_modeler(progress_callback: Callable[[int], None] = None):
                 # permanently unmodeled and invisible in /api/docs.
                 or_(
                     Doc.id.is_(None),
-                    FileGroup.indexed == False,
+                    FileGroup.indexed != True,
                 ),
             )
             if failed_ids:

modules/docs/test/test_doc_modeler.py

@@ -10,7 +10,6 @@ from sqlalchemy.exc import OperationalError
 
 from modules.docs import doc_modeler
 from modules.docs.models import Doc, DocSection
-from wrolpi.dates import now
 from wrolpi.files import lib as files_lib
 from wrolpi.files.models import FileGroup
 from wrolpi.files.worker import file_worker
@@ -105,7 +104,7 @@ async def test_doc_modeler_replaces_sections(async_client, test_session, test_di
 @pytest.mark.asyncio
 async def test_doc_do_model_sets_model(test_session, example_pdf):
     """Doc.do_model sets file_group.model so the FileGroup is recognized as a doc."""
-    files_lib._upsert_files([example_pdf], now())
+    files_lib._upsert_files([example_pdf])
     file_group = test_session.query(FileGroup).one()
     # Reset to simulate a newly discovered file.
     file_group.model = None
@@ -394,7 +393,7 @@ async def test_doc_modeler_pdf_max_size(test_session, example_pdf):
     """The contents of a large PDF are not indexed."""
     example_pdf.write_bytes(example_pdf.read_bytes() * 5000)
 
-    files_lib._upsert_files([example_pdf], now())
+    files_lib._upsert_files([example_pdf])
 
     await doc_modeler()
     file_group = test_session.query(FileGroup).one()
@@ -407,7 +406,7 @@ async def test_doc_modeler_pdf_poster(test_session, example_pdf):
     poster_path = example_pdf.with_suffix('.jpg')
     Image.new('RGB', (25, 25), color='grey').save(poster_path)
 
-    files_lib._upsert_files([example_pdf, poster_path], now())
+    files_lib._upsert_files([example_pdf, poster_path])
 
     await doc_modeler()
     file_group = test_session.query(FileGroup).one()

modules/videos/__init__.py

@@ -57,7 +57,8 @@ async def video_modeler(progress_callback: Callable[[int], None] = None):
                 .filter(
                 or_(FileGroup.mimetype.like('video/%'), FileGroup.mimetype.like('audio/%')),
                 FileGroup.mimetype.notin_(AUDIO_PLAYLIST_MIMETYPES),
-                # Also model indexed FileGroups that never got a Video row.
+                # Exclusive-mimetype modeler: claim unmodeled rows even if apply_indexers
+                # already set indexed=True, and re-model when files changed (indexed != True).
                 or_(Video.id.is_(None), FileGroup.indexed != True),
             )
             if failed_ids:

modules/videos/schema.py

@@ -122,7 +122,6 @@ class Video:
     video_path: str
     name: str
     caption_path: str
-    idempotency: str
     info_json_path: str
     channel_id: int
 
@@ -138,7 +137,6 @@ class VideoWithChannel:
     video_path: str
     name: str
     caption_path: str
-    idempotency: str
     info_json_path: str
     channel_id: int
 

modules/zim/lib.py

@@ -8,6 +8,7 @@ from typing import Callable, List, Tuple, Dict
 
 import cachetools.func
 from libzim import Entry
+from sqlalchemy import or_
 from sqlalchemy.orm import Session
 from sqlalchemy.orm.exc import NoResultFound  # noqa
 
@@ -75,15 +76,20 @@ def model_zim(file_group: FileGroup, session: Session) -> Zim:
 @register_modeler
 async def zim_modeler(progress_callback: Callable[[int], None] = None):
     total_processed = 0
+    failed_ids: set = set()
     while True:
         with get_db_session(commit=True) as session:
-            file_groups = session.query(FileGroup, Zim) \
-                .filter(FileGroup.indexed != True,  # noqa
-                        FileGroup.primary_path.ilike('%.zim'),
-                        ) \
+            query = session.query(FileGroup, Zim) \
                 .outerjoin(Zim, Zim.file_group_id == FileGroup.id) \
-                .limit(10)
-            file_groups: List[Tuple[FileGroup, Zim]] = list(file_groups)
+                .filter(FileGroup.primary_path.ilike('%.zim')) \
+                .filter(
+                # Exclusive-mimetype modeler: claim unmodeled rows even if apply_indexers
+                # already set indexed=True, and re-model when files changed.
+                or_(Zim.id.is_(None), FileGroup.indexed != True),
+            )
+            if failed_ids:
+                query = query.filter(FileGroup.id.notin_(failed_ids))
+            file_groups: List[Tuple[FileGroup, Zim]] = list(query.limit(10))
 
             processed = 0
             for file_group, zim in file_groups:
@@ -101,6 +107,7 @@ async def zim_modeler(progress_callback: Callable[[int], None] = None):
                         file_group.indexed = True
                         file_group.model = 'zim'
                 except Exception as e:
+                    failed_ids.add(file_group.id)
                     if PYTEST:
                         raise
                     logger.error(f'Unable to model Zim {zim_id=} {file_group.primary_path=}', exc_info=e)