files: SQL FileGroup compare, stem column, drop unused discovery path

Refresh walked the tree twice (find, then stat every unchanged group),
and a two-directory refresh scanned the whole media directory.  Compare
now diffs in SQL on a TEMP table that already has mtime, accepts multiple
roots, and groups by a stored stem that is versioned so a suffix-rule
change cannot delete-and-reinsert tagged files.

Deletes refresh_discover_paths and /api/files/refresh_progress, which
nothing in production called.  Video/doc/zim modelers use the same
(no domain row OR indexed != True) gate.
This commit is contained in:
Roland 2026-09-16 20:20:42 -06:00
parent b37ad92563
commit bd15b85c86
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
18 changed files with 634 additions and 507 deletions

View file

@ -60,7 +60,7 @@ async def doc_modeler(progress_callback: Callable[[int], None] = None):
# permanently unmodeled and invisible in /api/docs.
or_(
Doc.id.is_(None),
FileGroup.indexed == False,
FileGroup.indexed != True,
),
)
if failed_ids:

View file

@ -10,7 +10,6 @@ from sqlalchemy.exc import OperationalError
from modules.docs import doc_modeler
from modules.docs.models import Doc, DocSection
from wrolpi.dates import now
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.files.worker import file_worker
@ -105,7 +104,7 @@ async def test_doc_modeler_replaces_sections(async_client, test_session, test_di
@pytest.mark.asyncio
async def test_doc_do_model_sets_model(test_session, example_pdf):
"""Doc.do_model sets file_group.model so the FileGroup is recognized as a doc."""
files_lib._upsert_files([example_pdf], now())
files_lib._upsert_files([example_pdf])
file_group = test_session.query(FileGroup).one()
# Reset to simulate a newly discovered file.
file_group.model = None
@ -394,7 +393,7 @@ async def test_doc_modeler_pdf_max_size(test_session, example_pdf):
"""The contents of a large PDF are not indexed."""
example_pdf.write_bytes(example_pdf.read_bytes() * 5000)
files_lib._upsert_files([example_pdf], now())
files_lib._upsert_files([example_pdf])
await doc_modeler()
file_group = test_session.query(FileGroup).one()
@ -407,7 +406,7 @@ async def test_doc_modeler_pdf_poster(test_session, example_pdf):
poster_path = example_pdf.with_suffix('.jpg')
Image.new('RGB', (25, 25), color='grey').save(poster_path)
files_lib._upsert_files([example_pdf, poster_path], now())
files_lib._upsert_files([example_pdf, poster_path])
await doc_modeler()
file_group = test_session.query(FileGroup).one()