mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
files: SQL FileGroup compare, stem column, drop unused discovery path
Refresh walked the tree twice (find, then stat every unchanged group), and a two-directory refresh scanned the whole media directory. Compare now diffs in SQL on a TEMP table that already has mtime, accepts multiple roots, and groups by a stored stem that is versioned so a suffix-rule change cannot delete-and-reinsert tagged files. Deletes refresh_discover_paths and /api/files/refresh_progress, which nothing in production called. Video/doc/zim modelers use the same (no domain row OR indexed != True) gate.
This commit is contained in:
parent
b37ad92563
commit
bd15b85c86
18 changed files with 634 additions and 507 deletions
|
|
@ -60,7 +60,7 @@ async def doc_modeler(progress_callback: Callable[[int], None] = None):
|
|||
# permanently unmodeled and invisible in /api/docs.
|
||||
or_(
|
||||
Doc.id.is_(None),
|
||||
FileGroup.indexed == False,
|
||||
FileGroup.indexed != True,
|
||||
),
|
||||
)
|
||||
if failed_ids:
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ from sqlalchemy.exc import OperationalError
|
|||
|
||||
from modules.docs import doc_modeler
|
||||
from modules.docs.models import Doc, DocSection
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.files.worker import file_worker
|
||||
|
|
@ -105,7 +104,7 @@ async def test_doc_modeler_replaces_sections(async_client, test_session, test_di
|
|||
@pytest.mark.asyncio
|
||||
async def test_doc_do_model_sets_model(test_session, example_pdf):
|
||||
"""Doc.do_model sets file_group.model so the FileGroup is recognized as a doc."""
|
||||
files_lib._upsert_files([example_pdf], now())
|
||||
files_lib._upsert_files([example_pdf])
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
# Reset to simulate a newly discovered file.
|
||||
file_group.model = None
|
||||
|
|
@ -394,7 +393,7 @@ async def test_doc_modeler_pdf_max_size(test_session, example_pdf):
|
|||
"""The contents of a large PDF are not indexed."""
|
||||
example_pdf.write_bytes(example_pdf.read_bytes() * 5000)
|
||||
|
||||
files_lib._upsert_files([example_pdf], now())
|
||||
files_lib._upsert_files([example_pdf])
|
||||
|
||||
await doc_modeler()
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
|
|
@ -407,7 +406,7 @@ async def test_doc_modeler_pdf_poster(test_session, example_pdf):
|
|||
poster_path = example_pdf.with_suffix('.jpg')
|
||||
Image.new('RGB', (25, 25), color='grey').save(poster_path)
|
||||
|
||||
files_lib._upsert_files([example_pdf, poster_path], now())
|
||||
files_lib._upsert_files([example_pdf, poster_path])
|
||||
|
||||
await doc_modeler()
|
||||
file_group = test_session.query(FileGroup).one()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue