From 943b3a0b60c702727a422a17833653b4e68f7955 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 12 Jul 2026 15:45:27 -0600 Subject: [PATCH] SQLite: keep large collection/file queries under the expression-tree depth limit. Collection.batch_from_config OR'd a (name AND kind) pair per collection, so importing more than ~1000 collections (e.g. a 1,670-domain domains.yaml) failed with "Expression tree is too large (maximum depth 1000)". Query a flat name IN (...) and match (name, kind) in Python instead. get_bulk_tag_preview had the same unbounded pattern (an OR'd LIKE per stem); chunk the query at 500 stems. Found upgrading 10.0.0.9 (decay). Co-Authored-By: Claude Fable 5 --- wrolpi/collections/models.py | 16 ++++++------- .../collections/test/test_domains_config.py | 24 +++++++++++++++++++ wrolpi/files/lib.py | 10 +++++--- wrolpi/files/test/test_lib.py | 12 ++++++++++ 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/wrolpi/collections/models.py b/wrolpi/collections/models.py index cca3d7e8..87ec1701 100644 --- a/wrolpi/collections/models.py +++ b/wrolpi/collections/models.py @@ -398,16 +398,16 @@ class Collection(ModelHelper, Base): # Step 3: Pre-fetch existing collections by (name, kind) for ALL collections # This handles cases where directory changed or directory-less collections - name_kind_pairs = [(d['name'], d.get('kind', 'channel')) for d in data_list] + # A flat IN on name stays within SQLite's expression-tree depth limit (an OR'd + # (name AND kind) pair per collection exceeds it past ~1000 collections); the + # exact (name, kind) match is applied in Python. + name_kind_pairs = {(d['name'], d.get('kind', 'channel')) for d in data_list} existing_by_name_kind = {} if name_kind_pairs: - from sqlalchemy import and_, or_ - conditions = [ - and_(Collection.name == name, Collection.kind == kind) - for name, kind in name_kind_pairs - ] - existing = session.query(Collection).filter(or_(*conditions)).all() - existing_by_name_kind = {(c.name, c.kind): c for c in existing} + names = {name for name, _ in name_kind_pairs} + existing = session.query(Collection).filter(Collection.name.in_(names)).all() + existing_by_name_kind = {(c.name, c.kind): c for c in existing + if (c.name, c.kind) in name_kind_pairs} # Step 4: Pre-fetch all tags in ONE query tag_names = {d['tag_name'] for d in data_list if d.get('tag_name')} diff --git a/wrolpi/collections/test/test_domains_config.py b/wrolpi/collections/test/test_domains_config.py index 0c3850bb..d96b096b 100644 --- a/wrolpi/collections/test/test_domains_config.py +++ b/wrolpi/collections/test/test_domains_config.py @@ -208,6 +208,30 @@ class TestDomainsConfig: assert 'test.org' in names assert 'invalid' not in names + def test_domains_config_import_many_collections(self, test_session: Session, test_directory: pathlib.Path, + async_client): + """Importing well over 1000 domains must not fail. + + Regression test for the 10.0.0.9 upgrade: `Collection.batch_from_config` OR'd a + (name = ? AND kind = ?) pair per collection, and 1,670 domains exceeded SQLite's + expression-tree depth limit of 1000 ("Expression tree is too large").""" + config_file = test_directory / 'domains.yaml' + config_data = { + 'version': 0, + 'collections': [{'name': f'domain{i}.example.com', 'kind': 'domain'} for i in range(1200)], + } + config_file.write_text(yaml.dump(config_data)) + + config = DomainsConfig() + config.import_config(file=config_file) + + assert config.successful_import is True + assert test_session.query(Collection).filter_by(kind='domain').count() == 1200 + + # Re-import (the update path) must also survive the pre-fetch query. + config.import_config(file=config_file) + assert test_session.query(Collection).filter_by(kind='domain').count() == 1200 + def test_domains_config_global_instance(self): """Test that the global domains_config instance exists.""" assert domains_config is not None diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 769e1cf2..e206bc8e 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -1974,9 +1974,13 @@ def get_bulk_tag_preview(paths: List[str]) -> BulkTagPreview: with get_db_session() as session: from sqlalchemy import or_ stems = [split_path_stem_and_suffix(f, full=True)[0] for f in unique_files] - file_groups = session.query(FileGroup).filter( - or_(*[FileGroup.primary_path.like(f'{stem}%') for stem in stems]) - ).all() + # Chunk the OR'd LIKEs to stay within SQLite's expression-tree depth limit of 1000. + file_groups = [] + for idx in range(0, len(stems), 500): + chunk = stems[idx:idx + 500] + file_groups.extend(session.query(FileGroup).filter( + or_(*[FileGroup.primary_path.like(f'{stem}%') for stem in chunk]) + ).all()) if not file_groups: # No FileGroups exist yet, no shared tags diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index a2da0071..71023e7f 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -1205,6 +1205,18 @@ async def test_get_bulk_tag_preview_directory(test_session, make_files_structure assert preview.file_count == 3 +@pytest.mark.asyncio +async def test_get_bulk_tag_preview_many_files(test_session, make_files_structure, test_directory): + """Previewing a directory containing well over 1000 files must not fail. + + Regression test: the stem-prefix lookup OR'd a LIKE per stem, and more than ~1000 + stems exceeded SQLite's expression-tree depth limit ("Expression tree is too large").""" + make_files_structure({f'mydir/file{i}.txt': str(i) for i in range(1200)}) + + preview = lib.get_bulk_tag_preview(['mydir/']) + assert preview.file_count == 1200 + + @pytest.mark.asyncio async def test_get_bulk_tag_preview_multi_file_filegroup(async_client, test_session, make_files_structure, test_directory, tag_factory):