SQLite: keep large collection/file queries under the expression-tree depth limit.

Collection.batch_from_config OR'd a (name AND kind) pair per collection, so
importing more than ~1000 collections (e.g. a 1,670-domain domains.yaml)
failed with "Expression tree is too large (maximum depth 1000)".  Query a
flat name IN (...) and match (name, kind) in Python instead.

get_bulk_tag_preview had the same unbounded pattern (an OR'd LIKE per stem);
chunk the query at 500 stems.

Found upgrading 10.0.0.9 (decay).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

wrolpi/collections/models.py

@@ -398,16 +398,16 @@ class Collection(ModelHelper, Base):
 
         # Step 3: Pre-fetch existing collections by (name, kind) for ALL collections
         # This handles cases where directory changed or directory-less collections
-        name_kind_pairs = [(d['name'], d.get('kind', 'channel')) for d in data_list]
+        # A flat IN on name stays within SQLite's expression-tree depth limit (an OR'd
+        # (name AND kind) pair per collection exceeds it past ~1000 collections); the
+        # exact (name, kind) match is applied in Python.
+        name_kind_pairs = {(d['name'], d.get('kind', 'channel')) for d in data_list}
         existing_by_name_kind = {}
         if name_kind_pairs:
-            from sqlalchemy import and_, or_
-            conditions = [
-                and_(Collection.name == name, Collection.kind == kind)
-                for name, kind in name_kind_pairs
-            ]
-            existing = session.query(Collection).filter(or_(*conditions)).all()
-            existing_by_name_kind = {(c.name, c.kind): c for c in existing}
+            names = {name for name, _ in name_kind_pairs}
+            existing = session.query(Collection).filter(Collection.name.in_(names)).all()
+            existing_by_name_kind = {(c.name, c.kind): c for c in existing
+                                     if (c.name, c.kind) in name_kind_pairs}
 
         # Step 4: Pre-fetch all tags in ONE query
         tag_names = {d['tag_name'] for d in data_list if d.get('tag_name')}

wrolpi/collections/test/test_domains_config.py

@@ -208,6 +208,30 @@ class TestDomainsConfig:
         assert 'test.org' in names
         assert 'invalid' not in names
 
+    def test_domains_config_import_many_collections(self, test_session: Session, test_directory: pathlib.Path,
+                                                    async_client):
+        """Importing well over 1000 domains must not fail.
+
+        Regression test for the 10.0.0.9 upgrade: `Collection.batch_from_config` OR'd a
+        (name = ? AND kind = ?) pair per collection, and 1,670 domains exceeded SQLite's
+        expression-tree depth limit of 1000 ("Expression tree is too large")."""
+        config_file = test_directory / 'domains.yaml'
+        config_data = {
+            'version': 0,
+            'collections': [{'name': f'domain{i}.example.com', 'kind': 'domain'} for i in range(1200)],
+        }
+        config_file.write_text(yaml.dump(config_data))
+
+        config = DomainsConfig()
+        config.import_config(file=config_file)
+
+        assert config.successful_import is True
+        assert test_session.query(Collection).filter_by(kind='domain').count() == 1200
+
+        # Re-import (the update path) must also survive the pre-fetch query.
+        config.import_config(file=config_file)
+        assert test_session.query(Collection).filter_by(kind='domain').count() == 1200
+
     def test_domains_config_global_instance(self):
         """Test that the global domains_config instance exists."""
         assert domains_config is not None

wrolpi/files/lib.py

@@ -1974,9 +1974,13 @@ def get_bulk_tag_preview(paths: List[str]) -> BulkTagPreview:
     with get_db_session() as session:
         from sqlalchemy import or_
         stems = [split_path_stem_and_suffix(f, full=True)[0] for f in unique_files]
-        file_groups = session.query(FileGroup).filter(
-            or_(*[FileGroup.primary_path.like(f'{stem}%') for stem in stems])
-        ).all()
+        # Chunk the OR'd LIKEs to stay within SQLite's expression-tree depth limit of 1000.
+        file_groups = []
+        for idx in range(0, len(stems), 500):
+            chunk = stems[idx:idx + 500]
+            file_groups.extend(session.query(FileGroup).filter(
+                or_(*[FileGroup.primary_path.like(f'{stem}%') for stem in chunk])
+            ).all())
 
         if not file_groups:
             # No FileGroups exist yet, no shared tags

wrolpi/files/test/test_lib.py

@@ -1205,6 +1205,18 @@ async def test_get_bulk_tag_preview_directory(test_session, make_files_structure
     assert preview.file_count == 3
 
 
+@pytest.mark.asyncio
+async def test_get_bulk_tag_preview_many_files(test_session, make_files_structure, test_directory):
+    """Previewing a directory containing well over 1000 files must not fail.
+
+    Regression test: the stem-prefix lookup OR'd a LIKE per stem, and more than ~1000
+    stems exceeded SQLite's expression-tree depth limit ("Expression tree is too large")."""
+    make_files_structure({f'mydir/file{i}.txt': str(i) for i in range(1200)})
+
+    preview = lib.get_bulk_tag_preview(['mydir/'])
+    assert preview.file_count == 1200
+
+
 @pytest.mark.asyncio
 async def test_get_bulk_tag_preview_multi_file_filegroup(async_client, test_session, make_files_structure,
                                                          test_directory, tag_factory):