From b3dcb0046e053b0eb11d2e501360ec60624142e7 Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 20 Jul 2026 10:20:20 -0600 Subject: [PATCH] Chunk upsert_directories INSERT to respect SQLite variable limit Refreshing all files failed on a large media directory with RuntimeError: values_clause: too many parameters (44454 rows x 3). After the Postgres->SQLite cutover, values_clause guards against exceeding SQLite's ~32k bound-variable limit, but upsert_directories still built a single multi-row VALUES clause over every discovered directory. Insert in BULK_SQL_CHUNK_SIZE batches, matching the existing bulk tag-file callers. Adds a regression test that inserts more directories than a single statement can bind. Co-Authored-By: Claude Opus 4.8 (1M context) --- wrolpi/files/lib.py | 21 ++++++++++++--------- wrolpi/files/test/test_lib.py | 21 ++++++++++++++++++++- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index ea6de5ae..7f6c3761 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -26,7 +26,7 @@ from wrolpi import flags from wrolpi.cmd import which from wrolpi.common import get_media_directory, wrol_mode_check, logger, \ partition, \ - get_files_and_directories, chunks_by_stem, walk, \ + get_files_and_directories, chunks, chunks_by_stem, walk, \ get_wrolpi_config, \ unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory, strip_surrogates from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where @@ -1006,15 +1006,18 @@ def upsert_directories(parent_directories, directories): if directories: # Insert any directories that were created, update any directories which previously existed. + # Chunk the rows so the multi-row VALUES clause stays under SQLite's bound-variable limit; + # a large media directory can hold tens of thousands of directories. + values = [(str(i.absolute()), i.name, idempotency) for i in directories] with get_db_curs(commit=True) as curs: - values = [(str(i.absolute()), i.name, idempotency) for i in directories] - placeholders, params = values_clause(values) - stmt = f''' - INSERT INTO directory (path, name, idempotency) VALUES {placeholders} - ON CONFLICT (path) DO UPDATE - SET idempotency = EXCLUDED.idempotency - ''' - curs.execute(stmt, params) + for chunk in chunks(values, BULK_SQL_CHUNK_SIZE): + placeholders, params = values_clause(chunk) + stmt = f''' + INSERT INTO directory (path, name, idempotency) VALUES {placeholders} + ON CONFLICT (path) DO UPDATE + SET idempotency = EXCLUDED.idempotency + ''' + curs.execute(stmt, params) with get_db_curs(commit=True) as curs: # Delete the children of any parent directory which no longer exists. diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index dd203b33..76cf4826 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -18,7 +18,7 @@ from wrolpi.conftest import await_switches from wrolpi.dates import now from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile from wrolpi.files import lib, indexers -from wrolpi.files.models import FileGroup +from wrolpi.files.models import FileGroup, Directory from wrolpi.tags import TagFile from wrolpi.test.common import only_macos, skip_circleci from wrolpi.vars import PROJECT_DIR, IS_MACOS @@ -856,6 +856,25 @@ async def test_refresh_directories(test_session, test_directory, assert_director assert_directories({'foo', 'baz'}) +@pytest.mark.asyncio +async def test_upsert_directories_chunks_large_batches(test_session, test_directory): + """A media directory with more directories than SQLite can bind in one statement is inserted in chunks. + + Regression: SQLite limits the number of bound variables per statement, so a single multi-row VALUES + clause over tens of thousands of directories raised RuntimeError('values_clause: too many parameters').""" + from wrolpi.db import SQLITE_MAX_VARIABLES + # upsert_directories binds 3 columns per directory; exceed the limit so the un-chunked path would fail. + count = SQLITE_MAX_VARIABLES // 3 + 50 + paths = [test_directory / f'd{i:05d}' for i in range(count)] + for path in paths: + path.mkdir() + + lib.upsert_directories([test_directory], paths) + + stored = {i.path for i in test_session.query(Directory).all()} + assert set(paths).issubset(stored), 'all discovered directories should be inserted' + + @pytest.mark.asyncio async def test_file_group_merge(async_client, test_session, test_directory, make_files_structure, tag_factory, video_bytes, srt_file3):