Chunk upsert_directories INSERT to respect SQLite variable limit

Refreshing all files failed on a large media directory with
RuntimeError: values_clause: too many parameters (44454 rows x 3).
After the Postgres->SQLite cutover, values_clause guards against
exceeding SQLite's ~32k bound-variable limit, but upsert_directories
still built a single multi-row VALUES clause over every discovered
directory.

Insert in BULK_SQL_CHUNK_SIZE batches, matching the existing bulk
tag-file callers.  Adds a regression test that inserts more directories
than a single statement can bind.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Roland 2026-07-20 10:20:20 -06:00
parent 70cbf695df
commit b3dcb0046e
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
2 changed files with 32 additions and 10 deletions

View file

@ -26,7 +26,7 @@ from wrolpi import flags
from wrolpi.cmd import which
from wrolpi.common import get_media_directory, wrol_mode_check, logger, \
partition, \
get_files_and_directories, chunks_by_stem, walk, \
get_files_and_directories, chunks, chunks_by_stem, walk, \
get_wrolpi_config, \
unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory, strip_surrogates
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
@ -1006,15 +1006,18 @@ def upsert_directories(parent_directories, directories):
if directories:
# Insert any directories that were created, update any directories which previously existed.
# Chunk the rows so the multi-row VALUES clause stays under SQLite's bound-variable limit;
# a large media directory can hold tens of thousands of directories.
values = [(str(i.absolute()), i.name, idempotency) for i in directories]
with get_db_curs(commit=True) as curs:
values = [(str(i.absolute()), i.name, idempotency) for i in directories]
placeholders, params = values_clause(values)
stmt = f'''
INSERT INTO directory (path, name, idempotency) VALUES {placeholders}
ON CONFLICT (path) DO UPDATE
SET idempotency = EXCLUDED.idempotency
'''
curs.execute(stmt, params)
for chunk in chunks(values, BULK_SQL_CHUNK_SIZE):
placeholders, params = values_clause(chunk)
stmt = f'''
INSERT INTO directory (path, name, idempotency) VALUES {placeholders}
ON CONFLICT (path) DO UPDATE
SET idempotency = EXCLUDED.idempotency
'''
curs.execute(stmt, params)
with get_db_curs(commit=True) as curs:
# Delete the children of any parent directory which no longer exists.

View file

@ -18,7 +18,7 @@ from wrolpi.conftest import await_switches
from wrolpi.dates import now
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
from wrolpi.files import lib, indexers
from wrolpi.files.models import FileGroup
from wrolpi.files.models import FileGroup, Directory
from wrolpi.tags import TagFile
from wrolpi.test.common import only_macos, skip_circleci
from wrolpi.vars import PROJECT_DIR, IS_MACOS
@ -856,6 +856,25 @@ async def test_refresh_directories(test_session, test_directory, assert_director
assert_directories({'foo', 'baz'})
@pytest.mark.asyncio
async def test_upsert_directories_chunks_large_batches(test_session, test_directory):
"""A media directory with more directories than SQLite can bind in one statement is inserted in chunks.
Regression: SQLite limits the number of bound variables per statement, so a single multi-row VALUES
clause over tens of thousands of directories raised RuntimeError('values_clause: too many parameters')."""
from wrolpi.db import SQLITE_MAX_VARIABLES
# upsert_directories binds 3 columns per directory; exceed the limit so the un-chunked path would fail.
count = SQLITE_MAX_VARIABLES // 3 + 50
paths = [test_directory / f'd{i:05d}' for i in range(count)]
for path in paths:
path.mkdir()
lib.upsert_directories([test_directory], paths)
stored = {i.path for i in test_session.query(Directory).all()}
assert set(paths).issubset(stored), 'all discovered directories should be inserted'
@pytest.mark.asyncio
async def test_file_group_merge(async_client, test_session, test_directory, make_files_structure, tag_factory,
video_bytes, srt_file3):