mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Chunk upsert_directories INSERT to respect SQLite variable limit
Refreshing all files failed on a large media directory with RuntimeError: values_clause: too many parameters (44454 rows x 3). After the Postgres->SQLite cutover, values_clause guards against exceeding SQLite's ~32k bound-variable limit, but upsert_directories still built a single multi-row VALUES clause over every discovered directory. Insert in BULK_SQL_CHUNK_SIZE batches, matching the existing bulk tag-file callers. Adds a regression test that inserts more directories than a single statement can bind. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
70cbf695df
commit
b3dcb0046e
2 changed files with 32 additions and 10 deletions
|
|
@ -26,7 +26,7 @@ from wrolpi import flags
|
|||
from wrolpi.cmd import which
|
||||
from wrolpi.common import get_media_directory, wrol_mode_check, logger, \
|
||||
partition, \
|
||||
get_files_and_directories, chunks_by_stem, walk, \
|
||||
get_files_and_directories, chunks, chunks_by_stem, walk, \
|
||||
get_wrolpi_config, \
|
||||
unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory, strip_surrogates
|
||||
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
|
||||
|
|
@ -1006,15 +1006,18 @@ def upsert_directories(parent_directories, directories):
|
|||
|
||||
if directories:
|
||||
# Insert any directories that were created, update any directories which previously existed.
|
||||
# Chunk the rows so the multi-row VALUES clause stays under SQLite's bound-variable limit;
|
||||
# a large media directory can hold tens of thousands of directories.
|
||||
values = [(str(i.absolute()), i.name, idempotency) for i in directories]
|
||||
with get_db_curs(commit=True) as curs:
|
||||
values = [(str(i.absolute()), i.name, idempotency) for i in directories]
|
||||
placeholders, params = values_clause(values)
|
||||
stmt = f'''
|
||||
INSERT INTO directory (path, name, idempotency) VALUES {placeholders}
|
||||
ON CONFLICT (path) DO UPDATE
|
||||
SET idempotency = EXCLUDED.idempotency
|
||||
'''
|
||||
curs.execute(stmt, params)
|
||||
for chunk in chunks(values, BULK_SQL_CHUNK_SIZE):
|
||||
placeholders, params = values_clause(chunk)
|
||||
stmt = f'''
|
||||
INSERT INTO directory (path, name, idempotency) VALUES {placeholders}
|
||||
ON CONFLICT (path) DO UPDATE
|
||||
SET idempotency = EXCLUDED.idempotency
|
||||
'''
|
||||
curs.execute(stmt, params)
|
||||
|
||||
with get_db_curs(commit=True) as curs:
|
||||
# Delete the children of any parent directory which no longer exists.
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from wrolpi.conftest import await_switches
|
|||
from wrolpi.dates import now
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
|
||||
from wrolpi.files import lib, indexers
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.files.models import FileGroup, Directory
|
||||
from wrolpi.tags import TagFile
|
||||
from wrolpi.test.common import only_macos, skip_circleci
|
||||
from wrolpi.vars import PROJECT_DIR, IS_MACOS
|
||||
|
|
@ -856,6 +856,25 @@ async def test_refresh_directories(test_session, test_directory, assert_director
|
|||
assert_directories({'foo', 'baz'})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upsert_directories_chunks_large_batches(test_session, test_directory):
|
||||
"""A media directory with more directories than SQLite can bind in one statement is inserted in chunks.
|
||||
|
||||
Regression: SQLite limits the number of bound variables per statement, so a single multi-row VALUES
|
||||
clause over tens of thousands of directories raised RuntimeError('values_clause: too many parameters')."""
|
||||
from wrolpi.db import SQLITE_MAX_VARIABLES
|
||||
# upsert_directories binds 3 columns per directory; exceed the limit so the un-chunked path would fail.
|
||||
count = SQLITE_MAX_VARIABLES // 3 + 50
|
||||
paths = [test_directory / f'd{i:05d}' for i in range(count)]
|
||||
for path in paths:
|
||||
path.mkdir()
|
||||
|
||||
lib.upsert_directories([test_directory], paths)
|
||||
|
||||
stored = {i.path for i in test_session.query(Directory).all()}
|
||||
assert set(paths).issubset(stored), 'all discovered directories should be inserted'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_group_merge(async_client, test_session, test_directory, make_files_structure, tag_factory,
|
||||
video_bytes, srt_file3):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue