db: address review — confirm WAL was applied; refuse fuseblk mounts

Two fixes from PR review:

- `PRAGMA journal_mode=WAL` returns the mode SQLite actually selected and can leave the
  prior rollback journal without raising.  Confirm the returned mode instead of assuming
  the request succeeded, so a silent WAL refusal correctly falls back to TRUNCATE +
  synchronous=FULL rather than caching WAL with the weaker synchronous=NORMAL.

- `fuseblk` (block-backed FUSE such as ntfs-3g) has unreliable byte-range locking that
  can corrupt the database under the workers' concurrent access.  Move it out of the
  WAL-incompatible allow-list into an UNSAFE_LOCKING_FS_TYPES refuse-list (renamed from
  NETWORK_FS_TYPES).  NTFS on a modern kernel mounts via the native `ntfs3` driver, which
  stays allowed; exFAT is unaffected.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Roland 2026-07-22 18:53:14 -06:00 • committed by lrnselfreliance
parent 8988a45195
commit 5bbbef3f0b
4 changed files with 68 additions and 23 deletions

View file

@ -62,10 +62,13 @@ def _configure_sqlite_connection(dbapi_conn, journal_mode: str = None) -> str:
WAL is preferred (readers never block writers, so the Sanic workers stay concurrent). But
FAT/exFAT/NTFS drives — common when a USB drive is formatted for Windows compatibility —
cannot support WAL: it needs mmap'd shared-memory those filesystems don't provide, so
`PRAGMA journal_mode=WAL` raises `disk I/O error`. There we fall back to a TRUNCATE rollback
journal so the drive still works, at the cost of write-concurrency; `synchronous=FULL` keeps
it crash-safe (WROLPi is off-grid, so power loss is expected).
cannot support WAL: it needs mmap'd shared-memory those filesystems don't provide. SQLite may
reject WAL either by raising `disk I/O error` OR by silently keeping the current rollback
journal (`PRAGMA journal_mode` returns the mode it actually selected, not necessarily the one
requested), so we confirm the returned mode rather than assume the request succeeded. When WAL
is unavailable we fall back to a TRUNCATE rollback journal so the drive still works, at the cost
of write-concurrency; `synchronous=FULL` keeps it crash-safe (WROLPi is off-grid, so power loss
is expected).
`journal_mode` is the mode chosen on a previous connection of the same engine: once WAL has
been ruled out we skip re-attempting (and re-logging) it on every fresh NullPool connection."""
@ -76,11 +79,17 @@ def _configure_sqlite_connection(dbapi_conn, journal_mode: str = None) -> str:
curs = dbapi_conn.cursor()
try:
if journal_mode != 'TRUNCATE':
wal_ok = False
try:
curs.execute('PRAGMA journal_mode=WAL')
row = curs.fetchone()
wal_ok = bool(row) and str(row[0]).lower() == 'wal'
except sqlite3.OperationalError:
wal_ok = False
if wal_ok:
curs.execute('PRAGMA synchronous=NORMAL')
journal_mode = 'WAL'
except sqlite3.OperationalError:
else:
logger.warning('SQLite WAL is unavailable on this filesystem (exFAT/FAT/NTFS?); '
'falling back to a slower TRUNCATE rollback journal with reduced '
'write-concurrency. Reformat the media drive as ext4 for best performance.')

View file

@ -23,14 +23,19 @@ logger = logger.getChild(__name__)
MINIMUM_SQLITE_VERSION = (3, 40, 0)
# WAL locking is unsafe on network filesystems; refuse to run there.
NETWORK_FS_TYPES = {'nfs', 'nfs4', 'cifs', 'smb3', 'smbfs', 'fuse.sshfs', 'fuse.rclone', '9p'}
# SQLite locking is unsafe on these filesystems; refuse to run there. Network filesystems break
# POSIX locking outright. `fuseblk` is a block-backed FUSE mount (ntfs-3g and the like) whose
# byte-range locking is unreliable enough to risk corruption under the workers' concurrent access;
# NTFS drives on a modern kernel mount via the native `ntfs3` driver instead (allowed below).
UNSAFE_LOCKING_FS_TYPES = {'nfs', 'nfs4', 'cifs', 'smb3', 'smbfs', 'fuse.sshfs', 'fuse.rclone', '9p',
'fuseblk'}
# FAT/exFAT/NTFS drives (common when a USB drive is formatted for Windows compatibility) cannot
# support SQLite WAL: it needs mmap'd shared-memory that these filesystems do not provide. These
# are still usable — the engine degrades to a rollback journal (see `_configure_sqlite_connection`)
# — but with reduced write-concurrency, so we warn the user to prefer ext4.
WAL_INCOMPATIBLE_FS_TYPES = {'vfat', 'exfat', 'msdos', 'ntfs', 'ntfs3', 'fuseblk'}
# — but with reduced write-concurrency, so we warn the user to prefer ext4. (Native kernel drivers
# only; their POSIX locking is handled by the VFS layer and is reliable.)
WAL_INCOMPATIBLE_FS_TYPES = {'vfat', 'exfat', 'msdos', 'ntfs', 'ntfs3'}
def _media_fs_type(path: Path) -> Optional[str]:
@ -59,8 +64,8 @@ def check_sqlite_environment(db_file: Path) -> Optional[str]:
if sqlite3.sqlite_version_info < MINIMUM_SQLITE_VERSION:
return f'SQLite {sqlite3.sqlite_version} is too old, WROLPi requires 3.40+'
fs_type = _media_fs_type(db_file.parent if db_file.parent.exists() else db_file.parent.parent)
if fs_type in NETWORK_FS_TYPES:
return f'Refusing to use a database on a network filesystem ({fs_type}); WAL locking is unsafe there'
if fs_type in UNSAFE_LOCKING_FS_TYPES:
return f'Refusing to use a database on a {fs_type} filesystem; SQLite locking is unsafe there'
if fs_type in WAL_INCOMPATIBLE_FS_TYPES:
# Usable, but WAL is impossible here so the engine will use a slower rollback journal.
logger.warning(f'Media filesystem is {fs_type}: SQLite WAL is unavailable, so the database will use a '

View file

@ -5,26 +5,39 @@ from wrolpi.db import get_db_session, get_immediate_db_session, _configure_sqlit
class _FakeCursor:
"""A minimal DBAPI cursor that records executed statements and can be told to fail WAL.
"""A minimal DBAPI cursor that records executed statements and simulates WAL rejection.
Simulates a FAT/exFAT/NTFS drive, where `PRAGMA journal_mode=WAL` raises `disk I/O error`."""
A FAT/exFAT/NTFS drive rejects WAL either by raising `disk I/O error` (`fail_wal`) or by
silently keeping the current rollback journal — `PRAGMA journal_mode` returns the mode SQLite
actually selected, so a WAL request can return e.g. 'delete' without raising (`wal_returns`)."""
def __init__(self, fail_wal: bool):
def __init__(self, fail_wal: bool, wal_returns: str = 'wal'):
self.fail_wal = fail_wal
self.wal_returns = wal_returns
self.executed = []
self._last_row = None
def execute(self, statement):
if self.fail_wal and statement == 'PRAGMA journal_mode=WAL':
raise sqlite3.OperationalError('disk I/O error')
if statement == 'PRAGMA journal_mode=WAL':
if self.fail_wal:
raise sqlite3.OperationalError('disk I/O error')
self._last_row = (self.wal_returns,)
elif statement.startswith('PRAGMA journal_mode='):
self._last_row = (statement.split('=', 1)[1].lower(),)
else:
self._last_row = None
self.executed.append(statement)
def fetchone(self):
return self._last_row
def close(self):
pass
class _FakeConnection:
def __init__(self, fail_wal: bool):
self._cursor = _FakeCursor(fail_wal)
def __init__(self, fail_wal: bool, wal_returns: str = 'wal'):
self._cursor = _FakeCursor(fail_wal, wal_returns)
self.isolation_level = 'DEFERRED'
def cursor(self):
@ -61,6 +74,20 @@ def test_configure_connection_falls_back_to_rollback_journal_on_wal_failure():
assert 'PRAGMA foreign_keys=ON' in conn._cursor.executed
def test_configure_connection_falls_back_when_wal_silently_refused():
"""SQLite can refuse WAL WITHOUT raising, returning the still-active rollback mode instead.
The requested mode is not always the applied mode, so we must confirm `PRAGMA journal_mode`'s
return value. Here WAL 'succeeds' but reports 'delete'; that must trigger the durable fallback
(TRUNCATE + synchronous=FULL), not cache WAL with the weaker synchronous=NORMAL."""
conn = _FakeConnection(fail_wal=False, wal_returns='delete')
mode = _configure_sqlite_connection(conn)
assert mode == 'TRUNCATE'
assert 'PRAGMA journal_mode=TRUNCATE' in conn._cursor.executed
assert 'PRAGMA synchronous=FULL' in conn._cursor.executed
assert 'PRAGMA synchronous=NORMAL' not in conn._cursor.executed
def test_configure_connection_does_not_reattempt_wal_once_ruled_out():
"""Once WAL is known-unavailable for an engine, later connections skip (and don't re-log) it."""
conn = _FakeConnection(fail_wal=True)

View file

@ -92,9 +92,9 @@ def test_check_sqlite_environment_version_gate(test_directory):
assert db_bootstrap.check_sqlite_environment(get_db_file()) is None
@pytest.mark.parametrize('fs_type', ['vfat', 'exfat', 'msdos', 'ntfs', 'fuseblk'])
@pytest.mark.parametrize('fs_type', ['vfat', 'exfat', 'msdos', 'ntfs', 'ntfs3'])
def test_check_sqlite_environment_allows_wal_incompatible_fs_with_warning(test_directory, fs_type):
"""A FAT/exFAT/NTFS drive is allowed (with a warning), not refused.
"""A FAT/exFAT/NTFS drive on a native kernel driver is allowed (with a warning), not refused.
These filesystems cannot support SQLite WAL — it needs mmap shared-memory they don't provide,
so `PRAGMA journal_mode=WAL` raises `(sqlite3.OperationalError) disk I/O error` (the failure
@ -116,9 +116,13 @@ def test_check_sqlite_environment_allows_wal_compatible_fs(test_directory, fs_ty
assert db_bootstrap.check_sqlite_environment(get_db_file()) is None
@pytest.mark.parametrize('fs_type', ['nfs', 'nfs4', 'cifs', 'fuse.sshfs'])
def test_check_sqlite_environment_refuses_network_fs(test_directory, fs_type):
"""Network filesystems remain refused: their locking is unsafe and no journal mode fixes it."""
@pytest.mark.parametrize('fs_type', ['nfs', 'nfs4', 'cifs', 'fuse.sshfs', 'fuseblk'])
def test_check_sqlite_environment_refuses_unsafe_locking_fs(test_directory, fs_type):
"""Filesystems with unreliable locking are refused: no journal mode makes SQLite safe there.
Includes `fuseblk` (block-backed FUSE, e.g. ntfs-3g), whose byte-range locking is unreliable
under the workers' concurrent access. NTFS on a modern kernel mounts as native `ntfs3` instead
and is allowed (with the WAL-unavailable warning)."""
with mock.patch.object(db_bootstrap, '_media_fs_type', return_value=fs_type):
error = db_bootstrap.check_sqlite_environment(get_db_file())
assert error and fs_type in error