mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
db: address review — confirm WAL was applied; refuse fuseblk mounts
Two fixes from PR review: - `PRAGMA journal_mode=WAL` returns the mode SQLite actually selected and can leave the prior rollback journal without raising. Confirm the returned mode instead of assuming the request succeeded, so a silent WAL refusal correctly falls back to TRUNCATE + synchronous=FULL rather than caching WAL with the weaker synchronous=NORMAL. - `fuseblk` (block-backed FUSE such as ntfs-3g) has unreliable byte-range locking that can corrupt the database under the workers' concurrent access. Move it out of the WAL-incompatible allow-list into an UNSAFE_LOCKING_FS_TYPES refuse-list (renamed from NETWORK_FS_TYPES). NTFS on a modern kernel mounts via the native `ntfs3` driver, which stays allowed; exFAT is unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
8988a45195
commit
5bbbef3f0b
4 changed files with 68 additions and 23 deletions
19
wrolpi/db.py
19
wrolpi/db.py
|
|
@ -62,10 +62,13 @@ def _configure_sqlite_connection(dbapi_conn, journal_mode: str = None) -> str:
|
|||
|
||||
WAL is preferred (readers never block writers, so the Sanic workers stay concurrent). But
|
||||
FAT/exFAT/NTFS drives — common when a USB drive is formatted for Windows compatibility —
|
||||
cannot support WAL: it needs mmap'd shared-memory those filesystems don't provide, so
|
||||
`PRAGMA journal_mode=WAL` raises `disk I/O error`. There we fall back to a TRUNCATE rollback
|
||||
journal so the drive still works, at the cost of write-concurrency; `synchronous=FULL` keeps
|
||||
it crash-safe (WROLPi is off-grid, so power loss is expected).
|
||||
cannot support WAL: it needs mmap'd shared-memory those filesystems don't provide. SQLite may
|
||||
reject WAL either by raising `disk I/O error` OR by silently keeping the current rollback
|
||||
journal (`PRAGMA journal_mode` returns the mode it actually selected, not necessarily the one
|
||||
requested), so we confirm the returned mode rather than assume the request succeeded. When WAL
|
||||
is unavailable we fall back to a TRUNCATE rollback journal so the drive still works, at the cost
|
||||
of write-concurrency; `synchronous=FULL` keeps it crash-safe (WROLPi is off-grid, so power loss
|
||||
is expected).
|
||||
|
||||
`journal_mode` is the mode chosen on a previous connection of the same engine: once WAL has
|
||||
been ruled out we skip re-attempting (and re-logging) it on every fresh NullPool connection."""
|
||||
|
|
@ -76,11 +79,17 @@ def _configure_sqlite_connection(dbapi_conn, journal_mode: str = None) -> str:
|
|||
curs = dbapi_conn.cursor()
|
||||
try:
|
||||
if journal_mode != 'TRUNCATE':
|
||||
wal_ok = False
|
||||
try:
|
||||
curs.execute('PRAGMA journal_mode=WAL')
|
||||
row = curs.fetchone()
|
||||
wal_ok = bool(row) and str(row[0]).lower() == 'wal'
|
||||
except sqlite3.OperationalError:
|
||||
wal_ok = False
|
||||
if wal_ok:
|
||||
curs.execute('PRAGMA synchronous=NORMAL')
|
||||
journal_mode = 'WAL'
|
||||
except sqlite3.OperationalError:
|
||||
else:
|
||||
logger.warning('SQLite WAL is unavailable on this filesystem (exFAT/FAT/NTFS?); '
|
||||
'falling back to a slower TRUNCATE rollback journal with reduced '
|
||||
'write-concurrency. Reformat the media drive as ext4 for best performance.')
|
||||
|
|
|
|||
|
|
@ -23,14 +23,19 @@ logger = logger.getChild(__name__)
|
|||
|
||||
MINIMUM_SQLITE_VERSION = (3, 40, 0)
|
||||
|
||||
# WAL locking is unsafe on network filesystems; refuse to run there.
|
||||
NETWORK_FS_TYPES = {'nfs', 'nfs4', 'cifs', 'smb3', 'smbfs', 'fuse.sshfs', 'fuse.rclone', '9p'}
|
||||
# SQLite locking is unsafe on these filesystems; refuse to run there. Network filesystems break
|
||||
# POSIX locking outright. `fuseblk` is a block-backed FUSE mount (ntfs-3g and the like) whose
|
||||
# byte-range locking is unreliable enough to risk corruption under the workers' concurrent access;
|
||||
# NTFS drives on a modern kernel mount via the native `ntfs3` driver instead (allowed below).
|
||||
UNSAFE_LOCKING_FS_TYPES = {'nfs', 'nfs4', 'cifs', 'smb3', 'smbfs', 'fuse.sshfs', 'fuse.rclone', '9p',
|
||||
'fuseblk'}
|
||||
|
||||
# FAT/exFAT/NTFS drives (common when a USB drive is formatted for Windows compatibility) cannot
|
||||
# support SQLite WAL: it needs mmap'd shared-memory that these filesystems do not provide. These
|
||||
# are still usable — the engine degrades to a rollback journal (see `_configure_sqlite_connection`)
|
||||
# — but with reduced write-concurrency, so we warn the user to prefer ext4.
|
||||
WAL_INCOMPATIBLE_FS_TYPES = {'vfat', 'exfat', 'msdos', 'ntfs', 'ntfs3', 'fuseblk'}
|
||||
# — but with reduced write-concurrency, so we warn the user to prefer ext4. (Native kernel drivers
|
||||
# only; their POSIX locking is handled by the VFS layer and is reliable.)
|
||||
WAL_INCOMPATIBLE_FS_TYPES = {'vfat', 'exfat', 'msdos', 'ntfs', 'ntfs3'}
|
||||
|
||||
|
||||
def _media_fs_type(path: Path) -> Optional[str]:
|
||||
|
|
@ -59,8 +64,8 @@ def check_sqlite_environment(db_file: Path) -> Optional[str]:
|
|||
if sqlite3.sqlite_version_info < MINIMUM_SQLITE_VERSION:
|
||||
return f'SQLite {sqlite3.sqlite_version} is too old, WROLPi requires 3.40+'
|
||||
fs_type = _media_fs_type(db_file.parent if db_file.parent.exists() else db_file.parent.parent)
|
||||
if fs_type in NETWORK_FS_TYPES:
|
||||
return f'Refusing to use a database on a network filesystem ({fs_type}); WAL locking is unsafe there'
|
||||
if fs_type in UNSAFE_LOCKING_FS_TYPES:
|
||||
return f'Refusing to use a database on a {fs_type} filesystem; SQLite locking is unsafe there'
|
||||
if fs_type in WAL_INCOMPATIBLE_FS_TYPES:
|
||||
# Usable, but WAL is impossible here so the engine will use a slower rollback journal.
|
||||
logger.warning(f'Media filesystem is {fs_type}: SQLite WAL is unavailable, so the database will use a '
|
||||
|
|
|
|||
|
|
@ -5,26 +5,39 @@ from wrolpi.db import get_db_session, get_immediate_db_session, _configure_sqlit
|
|||
|
||||
|
||||
class _FakeCursor:
|
||||
"""A minimal DBAPI cursor that records executed statements and can be told to fail WAL.
|
||||
"""A minimal DBAPI cursor that records executed statements and simulates WAL rejection.
|
||||
|
||||
Simulates a FAT/exFAT/NTFS drive, where `PRAGMA journal_mode=WAL` raises `disk I/O error`."""
|
||||
A FAT/exFAT/NTFS drive rejects WAL either by raising `disk I/O error` (`fail_wal`) or by
|
||||
silently keeping the current rollback journal — `PRAGMA journal_mode` returns the mode SQLite
|
||||
actually selected, so a WAL request can return e.g. 'delete' without raising (`wal_returns`)."""
|
||||
|
||||
def __init__(self, fail_wal: bool):
|
||||
def __init__(self, fail_wal: bool, wal_returns: str = 'wal'):
|
||||
self.fail_wal = fail_wal
|
||||
self.wal_returns = wal_returns
|
||||
self.executed = []
|
||||
self._last_row = None
|
||||
|
||||
def execute(self, statement):
|
||||
if self.fail_wal and statement == 'PRAGMA journal_mode=WAL':
|
||||
raise sqlite3.OperationalError('disk I/O error')
|
||||
if statement == 'PRAGMA journal_mode=WAL':
|
||||
if self.fail_wal:
|
||||
raise sqlite3.OperationalError('disk I/O error')
|
||||
self._last_row = (self.wal_returns,)
|
||||
elif statement.startswith('PRAGMA journal_mode='):
|
||||
self._last_row = (statement.split('=', 1)[1].lower(),)
|
||||
else:
|
||||
self._last_row = None
|
||||
self.executed.append(statement)
|
||||
|
||||
def fetchone(self):
|
||||
return self._last_row
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class _FakeConnection:
|
||||
def __init__(self, fail_wal: bool):
|
||||
self._cursor = _FakeCursor(fail_wal)
|
||||
def __init__(self, fail_wal: bool, wal_returns: str = 'wal'):
|
||||
self._cursor = _FakeCursor(fail_wal, wal_returns)
|
||||
self.isolation_level = 'DEFERRED'
|
||||
|
||||
def cursor(self):
|
||||
|
|
@ -61,6 +74,20 @@ def test_configure_connection_falls_back_to_rollback_journal_on_wal_failure():
|
|||
assert 'PRAGMA foreign_keys=ON' in conn._cursor.executed
|
||||
|
||||
|
||||
def test_configure_connection_falls_back_when_wal_silently_refused():
|
||||
"""SQLite can refuse WAL WITHOUT raising, returning the still-active rollback mode instead.
|
||||
|
||||
The requested mode is not always the applied mode, so we must confirm `PRAGMA journal_mode`'s
|
||||
return value. Here WAL 'succeeds' but reports 'delete'; that must trigger the durable fallback
|
||||
(TRUNCATE + synchronous=FULL), not cache WAL with the weaker synchronous=NORMAL."""
|
||||
conn = _FakeConnection(fail_wal=False, wal_returns='delete')
|
||||
mode = _configure_sqlite_connection(conn)
|
||||
assert mode == 'TRUNCATE'
|
||||
assert 'PRAGMA journal_mode=TRUNCATE' in conn._cursor.executed
|
||||
assert 'PRAGMA synchronous=FULL' in conn._cursor.executed
|
||||
assert 'PRAGMA synchronous=NORMAL' not in conn._cursor.executed
|
||||
|
||||
|
||||
def test_configure_connection_does_not_reattempt_wal_once_ruled_out():
|
||||
"""Once WAL is known-unavailable for an engine, later connections skip (and don't re-log) it."""
|
||||
conn = _FakeConnection(fail_wal=True)
|
||||
|
|
|
|||
|
|
@ -92,9 +92,9 @@ def test_check_sqlite_environment_version_gate(test_directory):
|
|||
assert db_bootstrap.check_sqlite_environment(get_db_file()) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fs_type', ['vfat', 'exfat', 'msdos', 'ntfs', 'fuseblk'])
|
||||
@pytest.mark.parametrize('fs_type', ['vfat', 'exfat', 'msdos', 'ntfs', 'ntfs3'])
|
||||
def test_check_sqlite_environment_allows_wal_incompatible_fs_with_warning(test_directory, fs_type):
|
||||
"""A FAT/exFAT/NTFS drive is allowed (with a warning), not refused.
|
||||
"""A FAT/exFAT/NTFS drive on a native kernel driver is allowed (with a warning), not refused.
|
||||
|
||||
These filesystems cannot support SQLite WAL — it needs mmap shared-memory they don't provide,
|
||||
so `PRAGMA journal_mode=WAL` raises `(sqlite3.OperationalError) disk I/O error` (the failure
|
||||
|
|
@ -116,9 +116,13 @@ def test_check_sqlite_environment_allows_wal_compatible_fs(test_directory, fs_ty
|
|||
assert db_bootstrap.check_sqlite_environment(get_db_file()) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fs_type', ['nfs', 'nfs4', 'cifs', 'fuse.sshfs'])
|
||||
def test_check_sqlite_environment_refuses_network_fs(test_directory, fs_type):
|
||||
"""Network filesystems remain refused: their locking is unsafe and no journal mode fixes it."""
|
||||
@pytest.mark.parametrize('fs_type', ['nfs', 'nfs4', 'cifs', 'fuse.sshfs', 'fuseblk'])
|
||||
def test_check_sqlite_environment_refuses_unsafe_locking_fs(test_directory, fs_type):
|
||||
"""Filesystems with unreliable locking are refused: no journal mode makes SQLite safe there.
|
||||
|
||||
Includes `fuseblk` (block-backed FUSE, e.g. ntfs-3g), whose byte-range locking is unreliable
|
||||
under the workers' concurrent access. NTFS on a modern kernel mounts as native `ntfs3` instead
|
||||
and is allowed (with the WAL-unavailable warning)."""
|
||||
with mock.patch.object(db_bootstrap, '_media_fs_type', return_value=fs_type):
|
||||
error = db_bootstrap.check_sqlite_environment(get_db_file())
|
||||
assert error and fs_type in error
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue