A config import records the version of the file it read (#897)

* A config import records the version of the file it read

Every Sanic process initializes the configs it shares with the others while one of them imports the
database configs.  `ConfigFile.initialize` reset the shared dict to the defaults (version 0) before
loading the file, and the import read the file's version from that shared dict.  When the two
overlapped, the import recorded version 0 in `config_version`; every later save of that config was
refused as older than its file, and the next startup's import undid the unsaved changes.

- `ConfigFile.initialize` sets the shared dict in one call, and `MultiFileConfig.initialize` no longer
  empties it first, so other processes never see the defaults in place of the file.
- An import reads the version and rows from the file it read (`import_file_config`), never the shared
  dict.

Tests run the race for every config.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Playlists import steps read the file's config; tests restore switch handlers

- `PlaylistsConfig._keep_unreadable_items` and `_merge_items` read `import_file_config`, not the
  shared `_config`.
- `test_import_steps_read_the_file`: no versioned config's `prepare`/`after_import` (or the methods
  they call) reads the shared config.
- The new tests create configs, which replaced the global configs' switch handlers for later tests
  (repos tests then dumped a test's config).  `restore_switch_handlers` puts them back.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Only the first process loads a shared config; the others attach

A restarted perpetual process initializes the configs while the other processes serve them.
Re-loading the files then reverted or dropped newer changes in the shared dict (an inventory
created or saved since, a config updated but not saved yet).

- `ConfigFile.initialize` loads the file only into an empty shared dict, under
  `config_update_lock`; otherwise it keeps what the dict holds.
- `MultiFileConfig.initialize` imports only into an empty shared store, under `save_lock()`, and
  points `_configs` back at the shared store if the import fails.
- `MultiFileConfig.create` checks for the slug and inserts it under `save_lock()`.

`test_initialize_keeps_the_shared_config` now changes the shared config between two processes'
initializes (an inventory renamed and another created), and requires the second to keep it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-10-27 21:03:39 -06:00 • committed by Roland
parent 98ceb67321
commit ce7095c0a1
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
8 changed files with 266 additions and 40 deletions

View file

@ -509,13 +509,13 @@ class ChannelsConfig(ConfigFile):
# An invalid entry keeps its Channel (found by the entry's name when its directory is invalid), and the
# Channel's Downloads; so does an entry without a list of Downloads (an old config), or with an unreadable
# Download.
entries = [i for i in self._config.get('channels') or [] if isinstance(i, dict)]
entries = [i for i in self.import_file_config.get('channels') or [] if isinstance(i, dict)]
invalid_names = {i.get('name') for i in entries if i.get('name')} - {i['name'] for i in self._channel_rows()}
for key in [i for i in channels.removed if db_rows[i]['name'] in invalid_names]:
channels.removed.discard(key)
kept = set(db_rows) - set(self._channel_rows_by_key()) - channels.removed - set(moves)
kept |= {key for key, data in self._channel_entries() if 'downloads' not in data}
kept |= {directory for directory, _ in ChannelDownloadsSection().file_rows(self._config)[1]}
kept |= {directory for directory, _ in ChannelDownloadsSection().file_rows(self.import_file_config)[1]}
# A moved Channel's Downloads are deleted after the move, under its new directory; one its new entry lists
# is the same Download.
@ -539,7 +539,7 @@ class ChannelsConfig(ConfigFile):
def _channel_entries(self) -> List[Tuple[str, dict]]:
"""The valid entries of the file, keyed by directory."""
entries = list()
for data in self._config.get('channels') or []:
for data in self.import_file_config.get('channels') or []:
try:
entries.append((_channel_key(data), data))
except Exception:
@ -547,7 +547,7 @@ class ChannelsConfig(ConfigFile):
return entries
def _channel_rows_by_key(self) -> dict:
return ChannelsSection().file_rows(self._config)[0]
return ChannelsSection().file_rows(self.import_file_config)[0]
def _channel_rows(self) -> list:
return list(self._channel_rows_by_key().values())

View file

@ -298,7 +298,7 @@ class PlaylistsConfig(ConfigFile):
def _keep_unreadable_items(self, diff: SectionDiff):
"""A playlist whose items cannot be read keeps the items it has."""
_, invalid = PlaylistItemsSection().file_rows(self._config)
_, invalid = PlaylistItemsSection().file_rows(self.import_file_config)
if names := {i[0] for i in invalid if i[2] == ITEMS_UNREADABLE}:
diff.removed = {i for i in diff.removed if i[0] not in names}
@ -331,7 +331,7 @@ class PlaylistsConfig(ConfigFile):
return
with get_db_session() as session:
db_rows = section.db_rows(session)
file_rows, _ = section.file_rows(self._config)
file_rows, _ = section.file_rows(self.import_file_config)
for name in names:
merged = sorted((i for i in db_rows if i[0] == name), key=lambda i: db_rows[i]['position'])
previous = None

View file

@ -550,20 +550,32 @@ class ConfigFile:
return dict(self._config)
def initialize(self, multiprocessing_dict: Optional[DictProxy] = None):
"""Initializes this config dict using the default config and the config file."""
"""Initializes this config dict using the default config and the config file.
Every process initializes the shared dict, some while the others use it (a restarted perpetual process). Only
the first loads it; the others keep what it holds, which may be newer than the file."""
from wrolpi.api_utils import api_app
# Use the provided multiprocessing.Manager().dict() when live, or, dict() for testing.
self._config = multiprocessing_dict if multiprocessing_dict is not None else dict()
# Use the default settings to initialize the config.
self._config.update(deepcopy(self.default_config))
with getattr(api_app.shared_ctx, 'config_update_lock', None) or contextlib.nullcontext():
if self._config:
return self
file = self.get_file()
if file.is_file():
if not self.is_valid(file):
raise InvalidConfig(f'Config file is invalid: {str(self.get_relative_file())}')
# is_valid() already validated, just load the data
config_data = self.read_config_file(file)
config_data = {k: v for k, v in config_data.items() if k in self.default_config}
self._config.update(config_data)
# Use the default settings to initialize the config.
config = deepcopy(self.default_config)
try:
file = self.get_file()
if file.is_file():
if not self.is_valid(file):
raise InvalidConfig(f'Config file is invalid: {str(self.get_relative_file())}')
# is_valid() already validated, just load the data
config_data = self.read_config_file(file)
config.update({k: v for k, v in config_data.items() if k in self.default_config})
finally:
# In one call: other processes never see the defaults in place of the file (an import would record
# their version 0).
self._config.update(config)
return self
@property
@ -859,11 +871,12 @@ class ConfigFile:
data = self.read_config_file(file) if file.stat().st_size else None
if not data or not self.validate(data):
raise InvalidConfig(f'Config is {"invalid" if data else "empty"}: {file_str}')
self._config.update({k: v for k, v in data.items() if k in self.default_config})
config = {**deepcopy(self.default_config), **{k: v for k, v in data.items() if k in self.default_config}}
self._config.update(config)
# Another writer may change the database between the snapshot and the write; then decide again.
for _ in range(3):
if self._import_sections_once(forced):
if self._import_sections_once(config, forced):
break
else:
raise RuntimeError(f'The database kept changing while importing {file_str}')
@ -876,27 +889,32 @@ class ConfigFile:
Events.send_config_import_failed(message)
raise
def _import_sections_once(self, forced: bool) -> bool:
"""One pass of `import_sections`. Returns False when the database's version changed before the write."""
def _import_sections_once(self, config: dict, forced: bool) -> bool:
"""One pass of `import_sections` of the file's `config`. Returns False when the database's version changed
before the write.
It reads the file's `config`, not the shared `_config`, which other processes change meanwhile."""
from wrolpi.config_import import diff_section
from wrolpi.config_version import IMPORT, SAVE, get_config_version, set_config_version
from wrolpi.db import get_db_session
decision, db_version = self.version_decision(forced)
file_version = config['version'] or 0
decision, db_version = self.version_decision(file_version, forced)
if decision == SAVE:
self.save_database_instead()
return True
file_rows = {i.name: i.file_rows(self._config) for i in self.sections}
file_rows = {i.name: i.file_rows(config) for i in self.sections}
with get_db_session() as session:
db_rows = {i.name: i.db_rows(session) for i in self.sections}
diffs = {i.name: diff_section(*file_rows[i.name], db_rows[i.name], deletes=decision == IMPORT, same=i.same,
present=i.present(self._config, *file_rows[i.name]))
present=i.present(config, *file_rows[i.name]))
for i in self.sections}
# `prepare` may skip rows it cannot apply.
self.import_skipped = sum(len(invalid) for _, invalid in file_rows.values())
self.import_deletes = decision == IMPORT
self.import_db_rows = db_rows
self.import_file_config = config
if any(diffs.values()):
self.prepare(diffs)
# A row `prepare` brought in line with the database (e.g. it dropped a Tag the file names but no longer
@ -906,7 +924,6 @@ class ConfigFile:
for key in [i for i in changed if section.same(changed[i], db_rows[section.name][i])]:
del changed[key]
file_version = self.version or 0
if any(diffs.values()) or (decision == IMPORT and db_version != file_version):
try:
with get_db_session(commit=True) as session:
@ -946,6 +963,7 @@ class ConfigFile:
differences. `diffs` maps each section's name to its `SectionDiff`; it may drop rows it cannot apply
(count them in `import_skipped`), or add what it found to a row for `upsert`. `import_deletes`: whether
this import deletes what the file no longer lists. `import_db_rows`: each section's database rows.
`import_file_config`: the file's config (not the shared `_config`, which other processes change meanwhile).
It must be safe to repeat and to abandon: when the database changes before the write, the import decides
again (up to three passes), and the diffs it was given may never be written (the write fails, or the next
@ -977,15 +995,16 @@ class ConfigFile:
from wrolpi.config_version import versioned_config_names
return self.file_name in versioned_config_names()
def version_decision(self, forced: bool = False) -> Tuple[str, Optional[int]]:
"""What importing this config's file (already read) may do (IMPORT, IMPORT_WITHOUT_DELETES, or SAVE), and the
database version that decided it. `import_sections` records the file's version in its own write."""
def version_decision(self, file_version: int, forced: bool = False) -> Tuple[str, Optional[int]]:
"""What importing this config's file (already read, at `file_version`) may do (IMPORT, IMPORT_WITHOUT_DELETES,
or SAVE), and the database version that decided it. `import_sections` records the file's version in its own
write."""
from wrolpi.config_version import IMPORT, decide, get_config_version
from wrolpi.db import get_db_session
with get_db_session() as session:
db_version = get_config_version(session, self.file_name)
decision = IMPORT if forced else decide(self.version or 0, db_version)
logger_.debug(f'{self.file_name}: file version {self.version}, database version {db_version}: {decision}')
decision = IMPORT if forced else decide(file_version, db_version)
logger_.debug(f'{self.file_name}: file version {file_version}, database version {db_version}: {decision}')
return decision, db_version
def save_database_instead(self):
@ -1178,10 +1197,21 @@ class MultiFileConfig:
return sorted(f.stem for f in directory.glob(self.glob_pattern) if f.is_file())
def initialize(self, multiprocessing_dict: Optional[DictProxy] = None):
"""Assign the shared in-memory store, then import every entity file from disk."""
self._configs = multiprocessing_dict if multiprocessing_dict is not None else dict()
self._configs.clear()
self.import_all()
"""Assign the shared in-memory store, then import every entity file from disk.
Every process initializes the shared store, some while the others use it (a restarted perpetual process).
Only the first imports the files; the others keep what it holds, which may be newer than the files."""
configs = multiprocessing_dict if multiprocessing_dict is not None else dict()
with self.save_lock():
if not configs:
# Import into this process's dict, then add the entities in one call.
self._configs = dict()
try:
self.import_all()
configs.update(self._configs)
finally:
self._configs = configs
self._configs = configs
return self
def is_valid(self, data: dict) -> bool:
@ -1326,15 +1356,16 @@ class MultiFileConfig:
def create(self, slug: str, data: dict):
"""Add a new entity and persist it immediately."""
if slug in self._configs:
raise ValueError(f'{self.name} already exists: {slug}')
entity = deepcopy(self.default_entity)
entity.update(data)
entity['version'] = 0
if not self.is_valid(entity):
raise ValidationError(f'Invalid {self.name} config: {entity}')
self._configs[slug] = entity
self._save_slug(slug)
with self.save_lock():
if slug in self._configs:
raise ValueError(f'{self.name} already exists: {slug}')
self._configs[slug] = entity
self._persist_slug(slug)
def replace(self, slug: str, data: dict):
"""Replace an entity's config wholesale and persist it (background)."""

View file

@ -131,6 +131,17 @@ def no_unexpected_queries(request):
'\n'.join(f'{reason}: {" ".join(statement.split())[:300]}' for reason, statement in queries)
@pytest.fixture
def restore_switch_handlers():
"""Creating a config replaces the switch handlers of the global config with the same file (see
`ConfigFile.__init__`); put them back after the test."""
from wrolpi import switches
handlers = dict(switches.SWITCH_HANDLERS)
yield
switches.SWITCH_HANDLERS.clear()
switches.SWITCH_HANDLERS.update(handlers)
def config_was_imported(session, *file_names: str):
"""Record that these database configs were imported before, as on any WROLPi after its first startup with
wrolpi.config_version, so their next import deletes what the file no longer lists. (Version 0 is no newer than

View file

@ -268,13 +268,13 @@ class RecentlyViewedConfig(ConfigFile):
def prepare(self, diffs: dict):
diff, db_rows = diffs['viewed'], self.import_db_rows['viewed']
file_rows = ViewedSection().file_rows(self._config)[0]
file_rows = ViewedSection().file_rows(self.import_file_config)[0]
# Only views at least as recent as the oldest entry of the file are removed: an older view fell off the end
# of the saved window. A malformed entry keeps its file's history.
if diff.removed:
oldest = min(i['viewed'] for i in file_rows.values()) if file_rows else None
malformed = _malformed_keys(self._config)
malformed = _malformed_keys(self.import_file_config)
diff.removed = {i for i in diff.removed
if oldest and db_rows[i]['viewed'] >= oldest and i not in malformed}

View file

@ -2,6 +2,7 @@ import json
import pathlib
import tempfile
from contextlib import contextmanager
from copy import deepcopy
from functools import partialmethod
from itertools import zip_longest
from typing import List
@ -138,3 +139,57 @@ def assert_dict_contains(d1: dict, d2: dict):
d1_str = json.dumps(d1, indent=1, cls=CustomJSONEncoder)
d2_str = json.dumps(d2, indent=1, cls=CustomJSONEncoder)
raise AssertionError(f'{d1_str}\n\n{d2_str}\n\nDictionaries above are unequal') from e
def config_classes() -> list:
"""Every config class: each ConfigFile, and each MultiFileConfig (a directory of files)."""
from wrolpi.common import ConfigFile, MultiFileConfig
# Import every module that defines a config.
import modules.archive.lib # noqa
import modules.flasher.config # noqa
import modules.inventory.catalog # noqa
import modules.inventory.common # noqa
import modules.map.pins # noqa
import modules.repos.lib # noqa
import modules.videos.lib # noqa
import wrolpi.bookmarks # noqa
import wrolpi.collections.config # noqa
import wrolpi.downloader # noqa
import wrolpi.files.progress # noqa
import wrolpi.tags # noqa
return sorted(ConfigFile.__subclasses__() + MultiFileConfig.__subclasses__(), key=lambda i: i.__name__)
class SharedDict(dict):
"""Stands in for a `multiprocessing.Manager().dict()` the Sanic processes share: each call is atomic, and another
process may read between any two calls. `readable` records what it could read."""
def __init__(self):
super().__init__()
self.readable = []
def _changed(self):
self.readable.append(deepcopy(dict(self)))
def __setitem__(self, key, value):
super().__setitem__(key, value)
self._changed()
def __delitem__(self, key):
super().__delitem__(key)
self._changed()
def update(self, *args, **kwargs):
super().update(*args, **kwargs)
self._changed()
def clear(self):
super().clear()
self._changed()
def pop(self, *args):
value = super().pop(*args)
self._changed()
return value

View file

@ -1,4 +1,10 @@
"""A database config's file and the database each carry a version (see wrolpi.config_version)."""
import ast
import inspect
import textwrap
import threading
from copy import deepcopy
import pytest
import yaml
@ -7,6 +13,7 @@ from wrolpi.config_version import get_config_version, set_config_version
from wrolpi.dates import now
from wrolpi.files.models import FileGroup
from wrolpi.tags import Tag, get_tags_config, import_tags_config
from wrolpi.test.common import config_classes
def write_tags_config(path, version: int, names):
@ -338,3 +345,81 @@ def test_domain_downloads_count_for_domains(test_session):
test_session.commit()
assert db_version(test_session, 'domains.yaml') == 2
@pytest.mark.parametrize('config_class', [i for i in config_classes() if getattr(i, 'sections', None)],
ids=lambda i: i.__name__)
@pytest.mark.asyncio
async def test_import_while_another_process_initializes(test_session, test_directory, async_client,
restore_switch_handlers, config_class, monkeypatch):
"""The Sanic processes initialize the configs they share while one of them imports the database configs. The
import records the version of the file it read, whatever another process does to the shared config meanwhile;
otherwise the database's version goes back (e.g. to 0), and every later save is refused as older than the file."""
config = config_class()
file = config.get_file()
file.parent.mkdir(parents=True, exist_ok=True)
config.write_config_data({**deepcopy(config.default_config), 'version': 7}, file)
shared = dict()
config.initialize(shared)
set_config_version(test_session, config.file_name, 7)
test_session.commit()
# Configs refuse to import while testing unless they are the test's config.
monkeypatch.setattr('wrolpi.tags.TEST_TAGS_CONFIG', config)
monkeypatch.setattr('modules.videos.lib.TEST_CHANNELS_CONFIG', config)
# Another process initializes this config, and pauses if it reads the file.
other_process = threading.Thread(target=config.initialize, args=(shared,))
paused, resume = threading.Event(), threading.Event()
read_config_file = config.read_config_file
def pausing_read_config_file(*args, **kwargs):
if threading.current_thread() is other_process:
paused.set()
resume.wait(10)
return read_config_file(*args, **kwargs)
monkeypatch.setattr(config, 'read_config_file', pausing_read_config_file)
# It starts after this import has decided what to do, and before it writes.
version_decision = config.version_decision
def racing_version_decision(*args, **kwargs):
decision = version_decision(*args, **kwargs)
other_process.start()
while other_process.is_alive() and not paused.wait(0.01):
pass
return decision
monkeypatch.setattr(config, 'version_decision', racing_version_decision)
try:
config.import_config()
finally:
resume.set()
if other_process.is_alive():
other_process.join(10)
assert config.successful_import
assert db_version(test_session, config.file_name) == 7
@pytest.mark.parametrize('config_class', [i for i in config_classes() if getattr(i, 'sections', None)],
ids=lambda i: i.__name__)
def test_import_steps_read_the_file(config_class):
"""`prepare` and `after_import` (and the methods they call) read the file's config (`import_file_config`), never
the shared `_config`, which other processes change during the import."""
tree = ast.parse(textwrap.dedent(inspect.getsource(config_class)))
methods = {i.name: i for i in tree.body[0].body if isinstance(i, ast.FunctionDef)}
properties = {name for name, method in methods.items()
if any(getattr(i, 'id', None) == 'property' for i in method.decorator_list)}
reads, seen, todo = set(), set(), [i for i in ('prepare', 'after_import') if i in methods]
while todo:
if (name := todo.pop()) in seen:
continue
seen.add(name)
for node in ast.walk(methods[name]):
if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id == 'self':
if node.attr in methods and node.attr not in properties:
todo.append(node.attr)
elif node.attr in properties | {'_config', 'version'}:
reads.add(f'{name} reads self.{node.attr}')
assert not reads

View file

@ -1,4 +1,22 @@
"""Tests for shared context initialization."""
from copy import deepcopy
import pytest
from wrolpi.common import MultiFileConfig, write_config_data
from wrolpi.test.common import SharedDict, config_classes
def write_config_files(config, version: int):
"""Write a valid config file (or, for a MultiFileConfig, one entity's file) at `version`."""
if isinstance(config, MultiFileConfig):
file = config.get_file('one')
data = {**deepcopy(config.default_entity), 'version': version, 'slug': 'one', 'name': 'One'}
else:
file = config.get_file()
data = {**deepcopy(config.default_config), 'version': version}
file.parent.mkdir(parents=True, exist_ok=True)
write_config_data(data, file)
def test_domains_config_in_shared_context():
@ -56,3 +74,29 @@ def test_all_configs_in_shared_context():
not_shared = sorted(name for name, config in get_all_configs().items()
if not isinstance(getattr(config, '_config', None), DictProxy))
assert not_shared == [], 'Add these configs to attach_shared_contexts() and initialize_configs_contexts()'
@pytest.mark.parametrize('config_class', config_classes(), ids=lambda i: i.__name__)
def test_initialize_keeps_the_shared_config(test_directory, restore_switch_handlers, config_class):
"""Each Sanic process initializes the configs it shares with the others, some while the others already serve,
import and save them (a restarted perpetual process). Initializing must never show the others anything but what
the shared config holds: a config which showed its defaults (version 0) for a moment had that version recorded by
a concurrent import, and one which went back to its file lost another process's newer change."""
config = config_class()
write_config_files(config, 7)
shared = SharedDict()
config.initialize(shared) # The first process.
assert shared and shared.readable
# Another process changes the config, and has not saved it yet.
if isinstance(config, MultiFileConfig):
shared['one'] = {**shared['one'], 'name': 'Renamed'}
shared['two'] = {**shared['one'], 'slug': 'two', 'name': 'Two'}
else:
shared['version'] = 8
changed = deepcopy(dict(shared))
shared.readable.clear()
config.initialize(shared) # Another process starts.
assert dict(shared) == changed
assert all(i == changed for i in shared.readable), 'Another process could read something else'