mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 06:56:18 -05:00
A config import records the version of the file it read (#897)
* A config import records the version of the file it read Every Sanic process initializes the configs it shares with the others while one of them imports the database configs. `ConfigFile.initialize` reset the shared dict to the defaults (version 0) before loading the file, and the import read the file's version from that shared dict. When the two overlapped, the import recorded version 0 in `config_version`; every later save of that config was refused as older than its file, and the next startup's import undid the unsaved changes. - `ConfigFile.initialize` sets the shared dict in one call, and `MultiFileConfig.initialize` no longer empties it first, so other processes never see the defaults in place of the file. - An import reads the version and rows from the file it read (`import_file_config`), never the shared dict. Tests run the race for every config. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Playlists import steps read the file's config; tests restore switch handlers - `PlaylistsConfig._keep_unreadable_items` and `_merge_items` read `import_file_config`, not the shared `_config`. - `test_import_steps_read_the_file`: no versioned config's `prepare`/`after_import` (or the methods they call) reads the shared config. - The new tests create configs, which replaced the global configs' switch handlers for later tests (repos tests then dumped a test's config). `restore_switch_handlers` puts them back. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Only the first process loads a shared config; the others attach A restarted perpetual process initializes the configs while the other processes serve them. Re-loading the files then reverted or dropped newer changes in the shared dict (an inventory created or saved since, a config updated but not saved yet). - `ConfigFile.initialize` loads the file only into an empty shared dict, under `config_update_lock`; otherwise it keeps what the dict holds. - `MultiFileConfig.initialize` imports only into an empty shared store, under `save_lock()`, and points `_configs` back at the shared store if the import fails. - `MultiFileConfig.create` checks for the slug and inserts it under `save_lock()`. `test_initialize_keeps_the_shared_config` now changes the shared config between two processes' initializes (an inventory renamed and another created), and requires the second to keep it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
98ceb67321
commit
ce7095c0a1
8 changed files with 266 additions and 40 deletions
|
|
@ -509,13 +509,13 @@ class ChannelsConfig(ConfigFile):
|
|||
# An invalid entry keeps its Channel (found by the entry's name when its directory is invalid), and the
|
||||
# Channel's Downloads; so does an entry without a list of Downloads (an old config), or with an unreadable
|
||||
# Download.
|
||||
entries = [i for i in self._config.get('channels') or [] if isinstance(i, dict)]
|
||||
entries = [i for i in self.import_file_config.get('channels') or [] if isinstance(i, dict)]
|
||||
invalid_names = {i.get('name') for i in entries if i.get('name')} - {i['name'] for i in self._channel_rows()}
|
||||
for key in [i for i in channels.removed if db_rows[i]['name'] in invalid_names]:
|
||||
channels.removed.discard(key)
|
||||
kept = set(db_rows) - set(self._channel_rows_by_key()) - channels.removed - set(moves)
|
||||
kept |= {key for key, data in self._channel_entries() if 'downloads' not in data}
|
||||
kept |= {directory for directory, _ in ChannelDownloadsSection().file_rows(self._config)[1]}
|
||||
kept |= {directory for directory, _ in ChannelDownloadsSection().file_rows(self.import_file_config)[1]}
|
||||
|
||||
# A moved Channel's Downloads are deleted after the move, under its new directory; one its new entry lists
|
||||
# is the same Download.
|
||||
|
|
@ -539,7 +539,7 @@ class ChannelsConfig(ConfigFile):
|
|||
def _channel_entries(self) -> List[Tuple[str, dict]]:
|
||||
"""The valid entries of the file, keyed by directory."""
|
||||
entries = list()
|
||||
for data in self._config.get('channels') or []:
|
||||
for data in self.import_file_config.get('channels') or []:
|
||||
try:
|
||||
entries.append((_channel_key(data), data))
|
||||
except Exception:
|
||||
|
|
@ -547,7 +547,7 @@ class ChannelsConfig(ConfigFile):
|
|||
return entries
|
||||
|
||||
def _channel_rows_by_key(self) -> dict:
|
||||
return ChannelsSection().file_rows(self._config)[0]
|
||||
return ChannelsSection().file_rows(self.import_file_config)[0]
|
||||
|
||||
def _channel_rows(self) -> list:
|
||||
return list(self._channel_rows_by_key().values())
|
||||
|
|
|
|||
|
|
@ -298,7 +298,7 @@ class PlaylistsConfig(ConfigFile):
|
|||
|
||||
def _keep_unreadable_items(self, diff: SectionDiff):
|
||||
"""A playlist whose items cannot be read keeps the items it has."""
|
||||
_, invalid = PlaylistItemsSection().file_rows(self._config)
|
||||
_, invalid = PlaylistItemsSection().file_rows(self.import_file_config)
|
||||
if names := {i[0] for i in invalid if i[2] == ITEMS_UNREADABLE}:
|
||||
diff.removed = {i for i in diff.removed if i[0] not in names}
|
||||
|
||||
|
|
@ -331,7 +331,7 @@ class PlaylistsConfig(ConfigFile):
|
|||
return
|
||||
with get_db_session() as session:
|
||||
db_rows = section.db_rows(session)
|
||||
file_rows, _ = section.file_rows(self._config)
|
||||
file_rows, _ = section.file_rows(self.import_file_config)
|
||||
for name in names:
|
||||
merged = sorted((i for i in db_rows if i[0] == name), key=lambda i: db_rows[i]['position'])
|
||||
previous = None
|
||||
|
|
|
|||
|
|
@ -550,20 +550,32 @@ class ConfigFile:
|
|||
return dict(self._config)
|
||||
|
||||
def initialize(self, multiprocessing_dict: Optional[DictProxy] = None):
|
||||
"""Initializes this config dict using the default config and the config file."""
|
||||
"""Initializes this config dict using the default config and the config file.
|
||||
|
||||
Every process initializes the shared dict, some while the others use it (a restarted perpetual process). Only
|
||||
the first loads it; the others keep what it holds, which may be newer than the file."""
|
||||
from wrolpi.api_utils import api_app
|
||||
|
||||
# Use the provided multiprocessing.Manager().dict() when live, or, dict() for testing.
|
||||
self._config = multiprocessing_dict if multiprocessing_dict is not None else dict()
|
||||
# Use the default settings to initialize the config.
|
||||
self._config.update(deepcopy(self.default_config))
|
||||
with getattr(api_app.shared_ctx, 'config_update_lock', None) or contextlib.nullcontext():
|
||||
if self._config:
|
||||
return self
|
||||
|
||||
file = self.get_file()
|
||||
if file.is_file():
|
||||
if not self.is_valid(file):
|
||||
raise InvalidConfig(f'Config file is invalid: {str(self.get_relative_file())}')
|
||||
# is_valid() already validated, just load the data
|
||||
config_data = self.read_config_file(file)
|
||||
config_data = {k: v for k, v in config_data.items() if k in self.default_config}
|
||||
self._config.update(config_data)
|
||||
# Use the default settings to initialize the config.
|
||||
config = deepcopy(self.default_config)
|
||||
try:
|
||||
file = self.get_file()
|
||||
if file.is_file():
|
||||
if not self.is_valid(file):
|
||||
raise InvalidConfig(f'Config file is invalid: {str(self.get_relative_file())}')
|
||||
# is_valid() already validated, just load the data
|
||||
config_data = self.read_config_file(file)
|
||||
config.update({k: v for k, v in config_data.items() if k in self.default_config})
|
||||
finally:
|
||||
# In one call: other processes never see the defaults in place of the file (an import would record
|
||||
# their version 0).
|
||||
self._config.update(config)
|
||||
return self
|
||||
|
||||
@property
|
||||
|
|
@ -859,11 +871,12 @@ class ConfigFile:
|
|||
data = self.read_config_file(file) if file.stat().st_size else None
|
||||
if not data or not self.validate(data):
|
||||
raise InvalidConfig(f'Config is {"invalid" if data else "empty"}: {file_str}')
|
||||
self._config.update({k: v for k, v in data.items() if k in self.default_config})
|
||||
config = {**deepcopy(self.default_config), **{k: v for k, v in data.items() if k in self.default_config}}
|
||||
self._config.update(config)
|
||||
|
||||
# Another writer may change the database between the snapshot and the write; then decide again.
|
||||
for _ in range(3):
|
||||
if self._import_sections_once(forced):
|
||||
if self._import_sections_once(config, forced):
|
||||
break
|
||||
else:
|
||||
raise RuntimeError(f'The database kept changing while importing {file_str}')
|
||||
|
|
@ -876,27 +889,32 @@ class ConfigFile:
|
|||
Events.send_config_import_failed(message)
|
||||
raise
|
||||
|
||||
def _import_sections_once(self, forced: bool) -> bool:
|
||||
"""One pass of `import_sections`. Returns False when the database's version changed before the write."""
|
||||
def _import_sections_once(self, config: dict, forced: bool) -> bool:
|
||||
"""One pass of `import_sections` of the file's `config`. Returns False when the database's version changed
|
||||
before the write.
|
||||
|
||||
It reads the file's `config`, not the shared `_config`, which other processes change meanwhile."""
|
||||
from wrolpi.config_import import diff_section
|
||||
from wrolpi.config_version import IMPORT, SAVE, get_config_version, set_config_version
|
||||
from wrolpi.db import get_db_session
|
||||
|
||||
decision, db_version = self.version_decision(forced)
|
||||
file_version = config['version'] or 0
|
||||
decision, db_version = self.version_decision(file_version, forced)
|
||||
if decision == SAVE:
|
||||
self.save_database_instead()
|
||||
return True
|
||||
|
||||
file_rows = {i.name: i.file_rows(self._config) for i in self.sections}
|
||||
file_rows = {i.name: i.file_rows(config) for i in self.sections}
|
||||
with get_db_session() as session:
|
||||
db_rows = {i.name: i.db_rows(session) for i in self.sections}
|
||||
diffs = {i.name: diff_section(*file_rows[i.name], db_rows[i.name], deletes=decision == IMPORT, same=i.same,
|
||||
present=i.present(self._config, *file_rows[i.name]))
|
||||
present=i.present(config, *file_rows[i.name]))
|
||||
for i in self.sections}
|
||||
# `prepare` may skip rows it cannot apply.
|
||||
self.import_skipped = sum(len(invalid) for _, invalid in file_rows.values())
|
||||
self.import_deletes = decision == IMPORT
|
||||
self.import_db_rows = db_rows
|
||||
self.import_file_config = config
|
||||
if any(diffs.values()):
|
||||
self.prepare(diffs)
|
||||
# A row `prepare` brought in line with the database (e.g. it dropped a Tag the file names but no longer
|
||||
|
|
@ -906,7 +924,6 @@ class ConfigFile:
|
|||
for key in [i for i in changed if section.same(changed[i], db_rows[section.name][i])]:
|
||||
del changed[key]
|
||||
|
||||
file_version = self.version or 0
|
||||
if any(diffs.values()) or (decision == IMPORT and db_version != file_version):
|
||||
try:
|
||||
with get_db_session(commit=True) as session:
|
||||
|
|
@ -946,6 +963,7 @@ class ConfigFile:
|
|||
differences. `diffs` maps each section's name to its `SectionDiff`; it may drop rows it cannot apply
|
||||
(count them in `import_skipped`), or add what it found to a row for `upsert`. `import_deletes`: whether
|
||||
this import deletes what the file no longer lists. `import_db_rows`: each section's database rows.
|
||||
`import_file_config`: the file's config (not the shared `_config`, which other processes change meanwhile).
|
||||
|
||||
It must be safe to repeat and to abandon: when the database changes before the write, the import decides
|
||||
again (up to three passes), and the diffs it was given may never be written (the write fails, or the next
|
||||
|
|
@ -977,15 +995,16 @@ class ConfigFile:
|
|||
from wrolpi.config_version import versioned_config_names
|
||||
return self.file_name in versioned_config_names()
|
||||
|
||||
def version_decision(self, forced: bool = False) -> Tuple[str, Optional[int]]:
|
||||
"""What importing this config's file (already read) may do (IMPORT, IMPORT_WITHOUT_DELETES, or SAVE), and the
|
||||
database version that decided it. `import_sections` records the file's version in its own write."""
|
||||
def version_decision(self, file_version: int, forced: bool = False) -> Tuple[str, Optional[int]]:
|
||||
"""What importing this config's file (already read, at `file_version`) may do (IMPORT, IMPORT_WITHOUT_DELETES,
|
||||
or SAVE), and the database version that decided it. `import_sections` records the file's version in its own
|
||||
write."""
|
||||
from wrolpi.config_version import IMPORT, decide, get_config_version
|
||||
from wrolpi.db import get_db_session
|
||||
with get_db_session() as session:
|
||||
db_version = get_config_version(session, self.file_name)
|
||||
decision = IMPORT if forced else decide(self.version or 0, db_version)
|
||||
logger_.debug(f'{self.file_name}: file version {self.version}, database version {db_version}: {decision}')
|
||||
decision = IMPORT if forced else decide(file_version, db_version)
|
||||
logger_.debug(f'{self.file_name}: file version {file_version}, database version {db_version}: {decision}')
|
||||
return decision, db_version
|
||||
|
||||
def save_database_instead(self):
|
||||
|
|
@ -1178,10 +1197,21 @@ class MultiFileConfig:
|
|||
return sorted(f.stem for f in directory.glob(self.glob_pattern) if f.is_file())
|
||||
|
||||
def initialize(self, multiprocessing_dict: Optional[DictProxy] = None):
|
||||
"""Assign the shared in-memory store, then import every entity file from disk."""
|
||||
self._configs = multiprocessing_dict if multiprocessing_dict is not None else dict()
|
||||
self._configs.clear()
|
||||
self.import_all()
|
||||
"""Assign the shared in-memory store, then import every entity file from disk.
|
||||
|
||||
Every process initializes the shared store, some while the others use it (a restarted perpetual process).
|
||||
Only the first imports the files; the others keep what it holds, which may be newer than the files."""
|
||||
configs = multiprocessing_dict if multiprocessing_dict is not None else dict()
|
||||
with self.save_lock():
|
||||
if not configs:
|
||||
# Import into this process's dict, then add the entities in one call.
|
||||
self._configs = dict()
|
||||
try:
|
||||
self.import_all()
|
||||
configs.update(self._configs)
|
||||
finally:
|
||||
self._configs = configs
|
||||
self._configs = configs
|
||||
return self
|
||||
|
||||
def is_valid(self, data: dict) -> bool:
|
||||
|
|
@ -1326,15 +1356,16 @@ class MultiFileConfig:
|
|||
|
||||
def create(self, slug: str, data: dict):
|
||||
"""Add a new entity and persist it immediately."""
|
||||
if slug in self._configs:
|
||||
raise ValueError(f'{self.name} already exists: {slug}')
|
||||
entity = deepcopy(self.default_entity)
|
||||
entity.update(data)
|
||||
entity['version'] = 0
|
||||
if not self.is_valid(entity):
|
||||
raise ValidationError(f'Invalid {self.name} config: {entity}')
|
||||
self._configs[slug] = entity
|
||||
self._save_slug(slug)
|
||||
with self.save_lock():
|
||||
if slug in self._configs:
|
||||
raise ValueError(f'{self.name} already exists: {slug}')
|
||||
self._configs[slug] = entity
|
||||
self._persist_slug(slug)
|
||||
|
||||
def replace(self, slug: str, data: dict):
|
||||
"""Replace an entity's config wholesale and persist it (background)."""
|
||||
|
|
|
|||
|
|
@ -131,6 +131,17 @@ def no_unexpected_queries(request):
|
|||
'\n'.join(f'{reason}: {" ".join(statement.split())[:300]}' for reason, statement in queries)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def restore_switch_handlers():
|
||||
"""Creating a config replaces the switch handlers of the global config with the same file (see
|
||||
`ConfigFile.__init__`); put them back after the test."""
|
||||
from wrolpi import switches
|
||||
handlers = dict(switches.SWITCH_HANDLERS)
|
||||
yield
|
||||
switches.SWITCH_HANDLERS.clear()
|
||||
switches.SWITCH_HANDLERS.update(handlers)
|
||||
|
||||
|
||||
def config_was_imported(session, *file_names: str):
|
||||
"""Record that these database configs were imported before, as on any WROLPi after its first startup with
|
||||
wrolpi.config_version, so their next import deletes what the file no longer lists. (Version 0 is no newer than
|
||||
|
|
|
|||
|
|
@ -268,13 +268,13 @@ class RecentlyViewedConfig(ConfigFile):
|
|||
|
||||
def prepare(self, diffs: dict):
|
||||
diff, db_rows = diffs['viewed'], self.import_db_rows['viewed']
|
||||
file_rows = ViewedSection().file_rows(self._config)[0]
|
||||
file_rows = ViewedSection().file_rows(self.import_file_config)[0]
|
||||
|
||||
# Only views at least as recent as the oldest entry of the file are removed: an older view fell off the end
|
||||
# of the saved window. A malformed entry keeps its file's history.
|
||||
if diff.removed:
|
||||
oldest = min(i['viewed'] for i in file_rows.values()) if file_rows else None
|
||||
malformed = _malformed_keys(self._config)
|
||||
malformed = _malformed_keys(self.import_file_config)
|
||||
diff.removed = {i for i in diff.removed
|
||||
if oldest and db_rows[i]['viewed'] >= oldest and i not in malformed}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import json
|
|||
import pathlib
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from copy import deepcopy
|
||||
from functools import partialmethod
|
||||
from itertools import zip_longest
|
||||
from typing import List
|
||||
|
|
@ -138,3 +139,57 @@ def assert_dict_contains(d1: dict, d2: dict):
|
|||
d1_str = json.dumps(d1, indent=1, cls=CustomJSONEncoder)
|
||||
d2_str = json.dumps(d2, indent=1, cls=CustomJSONEncoder)
|
||||
raise AssertionError(f'{d1_str}\n\n{d2_str}\n\nDictionaries above are unequal') from e
|
||||
|
||||
|
||||
def config_classes() -> list:
|
||||
"""Every config class: each ConfigFile, and each MultiFileConfig (a directory of files)."""
|
||||
from wrolpi.common import ConfigFile, MultiFileConfig
|
||||
|
||||
# Import every module that defines a config.
|
||||
import modules.archive.lib # noqa
|
||||
import modules.flasher.config # noqa
|
||||
import modules.inventory.catalog # noqa
|
||||
import modules.inventory.common # noqa
|
||||
import modules.map.pins # noqa
|
||||
import modules.repos.lib # noqa
|
||||
import modules.videos.lib # noqa
|
||||
import wrolpi.bookmarks # noqa
|
||||
import wrolpi.collections.config # noqa
|
||||
import wrolpi.downloader # noqa
|
||||
import wrolpi.files.progress # noqa
|
||||
import wrolpi.tags # noqa
|
||||
|
||||
return sorted(ConfigFile.__subclasses__() + MultiFileConfig.__subclasses__(), key=lambda i: i.__name__)
|
||||
|
||||
|
||||
class SharedDict(dict):
|
||||
"""Stands in for a `multiprocessing.Manager().dict()` the Sanic processes share: each call is atomic, and another
|
||||
process may read between any two calls. `readable` records what it could read."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.readable = []
|
||||
|
||||
def _changed(self):
|
||||
self.readable.append(deepcopy(dict(self)))
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
super().__setitem__(key, value)
|
||||
self._changed()
|
||||
|
||||
def __delitem__(self, key):
|
||||
super().__delitem__(key)
|
||||
self._changed()
|
||||
|
||||
def update(self, *args, **kwargs):
|
||||
super().update(*args, **kwargs)
|
||||
self._changed()
|
||||
|
||||
def clear(self):
|
||||
super().clear()
|
||||
self._changed()
|
||||
|
||||
def pop(self, *args):
|
||||
value = super().pop(*args)
|
||||
self._changed()
|
||||
return value
|
||||
|
|
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
"""A database config's file and the database each carry a version (see wrolpi.config_version)."""
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
import threading
|
||||
from copy import deepcopy
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
|
|
@ -7,6 +13,7 @@ from wrolpi.config_version import get_config_version, set_config_version
|
|||
from wrolpi.dates import now
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import Tag, get_tags_config, import_tags_config
|
||||
from wrolpi.test.common import config_classes
|
||||
|
||||
|
||||
def write_tags_config(path, version: int, names):
|
||||
|
|
@ -338,3 +345,81 @@ def test_domain_downloads_count_for_domains(test_session):
|
|||
test_session.commit()
|
||||
|
||||
assert db_version(test_session, 'domains.yaml') == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize('config_class', [i for i in config_classes() if getattr(i, 'sections', None)],
|
||||
ids=lambda i: i.__name__)
|
||||
@pytest.mark.asyncio
|
||||
async def test_import_while_another_process_initializes(test_session, test_directory, async_client,
|
||||
restore_switch_handlers, config_class, monkeypatch):
|
||||
"""The Sanic processes initialize the configs they share while one of them imports the database configs. The
|
||||
import records the version of the file it read, whatever another process does to the shared config meanwhile;
|
||||
otherwise the database's version goes back (e.g. to 0), and every later save is refused as older than the file."""
|
||||
config = config_class()
|
||||
file = config.get_file()
|
||||
file.parent.mkdir(parents=True, exist_ok=True)
|
||||
config.write_config_data({**deepcopy(config.default_config), 'version': 7}, file)
|
||||
shared = dict()
|
||||
config.initialize(shared)
|
||||
set_config_version(test_session, config.file_name, 7)
|
||||
test_session.commit()
|
||||
# Configs refuse to import while testing unless they are the test's config.
|
||||
monkeypatch.setattr('wrolpi.tags.TEST_TAGS_CONFIG', config)
|
||||
monkeypatch.setattr('modules.videos.lib.TEST_CHANNELS_CONFIG', config)
|
||||
|
||||
# Another process initializes this config, and pauses if it reads the file.
|
||||
other_process = threading.Thread(target=config.initialize, args=(shared,))
|
||||
paused, resume = threading.Event(), threading.Event()
|
||||
read_config_file = config.read_config_file
|
||||
|
||||
def pausing_read_config_file(*args, **kwargs):
|
||||
if threading.current_thread() is other_process:
|
||||
paused.set()
|
||||
resume.wait(10)
|
||||
return read_config_file(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(config, 'read_config_file', pausing_read_config_file)
|
||||
|
||||
# It starts after this import has decided what to do, and before it writes.
|
||||
version_decision = config.version_decision
|
||||
|
||||
def racing_version_decision(*args, **kwargs):
|
||||
decision = version_decision(*args, **kwargs)
|
||||
other_process.start()
|
||||
while other_process.is_alive() and not paused.wait(0.01):
|
||||
pass
|
||||
return decision
|
||||
|
||||
monkeypatch.setattr(config, 'version_decision', racing_version_decision)
|
||||
try:
|
||||
config.import_config()
|
||||
finally:
|
||||
resume.set()
|
||||
if other_process.is_alive():
|
||||
other_process.join(10)
|
||||
|
||||
assert config.successful_import
|
||||
assert db_version(test_session, config.file_name) == 7
|
||||
|
||||
|
||||
@pytest.mark.parametrize('config_class', [i for i in config_classes() if getattr(i, 'sections', None)],
|
||||
ids=lambda i: i.__name__)
|
||||
def test_import_steps_read_the_file(config_class):
|
||||
"""`prepare` and `after_import` (and the methods they call) read the file's config (`import_file_config`), never
|
||||
the shared `_config`, which other processes change during the import."""
|
||||
tree = ast.parse(textwrap.dedent(inspect.getsource(config_class)))
|
||||
methods = {i.name: i for i in tree.body[0].body if isinstance(i, ast.FunctionDef)}
|
||||
properties = {name for name, method in methods.items()
|
||||
if any(getattr(i, 'id', None) == 'property' for i in method.decorator_list)}
|
||||
reads, seen, todo = set(), set(), [i for i in ('prepare', 'after_import') if i in methods]
|
||||
while todo:
|
||||
if (name := todo.pop()) in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
for node in ast.walk(methods[name]):
|
||||
if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id == 'self':
|
||||
if node.attr in methods and node.attr not in properties:
|
||||
todo.append(node.attr)
|
||||
elif node.attr in properties | {'_config', 'version'}:
|
||||
reads.add(f'{name} reads self.{node.attr}')
|
||||
assert not reads
|
||||
|
|
|
|||
|
|
@ -1,4 +1,22 @@
|
|||
"""Tests for shared context initialization."""
|
||||
from copy import deepcopy
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.common import MultiFileConfig, write_config_data
|
||||
from wrolpi.test.common import SharedDict, config_classes
|
||||
|
||||
|
||||
def write_config_files(config, version: int):
|
||||
"""Write a valid config file (or, for a MultiFileConfig, one entity's file) at `version`."""
|
||||
if isinstance(config, MultiFileConfig):
|
||||
file = config.get_file('one')
|
||||
data = {**deepcopy(config.default_entity), 'version': version, 'slug': 'one', 'name': 'One'}
|
||||
else:
|
||||
file = config.get_file()
|
||||
data = {**deepcopy(config.default_config), 'version': version}
|
||||
file.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_config_data(data, file)
|
||||
|
||||
|
||||
def test_domains_config_in_shared_context():
|
||||
|
|
@ -56,3 +74,29 @@ def test_all_configs_in_shared_context():
|
|||
not_shared = sorted(name for name, config in get_all_configs().items()
|
||||
if not isinstance(getattr(config, '_config', None), DictProxy))
|
||||
assert not_shared == [], 'Add these configs to attach_shared_contexts() and initialize_configs_contexts()'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('config_class', config_classes(), ids=lambda i: i.__name__)
|
||||
def test_initialize_keeps_the_shared_config(test_directory, restore_switch_handlers, config_class):
|
||||
"""Each Sanic process initializes the configs it shares with the others, some while the others already serve,
|
||||
import and save them (a restarted perpetual process). Initializing must never show the others anything but what
|
||||
the shared config holds: a config which showed its defaults (version 0) for a moment had that version recorded by
|
||||
a concurrent import, and one which went back to its file lost another process's newer change."""
|
||||
config = config_class()
|
||||
write_config_files(config, 7)
|
||||
shared = SharedDict()
|
||||
config.initialize(shared) # The first process.
|
||||
assert shared and shared.readable
|
||||
|
||||
# Another process changes the config, and has not saved it yet.
|
||||
if isinstance(config, MultiFileConfig):
|
||||
shared['one'] = {**shared['one'], 'name': 'Renamed'}
|
||||
shared['two'] = {**shared['one'], 'slug': 'two', 'name': 'Two'}
|
||||
else:
|
||||
shared['version'] = 8
|
||||
changed = deepcopy(dict(shared))
|
||||
|
||||
shared.readable.clear()
|
||||
config.initialize(shared) # Another process starts.
|
||||
assert dict(shared) == changed
|
||||
assert all(i == changed for i in shared.readable), 'Another process could read something else'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue