mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
* bookmarks: user bookmarks in the navigation bar, stored in bookmarks.yaml A "Bookmarks" tab between Calculators and Flasher opens on hover or click and lists the user's bookmarks, with directories as flyout submenus (Menu.Sub). The mobile hamburger shows directories as labelled groups instead. The tab always shows, with an "Edit bookmarks" item, so the feature can be found before any bookmark exists. Bookmarks live in a new config-only bookmarks.yaml: a nested tree where a node with `children` is a directory and a node with `url` is a bookmark. Every node carries an integer id so the API can rename, move and delete without relying on names. A URL may be a path on this WROLPi, a port on this host (`:8096/` or `http://:8096/`, resolved against whatever hostname the browser used, so it works over LAN IP, mDNS and the hotspot alike), or an absolute URL. Each bookmark chooses whether to open in a new tab. The editor at /more/bookmarks follows the playlist items table: numbered rows, up/down/edit/delete, and inline "Add a bookmark" / "Add a directory" forms with a Location picker. Deleting a directory asks first, since its contents go with it. The menu and the editor are in the component gallery. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * bookmarks: mark More current when the editor is folded inside it; rem indents in the mobile menu Found on the QA Pi at a width where Bookmarks overflows into More: the More button carried no mark on /more/bookmarks, because the bookmarks link has no `to` for isLinkActive to match. isLinkActive now knows the bookmarks link owns its editor page, and the Bookmarks tab uses the same rule. The mobile menu's directory labels sat a few pixels left of the labels around them: the indent was in em, and a Menu.Label is set smaller than a Menu.Item. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * bookmarks: parse URLs instead of matching a prefix; validate the whole tree on every write The URL contract was checked by prefix on both sides, and a prefix is not a form: `//evil.com` starts like a path and `:8096@evil.com` starts like a port, and a browser takes both off this host. A path is now a single leading slash that resolves to this origin, a port form is an optional scheme, `:`, digits and then only a path, query or fragment (no `@`, no backslash, no control characters), and anything else must parse as http(s). The same rules live in wrolpi/bookmarks.py and BookmarksContext.js, with the same bypass strings in both test suites. The config's dataclass validator only saw that `bookmarks` was a list, so a hand-edited file or the generic config API could store a `javascript:` URL that the bookmarks API would refuse, and the nav rendered it. The config now validates the tree itself on every write and import: URL forms, exactly one of url/children, unique integer ids, boolean new_tab, and no cycles (a YAML anchor can build one). An invalid file is not imported and, since it was never successfully imported, is never overwritten. On the page, a stored URL that does not resolve is rendered as text, never as an href. A refused add keeps what was typed instead of clearing the form, and an edit whose move is refused still refreshes the nav (the rename was committed) and keeps the modal open. An IPv6 host is bracketed before a port. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * help: document Bookmarks Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
326 lines
12 KiB
Python
326 lines
12 KiB
Python
import ctypes
|
|
import multiprocessing
|
|
import queue
|
|
|
|
from sanic import Sanic
|
|
|
|
from wrolpi.common import logger
|
|
from wrolpi.vars import LOG_LEVEL_INT
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
|
|
def attach_shared_contexts(app: Sanic):
|
|
"""Initializes Sanic's shared context with WROLPi's multiprocessing tools.
|
|
|
|
This is called by main.py, and by testing."""
|
|
manager = multiprocessing.Manager()
|
|
|
|
# Many things wait for flags.db_up, initialize before starting.
|
|
from wrolpi import flags
|
|
|
|
app.shared_ctx.flags = manager.dict({i: False for i in flags.FLAG_NAMES})
|
|
# Flag name -> pid of the process inside `with flag:` (see Flag.holder_pid).
|
|
app.shared_ctx.flag_holders = manager.dict()
|
|
|
|
# ConfigFile multiprocessing_dict's.
|
|
# Shared Configs
|
|
app.shared_ctx.wrolpi_config = manager.dict()
|
|
app.shared_ctx.tags_config = manager.dict()
|
|
app.shared_ctx.inventories_config = manager.dict()
|
|
app.shared_ctx.catalog_config = manager.dict()
|
|
app.shared_ctx.channels_config = manager.dict()
|
|
app.shared_ctx.download_manager_config = manager.dict()
|
|
app.shared_ctx.videos_downloader_config = manager.dict()
|
|
app.shared_ctx.domains_config = manager.dict()
|
|
app.shared_ctx.archive_downloader_config = manager.dict()
|
|
app.shared_ctx.download_cache_config = manager.dict()
|
|
app.shared_ctx.map_pins_config = manager.dict()
|
|
app.shared_ctx.flasher_config = manager.dict()
|
|
app.shared_ctx.bookmarks_config = manager.dict()
|
|
# Shared dicts.
|
|
app.shared_ctx.uploaded_files = manager.dict()
|
|
app.shared_ctx.status = manager.dict()
|
|
app.shared_ctx.cache = manager.dict()
|
|
# Secure cookies shared storage
|
|
app.shared_ctx.secure_cookies = manager.dict()
|
|
app.shared_ctx.secure_cookies_lock = multiprocessing.Lock()
|
|
# Shared ints
|
|
app.shared_ctx.log_level = multiprocessing.Value(ctypes.c_int, LOG_LEVEL_INT)
|
|
|
|
# Download timing for per-domain rate limiting
|
|
app.shared_ctx.domain_last_download = manager.dict()
|
|
|
|
# Download Manager
|
|
app.shared_ctx.download_manager_data = manager.dict()
|
|
app.shared_ctx.download_manager_disabled = multiprocessing.Event()
|
|
app.shared_ctx.download_manager_stopped = multiprocessing.Event()
|
|
# Downloads are only begun on startup using the wrolpi config's value. Do not start downloads before importing
|
|
# config.
|
|
app.shared_ctx.download_manager_disabled.set()
|
|
app.shared_ctx.download_manager_stopped.set()
|
|
|
|
app.shared_ctx.single_tasks_started = multiprocessing.Event()
|
|
app.shared_ctx.flags_initialized = multiprocessing.Event()
|
|
|
|
# Transcode
|
|
# Transcoding saturates every core; only one ffmpeg transcode may run machine-wide,
|
|
# across all Sanic workers. If the holding worker dies (SIGKILL/crash) the lock stays
|
|
# held until the API restarts; that is accepted, transcoding is not critical work.
|
|
app.shared_ctx.transcode_lock = multiprocessing.Lock()
|
|
|
|
# Speed tests in flight across all workers (wrolpi.speedtest). Informational only, no lock.
|
|
app.shared_ctx.speedtest_active = multiprocessing.Value(ctypes.c_int, 0)
|
|
|
|
# When any server worker last asked the manager to restart the perpetual process (wrolpi/perpetual.py).
|
|
# Shared so the N workers issue one request per cooldown, not N.
|
|
app.shared_ctx.perpetual_restart_requested_at = multiprocessing.Value(ctypes.c_double, 0.0)
|
|
|
|
# Switches
|
|
app.shared_ctx.switches = manager.dict()
|
|
app.shared_ctx.switches_lock = multiprocessing.Lock()
|
|
app.shared_ctx.switches_changed = multiprocessing.Event()
|
|
app.shared_ctx.archive_singlefiles = multiprocessing.Queue()
|
|
app.shared_ctx.archive_screenshots = multiprocessing.Queue()
|
|
|
|
# Bulk tagging
|
|
app.shared_ctx.bulk_tag = manager.dict()
|
|
app.shared_ctx.bulk_tag_queue = manager.Queue()
|
|
|
|
# Warnings
|
|
app.shared_ctx.warn_once = manager.dict()
|
|
|
|
# Configs
|
|
app.shared_ctx.config_save_lock = multiprocessing.Lock()
|
|
app.shared_ctx.config_update_lock = multiprocessing.Lock()
|
|
app.shared_ctx.configs_imported = manager.dict()
|
|
# Count of config entries skipped during each config's import (blocks dumps until re-imported cleanly).
|
|
app.shared_ctx.configs_import_skipped = manager.dict()
|
|
|
|
# Events.
|
|
app.shared_ctx.events_lock = multiprocessing.Lock()
|
|
app.shared_ctx.events_history = manager.list()
|
|
|
|
app.shared_ctx.file_worker_public_queue = manager.Queue()
|
|
app.shared_ctx.file_worker_status = manager.dict()
|
|
app.shared_ctx.file_worker_jobs = manager.dict()
|
|
|
|
# Jobs (wrolpi/jobs.py): FIFO queue of registered functions run one at a time by the
|
|
# perpetual process. Never persisted; an API restart loses them.
|
|
app.shared_ctx.jobs = manager.dict()
|
|
app.shared_ctx.jobs_queue = manager.Queue()
|
|
app.shared_ctx.jobs_lock = multiprocessing.Lock()
|
|
|
|
reset_shared_contexts(app)
|
|
|
|
|
|
def reset_shared_contexts(app: Sanic):
|
|
"""Resets shared contexts (dicts/lists/Events,etc.).
|
|
|
|
Should only be called when server is starting, or could start back up."""
|
|
# Should only be called when server is expected to start again.
|
|
app.shared_ctx.wrolpi_config.clear()
|
|
app.shared_ctx.tags_config.clear()
|
|
app.shared_ctx.inventories_config.clear()
|
|
app.shared_ctx.catalog_config.clear()
|
|
app.shared_ctx.channels_config.clear()
|
|
app.shared_ctx.download_manager_config.clear()
|
|
app.shared_ctx.videos_downloader_config.clear()
|
|
app.shared_ctx.domains_config.clear()
|
|
app.shared_ctx.archive_downloader_config.clear()
|
|
app.shared_ctx.download_cache_config.clear()
|
|
# Shared dicts.
|
|
app.shared_ctx.uploaded_files.clear()
|
|
app.shared_ctx.status.clear()
|
|
app.shared_ctx.speedtest_active.value = 0
|
|
app.shared_ctx.status.update(dict(
|
|
cpu_stats=dict(),
|
|
load_stats=dict(),
|
|
drives_stats=list(),
|
|
processes_stats=list(),
|
|
memory_stats=dict(),
|
|
# Upgrade info defaults
|
|
update_available=False,
|
|
latest_commit=None,
|
|
current_commit=None,
|
|
commits_behind=0,
|
|
git_branch=None,
|
|
))
|
|
app.shared_ctx.cache.clear()
|
|
# Secure cookies
|
|
app.shared_ctx.secure_cookies.clear()
|
|
# Shared ints
|
|
app.shared_ctx.log_level.value = LOG_LEVEL_INT
|
|
|
|
# Download timing
|
|
app.shared_ctx.domain_last_download.clear()
|
|
|
|
# Download Manager
|
|
app.shared_ctx.download_manager_data.clear()
|
|
app.shared_ctx.download_manager_data.update(dict(
|
|
processing_domains=[],
|
|
killed_downloads=[],
|
|
download_progress={},
|
|
))
|
|
|
|
# Configs
|
|
app.shared_ctx.configs_imported.clear()
|
|
app.shared_ctx.configs_import_skipped.clear()
|
|
|
|
# Switches
|
|
app.shared_ctx.switches.clear()
|
|
app.shared_ctx.switches_changed.clear()
|
|
while True:
|
|
# Clear out any pending singlefile archive switches.
|
|
try:
|
|
app.shared_ctx.archive_singlefiles.get_nowait()
|
|
except queue.Empty:
|
|
break
|
|
while True:
|
|
# Clear out any pending screenshot generation switches.
|
|
try:
|
|
app.shared_ctx.archive_screenshots.get_nowait()
|
|
except queue.Empty:
|
|
break
|
|
|
|
# Bulk tagging
|
|
app.shared_ctx.bulk_tag.clear()
|
|
app.shared_ctx.bulk_tag.update(dict(
|
|
status='idle', # 'idle', 'running'
|
|
total=0,
|
|
completed=0,
|
|
add_tag_names=[],
|
|
remove_tag_names=[],
|
|
error=None,
|
|
))
|
|
while True:
|
|
# Clear out any pending bulk tag jobs.
|
|
try:
|
|
app.shared_ctx.bulk_tag_queue.get_nowait()
|
|
except queue.Empty:
|
|
break
|
|
|
|
# File Worker
|
|
while True:
|
|
# Clear out any pending file tasks.
|
|
try:
|
|
app.shared_ctx.file_worker_public_queue.get_nowait()
|
|
except queue.Empty:
|
|
break
|
|
app.shared_ctx.file_worker_status.clear()
|
|
app.shared_ctx.file_worker_jobs.clear()
|
|
|
|
# Jobs
|
|
while True:
|
|
try:
|
|
app.shared_ctx.jobs_queue.get_nowait()
|
|
except queue.Empty:
|
|
break
|
|
app.shared_ctx.jobs.clear()
|
|
app.shared_ctx.file_worker_status.update(dict(
|
|
status='idle',
|
|
task_type=None,
|
|
paths=[],
|
|
destination=None,
|
|
error=None,
|
|
operation_total=0,
|
|
operation_processed=0,
|
|
operation_percent=0,
|
|
))
|
|
|
|
# Events.
|
|
# Clear events_history in place to preserve references held by fixtures
|
|
del app.shared_ctx.events_history[:]
|
|
app.shared_ctx.single_tasks_started.clear()
|
|
app.shared_ctx.flags_initialized.clear()
|
|
if hasattr(app.shared_ctx, 'flag_holders'):
|
|
app.shared_ctx.flag_holders.clear()
|
|
if hasattr(app.shared_ctx, 'perpetual_restart_requested_at'):
|
|
app.shared_ctx.perpetual_restart_requested_at.value = 0.0
|
|
|
|
# Do not start downloads when reloading.
|
|
app.shared_ctx.download_manager_stopped.set()
|
|
app.shared_ctx.download_manager_disabled.set()
|
|
|
|
|
|
def initialize_configs_contexts(app: Sanic):
|
|
"""Assign multiprocessing Dicts to their respective FileConfigs in this process."""
|
|
from modules.inventory.common import get_inventory_configs
|
|
from modules.videos.lib import CHANNELS_CONFIG
|
|
from modules.videos.lib import VIDEOS_DOWNLOADER_CONFIG
|
|
from wrolpi.common import WROLPI_CONFIG
|
|
from wrolpi.tags import TAGS_CONFIG
|
|
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
|
|
|
|
try: # noqa
|
|
# Inventory is config-only (no DB); the shared dict is keyed by inventory slug.
|
|
get_inventory_configs().initialize(app.shared_ctx.inventories_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory inventories config: {e}')
|
|
|
|
try:
|
|
from modules.inventory.catalog import get_catalog_config
|
|
get_catalog_config().initialize(app.shared_ctx.catalog_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory food catalog config: {e}')
|
|
|
|
try:
|
|
CHANNELS_CONFIG.initialize(app.shared_ctx.channels_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory channels config: {e}')
|
|
|
|
try:
|
|
VIDEOS_DOWNLOADER_CONFIG.initialize(app.shared_ctx.videos_downloader_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory videos downloader config: {e}')
|
|
|
|
try: # noqa
|
|
WROLPI_CONFIG.initialize(app.shared_ctx.wrolpi_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory wrolpi config config: {e}')
|
|
|
|
try:
|
|
TAGS_CONFIG.initialize(app.shared_ctx.tags_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory tags config: {e}')
|
|
|
|
try:
|
|
DOWNLOAD_MANAGER_CONFIG.initialize(app.shared_ctx.download_manager_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory download manager config: {e}')
|
|
|
|
try:
|
|
from modules.archive.lib import domains_config
|
|
domains_config.initialize(app.shared_ctx.domains_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory domains config: {e}')
|
|
|
|
try:
|
|
from modules.archive.lib import ARCHIVE_DOWNLOADER_CONFIG
|
|
ARCHIVE_DOWNLOADER_CONFIG.initialize(app.shared_ctx.archive_downloader_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory archive downloader config: {e}')
|
|
|
|
try:
|
|
from wrolpi.downloader import DOWNLOAD_CACHE_CONFIG
|
|
DOWNLOAD_CACHE_CONFIG.initialize(app.shared_ctx.download_cache_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory download cache config: {e}')
|
|
|
|
try:
|
|
from modules.map.pins import MAP_PINS_CONFIG
|
|
MAP_PINS_CONFIG.initialize(app.shared_ctx.map_pins_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory map pins config: {e}')
|
|
|
|
try:
|
|
from modules.flasher.config import FLASHER_CONFIG
|
|
FLASHER_CONFIG.initialize(app.shared_ctx.flasher_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory flasher config: {e}')
|
|
|
|
try:
|
|
from wrolpi.bookmarks import BOOKMARKS_CONFIG
|
|
BOOKMARKS_CONFIG.initialize(app.shared_ctx.bookmarks_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory bookmarks config: {e}')
|