mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 15:36:18 -05:00
Browser-based ESP32/ESP8266 flasher (esptool-js + Web Serial); nothing is uploaded to WROLPi, the flash is written straight to the device over USB. Firmware sources (themed, tabbed): a saved set, the WROLPi media directory, or a local file. Local files can be dropped anywhere on the page, which jumps to the "Add from computer" dropzone. The media picker searches .bin files by path and can detect the connected chip to filter firmware to matching images. - Indexed file_group.suffix + path filter for fast .bin search (alembic migration backfills via split_path_stem_and_suffix) - modules/flasher: chip detection from ESP image headers, /search endpoint, and flasher.yaml saved configurations (/configs CRUD) - Flash progress modal with ETA, connect timeout, boot-mode hint, auto- scrolling logs Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
286 lines
10 KiB
Python
286 lines
10 KiB
Python
import ctypes
|
|
import multiprocessing
|
|
import queue
|
|
|
|
from sanic import Sanic
|
|
|
|
from wrolpi.common import logger
|
|
from wrolpi.vars import LOG_LEVEL_INT
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
|
|
def attach_shared_contexts(app: Sanic):
|
|
"""Initializes Sanic's shared context with WROLPi's multiprocessing tools.
|
|
|
|
This is called by main.py, and by testing."""
|
|
manager = multiprocessing.Manager()
|
|
|
|
# Many things wait for flags.db_up, initialize before starting.
|
|
from wrolpi import flags
|
|
|
|
app.shared_ctx.flags = manager.dict({i: False for i in flags.FLAG_NAMES})
|
|
|
|
# ConfigFile multiprocessing_dict's.
|
|
# Shared Configs
|
|
app.shared_ctx.wrolpi_config = manager.dict()
|
|
app.shared_ctx.tags_config = manager.dict()
|
|
app.shared_ctx.inventories_config = manager.dict()
|
|
app.shared_ctx.catalog_config = manager.dict()
|
|
app.shared_ctx.channels_config = manager.dict()
|
|
app.shared_ctx.download_manager_config = manager.dict()
|
|
app.shared_ctx.videos_downloader_config = manager.dict()
|
|
app.shared_ctx.domains_config = manager.dict()
|
|
app.shared_ctx.archive_downloader_config = manager.dict()
|
|
app.shared_ctx.download_cache_config = manager.dict()
|
|
app.shared_ctx.map_pins_config = manager.dict()
|
|
app.shared_ctx.flasher_config = manager.dict()
|
|
# Shared dicts.
|
|
app.shared_ctx.refresh = manager.dict()
|
|
app.shared_ctx.uploaded_files = manager.dict()
|
|
app.shared_ctx.status = manager.dict()
|
|
app.shared_ctx.cache = manager.dict()
|
|
# Secure cookies shared storage
|
|
app.shared_ctx.secure_cookies = manager.dict()
|
|
app.shared_ctx.secure_cookies_lock = multiprocessing.Lock()
|
|
# Shared ints
|
|
app.shared_ctx.log_level = multiprocessing.Value(ctypes.c_int, LOG_LEVEL_INT)
|
|
|
|
# Download timing for per-domain rate limiting
|
|
app.shared_ctx.domain_last_download = manager.dict()
|
|
|
|
# Download Manager
|
|
app.shared_ctx.download_manager_data = manager.dict()
|
|
app.shared_ctx.download_manager_disabled = multiprocessing.Event()
|
|
app.shared_ctx.download_manager_stopped = multiprocessing.Event()
|
|
# Downloads are only begun on startup using the wrolpi config's value. Do not start downloads before importing
|
|
# config.
|
|
app.shared_ctx.download_manager_disabled.set()
|
|
app.shared_ctx.download_manager_stopped.set()
|
|
|
|
app.shared_ctx.single_tasks_started = multiprocessing.Event()
|
|
app.shared_ctx.flags_initialized = multiprocessing.Event()
|
|
app.shared_ctx.perpetual_tasks_started = multiprocessing.Event()
|
|
|
|
# Switches
|
|
app.shared_ctx.switches = manager.dict()
|
|
app.shared_ctx.switches_lock = multiprocessing.Lock()
|
|
app.shared_ctx.switches_changed = multiprocessing.Event()
|
|
app.shared_ctx.archive_singlefiles = multiprocessing.Queue()
|
|
app.shared_ctx.archive_screenshots = multiprocessing.Queue()
|
|
|
|
# Bulk tagging
|
|
app.shared_ctx.bulk_tag = manager.dict()
|
|
app.shared_ctx.bulk_tag_queue = manager.Queue()
|
|
|
|
# Warnings
|
|
app.shared_ctx.warn_once = manager.dict()
|
|
|
|
# Configs
|
|
app.shared_ctx.config_save_lock = multiprocessing.Lock()
|
|
app.shared_ctx.config_update_lock = multiprocessing.Lock()
|
|
app.shared_ctx.configs_imported = manager.dict()
|
|
|
|
# Events.
|
|
app.shared_ctx.events_lock = multiprocessing.Lock()
|
|
app.shared_ctx.events_history = manager.list()
|
|
|
|
app.shared_ctx.file_worker_public_queue = manager.Queue()
|
|
app.shared_ctx.file_worker_status = manager.dict()
|
|
app.shared_ctx.file_worker_jobs = manager.dict()
|
|
|
|
reset_shared_contexts(app)
|
|
|
|
|
|
def reset_shared_contexts(app: Sanic):
|
|
"""Resets shared contexts (dicts/lists/Events,etc.).
|
|
|
|
Should only be called when server is starting, or could start back up."""
|
|
# Should only be called when server is expected to start again.
|
|
app.shared_ctx.wrolpi_config.clear()
|
|
app.shared_ctx.tags_config.clear()
|
|
app.shared_ctx.inventories_config.clear()
|
|
app.shared_ctx.catalog_config.clear()
|
|
app.shared_ctx.channels_config.clear()
|
|
app.shared_ctx.download_manager_config.clear()
|
|
app.shared_ctx.videos_downloader_config.clear()
|
|
app.shared_ctx.domains_config.clear()
|
|
app.shared_ctx.archive_downloader_config.clear()
|
|
app.shared_ctx.download_cache_config.clear()
|
|
# Shared dicts.
|
|
app.shared_ctx.refresh.clear()
|
|
app.shared_ctx.uploaded_files.clear()
|
|
app.shared_ctx.status.clear()
|
|
app.shared_ctx.status.update(dict(
|
|
cpu_stats=dict(),
|
|
load_stats=dict(),
|
|
drives_stats=list(),
|
|
processes_stats=list(),
|
|
memory_stats=dict(),
|
|
# Upgrade info defaults
|
|
update_available=False,
|
|
latest_commit=None,
|
|
current_commit=None,
|
|
commits_behind=0,
|
|
git_branch=None,
|
|
))
|
|
app.shared_ctx.cache.clear()
|
|
# Secure cookies
|
|
app.shared_ctx.secure_cookies.clear()
|
|
# Shared ints
|
|
app.shared_ctx.log_level.value = LOG_LEVEL_INT
|
|
|
|
# Download timing
|
|
app.shared_ctx.domain_last_download.clear()
|
|
|
|
# Download Manager
|
|
app.shared_ctx.download_manager_data.clear()
|
|
app.shared_ctx.download_manager_data.update(dict(
|
|
processing_domains=[],
|
|
killed_downloads=[],
|
|
download_progress={},
|
|
))
|
|
|
|
# Configs
|
|
app.shared_ctx.configs_imported.clear()
|
|
|
|
# Switches
|
|
app.shared_ctx.switches.clear()
|
|
app.shared_ctx.switches_changed.clear()
|
|
while True:
|
|
# Clear out any pending singlefile archive switches.
|
|
try:
|
|
app.shared_ctx.archive_singlefiles.get_nowait()
|
|
except queue.Empty:
|
|
break
|
|
while True:
|
|
# Clear out any pending screenshot generation switches.
|
|
try:
|
|
app.shared_ctx.archive_screenshots.get_nowait()
|
|
except queue.Empty:
|
|
break
|
|
|
|
# Bulk tagging
|
|
app.shared_ctx.bulk_tag.clear()
|
|
app.shared_ctx.bulk_tag.update(dict(
|
|
status='idle', # 'idle', 'running'
|
|
total=0,
|
|
completed=0,
|
|
add_tag_names=[],
|
|
remove_tag_names=[],
|
|
error=None,
|
|
))
|
|
while True:
|
|
# Clear out any pending bulk tag jobs.
|
|
try:
|
|
app.shared_ctx.bulk_tag_queue.get_nowait()
|
|
except queue.Empty:
|
|
break
|
|
|
|
# File Worker
|
|
while True:
|
|
# Clear out any pending file tasks.
|
|
try:
|
|
app.shared_ctx.file_worker_public_queue.get_nowait()
|
|
except queue.Empty:
|
|
break
|
|
app.shared_ctx.file_worker_status.clear()
|
|
app.shared_ctx.file_worker_jobs.clear()
|
|
app.shared_ctx.file_worker_status.update(dict(
|
|
status='idle',
|
|
task_type=None,
|
|
paths=[],
|
|
destination=None,
|
|
error=None,
|
|
operation_total=0,
|
|
operation_processed=0,
|
|
operation_percent=0,
|
|
))
|
|
|
|
# Events.
|
|
# Clear events_history in place to preserve references held by fixtures
|
|
del app.shared_ctx.events_history[:]
|
|
app.shared_ctx.single_tasks_started.clear()
|
|
app.shared_ctx.flags_initialized.clear()
|
|
app.shared_ctx.perpetual_tasks_started.clear()
|
|
|
|
# Do not start downloads when reloading.
|
|
app.shared_ctx.download_manager_stopped.set()
|
|
app.shared_ctx.download_manager_disabled.set()
|
|
|
|
|
|
def initialize_configs_contexts(app: Sanic):
|
|
"""Assign multiprocessing Dicts to their respective FileConfigs in this process."""
|
|
from modules.inventory.common import get_inventory_configs
|
|
from modules.videos.lib import CHANNELS_CONFIG
|
|
from modules.videos.lib import VIDEOS_DOWNLOADER_CONFIG
|
|
from wrolpi.common import WROLPI_CONFIG
|
|
from wrolpi.tags import TAGS_CONFIG
|
|
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
|
|
|
|
try: # noqa
|
|
# Inventory is config-only (no DB); the shared dict is keyed by inventory slug.
|
|
get_inventory_configs().initialize(app.shared_ctx.inventories_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory inventories config: {e}')
|
|
|
|
try:
|
|
from modules.inventory.catalog import get_catalog_config
|
|
get_catalog_config().initialize(app.shared_ctx.catalog_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory food catalog config: {e}')
|
|
|
|
try:
|
|
CHANNELS_CONFIG.initialize(app.shared_ctx.channels_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory channels config: {e}')
|
|
|
|
try:
|
|
VIDEOS_DOWNLOADER_CONFIG.initialize(app.shared_ctx.videos_downloader_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory videos downloader config: {e}')
|
|
|
|
try: # noqa
|
|
WROLPI_CONFIG.initialize(app.shared_ctx.wrolpi_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory wrolpi config config: {e}')
|
|
|
|
try:
|
|
TAGS_CONFIG.initialize(app.shared_ctx.tags_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory tags config: {e}')
|
|
|
|
try:
|
|
DOWNLOAD_MANAGER_CONFIG.initialize(app.shared_ctx.download_manager_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory download manager config: {e}')
|
|
|
|
try:
|
|
from modules.archive.lib import domains_config
|
|
domains_config.initialize(app.shared_ctx.domains_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory domains config: {e}')
|
|
|
|
try:
|
|
from modules.archive.lib import ARCHIVE_DOWNLOADER_CONFIG
|
|
ARCHIVE_DOWNLOADER_CONFIG.initialize(app.shared_ctx.archive_downloader_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory archive downloader config: {e}')
|
|
|
|
try:
|
|
from wrolpi.downloader import DOWNLOAD_CACHE_CONFIG
|
|
DOWNLOAD_CACHE_CONFIG.initialize(app.shared_ctx.download_cache_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory download cache config: {e}')
|
|
|
|
try:
|
|
from modules.map.pins import MAP_PINS_CONFIG
|
|
MAP_PINS_CONFIG.initialize(app.shared_ctx.map_pins_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory map pins config: {e}')
|
|
|
|
try:
|
|
from modules.flasher.config import FLASHER_CONFIG
|
|
FLASHER_CONFIG.initialize(app.shared_ctx.flasher_config)
|
|
except Exception as e:
|
|
logger.error(f'Failed to initialize in-memory flasher config: {e}')
|