From 2cdb2a4b3e29282e533f5da017b34ac74bf793fa Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 28 Jun 2026 16:38:43 -0600 Subject: [PATCH 01/15] Fix dark-theme Accordion background mismatch outside modals The dark-theme rule hardcoded the inverted Accordion's title/content background to #171616, which is the Modal content color. Inside a Modal this matched, but on the /admin/status page (and any Segment, which renders at #1B1C1D) the Accordion showed up as a darker band. Make the inverted Accordion transparent so it inherits whatever container it sits in, matching how the light theme already renders accordions. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/src/App.css | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/src/App.css b/app/src/App.css index 6ca76aae..822987dd 100644 --- a/app/src/App.css +++ b/app/src/App.css @@ -247,11 +247,15 @@ textarea.otp { color: #DDDDDD !important; } -/* Direct children only, so nested components that use the `content`/`title` class names (e.g. a Message's - Message.Content) are not given the accordion's dark background. */ +/* The accordion has no background of its own; it inherits whatever container it sits in (Modal + content #171616, Segment #1B1C1D, the page body, etc.). This matches how the light theme already + renders accordions (transparent), and avoids a darker band when the parent isn't #171616. + Direct children only, so nested components that use the `content`/`title` class names (e.g. a + Message's Message.Content) are not affected. */ +.ui.inverted.accordion, .ui.inverted.accordion > .title, .ui.inverted.accordion > .content { - background-color: #171616 !important; + background-color: transparent !important; } /* Mobile-responsive Confirm modal */ From d6862ecd2dd5a9d76703704464b5b5c7096d0a38 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 28 Jun 2026 13:44:18 -0600 Subject: [PATCH 02/15] Bump version to 0.25.3-beta --- wrolpi/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrolpi/version.txt b/wrolpi/version.txt index e1060e2c..a789ca14 100644 --- a/wrolpi/version.txt +++ b/wrolpi/version.txt @@ -1 +1 @@ -0.25.2-beta +0.25.3-beta From d95249218b3b18b2000565e03d813afefc132713 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 28 Jun 2026 16:51:21 -0600 Subject: [PATCH 03/15] Add per-domain and global daily download limits Adds two optional DownloadManager settings: - download_daily_limit_per_domain: max non-recurring downloads per domain per day - download_daily_limit_global: max non-recurring downloads across all domains per day Recurring Channel/RSS catalog updates are never gated and never counted; only the videos/archives they spawn and one-off downloads count. Both successful and failed attempts consume a slot (counted once per download per day). Limits reset at local midnight (configured timezone). Over-limit downloads stay "new" and resume the next day. Implementation: - New Download.last_download_attempt column (+ index, migration) set at dispatch time so the count survives restarts and is correct across workers. - dispatch_downloads gates non-recurring downloads: per-domain skips that domain, global stops dispatching counted downloads for the cycle. daily_download_counts derives the day's totals from the DB. - Domain grouping uses normalized URLs via a new Downloader.normalize_domain method (base strips www/m/mobile + port); VideoDownloader/ChannelDownloader override it to collapse youtu.be, /shorts/, and tracking params to youtube.com. ArchiveDownloader is the seam for future archive URL normalization. - Settings wired through WROLPiConfig, the /settings API (GET/PATCH, non-negative validation, 0 = unlimited), and the React Settings page. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...f7a9b1e2_download_last_download_attempt.py | 29 +++ app/src/components/admin/Settings.js | 35 +++ modules/videos/downloader.py | 10 +- wrolpi/common.py | 48 ++++ wrolpi/downloader.py | 97 +++++++- wrolpi/root_api.py | 8 + wrolpi/schema.py | 4 + wrolpi/test/test_downloader.py | 229 +++++++++++++++++- wrolpi/test/test_root_api.py | 29 +++ 9 files changed, 478 insertions(+), 11 deletions(-) create mode 100644 alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py diff --git a/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py b/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py new file mode 100644 index 00000000..1ad2c86a --- /dev/null +++ b/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py @@ -0,0 +1,29 @@ +"""download_last_download_attempt + +Add the `last_download_attempt` column to the `download` table. This records the last time a Download was +dispatched (whether the attempt succeeds or fails) and is used to enforce per-domain and global daily download +limits. An index supports the "attempted since local midnight" count query. + +Revision ID: c3d5f7a9b1e2 +Revises: b2e4f6a8c1d3 +Create Date: 2026-06-28 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'c3d5f7a9b1e2' +down_revision = 'b2e4f6a8c1d3' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('download', sa.Column('last_download_attempt', sa.DateTime(timezone=True), nullable=True)) + op.create_index('idx_download_last_download_attempt', 'download', ['last_download_attempt']) + + +def downgrade(): + op.drop_index('idx_download_last_download_attempt', table_name='download') + op.drop_column('download', 'last_download_attempt') diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js index c73b1548..85e232b8 100644 --- a/app/src/components/admin/Settings.js +++ b/app/src/components/admin/Settings.js @@ -367,6 +367,11 @@ export function SettingsPage() { setPendingSave(true); newSettings.download_timeout = parseInt(newSettings.download_timeout); newSettings.download_wait = parseInt(newSettings.download_wait); + // Empty daily-limit fields become null (no change / unlimited); a number is sent as-is. + newSettings.download_daily_limit_per_domain = newSettings.download_daily_limit_per_domain === '' + ? null : parseInt(newSettings.download_daily_limit_per_domain); + newSettings.download_daily_limit_global = newSettings.download_daily_limit_global === '' + ? null : parseInt(newSettings.download_daily_limit_global); newSettings.log_level = toApiLogLevel(newSettings.log_level); try { await saveSettings(newSettings); @@ -456,6 +461,8 @@ export function SettingsPage() { require_media_mounted: settings.require_media_mounted, download_timeout: settings.download_timeout, download_wait: settings.download_wait, + download_daily_limit_per_domain: settings.download_daily_limit_per_domain ?? '', + download_daily_limit_global: settings.download_daily_limit_global ?? '', download_window_start: settings.download_window_start || '', download_window_end: settings.download_window_end || '', timezone: settings.timezone || '', @@ -726,6 +733,34 @@ export function SettingsPage() { /> + + + Daily Download Limit (per domain) + + } + value={state.download_daily_limit_per_domain} + disabled={disabled} + onChange={(e, i) => handleTimeoutChange(e, 'download_daily_limit_per_domain', i.value)} + /> + + + + Daily Download Limit (total) + + } + value={state.download_daily_limit_global} + disabled={disabled} + onChange={(e, i) => handleTimeoutChange(e, 'download_daily_limit_global', i.value)} + /> + + diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index ede0e364..ed0a2a26 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -23,7 +23,7 @@ from yt_dlp.extractor import YoutubeTabIE # noqa from wrolpi.cmd import YT_DLP_BIN from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \ - trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post + trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post, normalize_domain from wrolpi.dates import now from wrolpi.db import get_db_session from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, \ @@ -529,6 +529,10 @@ class ChannelDownloader(Downloader, ABC): def __repr__(self): return f'' + def normalize_domain(self, url: str) -> str: + # Collapse youtu.be / youtube.com/shorts / tracking params to the canonical youtube.com domain. + return normalize_domain(normalize_video_url(url)) + @staticmethod def is_a_playlist(info: dict): # A playlist may have an id different from its channel. @@ -793,6 +797,10 @@ class VideoDownloader(Downloader, ABC): def __repr__(self): return f'' + def normalize_domain(self, url: str) -> str: + # Collapse youtu.be / youtube.com/shorts / tracking params to the canonical youtube.com domain. + return normalize_domain(normalize_video_url(url)) + def already_downloaded(self, session: Session, *urls: str) -> List: # We only consider a video record with a video file as "downloaded". file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video')) diff --git a/wrolpi/common.py b/wrolpi/common.py index 48c4989b..2148e158 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -1068,6 +1068,8 @@ class MultiFileConfig: class WROLPiConfigValidator: archive_destination: str = None check_for_upgrades: bool = None + download_daily_limit_global: int = None + download_daily_limit_per_domain: int = None download_on_startup: bool = None download_timeout: int = None download_wait: int = None @@ -1251,6 +1253,8 @@ class WROLPiConfig(ConfigFile): file_name = 'wrolpi.yaml' default_config = dict( archive_destination='archive/%(domain_tag)s/%(domain)s', + download_daily_limit_global=None, + download_daily_limit_per_domain=None, download_on_startup=True, download_timeout=0, download_wait=20, @@ -1371,6 +1375,24 @@ class WROLPiConfig(ConfigFile): def download_wait(self, value: int): self.update({'download_wait': value}) + @property + def download_daily_limit_per_domain(self) -> Optional[int]: + """Maximum non-recurring downloads allowed per domain per day. ``None``/``0`` means unlimited.""" + return self._config.get('download_daily_limit_per_domain') + + @download_daily_limit_per_domain.setter + def download_daily_limit_per_domain(self, value: Optional[int]): + self.update({'download_daily_limit_per_domain': value}) + + @property + def download_daily_limit_global(self) -> Optional[int]: + """Maximum non-recurring downloads allowed across all domains per day. ``None``/``0`` means unlimited.""" + return self._config.get('download_daily_limit_global') + + @download_daily_limit_global.setter + def download_daily_limit_global(self, value: Optional[int]): + self.update({'download_daily_limit_global': value}) + @staticmethod def _normalize_time_value(value) -> str: """YAML may parse HH:MM values as sexagesimal integers (e.g. 21:00 -> 1260). Convert back to HH:MM.""" @@ -1825,6 +1847,32 @@ def extract_domain(url: str) -> str: return domain +# Common subdomains stripped when grouping downloads by domain (e.g. for daily download limits). +COMMON_DOWNLOAD_SUBDOMAINS = ('m.', 'mobile.') + + +def normalize_domain(url: str) -> str: + """Return the canonical domain used to group downloads (e.g. for daily download limits). + + Strips the port, a leading ``www.`` (via :func:`extract_domain`), and other common subdomains + (``m.``, ``mobile.``) so that, for example, ``m.example.com`` and ``www.example.com`` share one bucket. + + Downloaders may apply type-specific URL normalization before calling this (see + ``Downloader.normalize_domain``). + + >>> normalize_domain('https://m.example.com/foo') + 'example.com' + >>> normalize_domain('https://www.example.com:443/foo') + 'example.com' + """ + domain = extract_domain(url) + for subdomain in COMMON_DOWNLOAD_SUBDOMAINS: + if domain.startswith(subdomain): + domain = domain[len(subdomain):] + break + return domain + + def api_param_limiter(maximum: int, default: int = 20) -> callable: """Create a function which restricts the maximum number that can be returned. Useful for restricting API limit params. diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index d49b88f2..ef5b5356 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -33,7 +33,7 @@ from wrolpi.api_utils import api_app, perpetual_signal from wrolpi.cmd import which, run_command, CommandResult from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \ wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \ - get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL + get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL, normalize_domain from wrolpi.dates import TZDateTime, now, Seconds from wrolpi.db import get_db_session, get_db_curs from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, BotBlockedDownloadError, UnknownDownload, \ @@ -241,6 +241,7 @@ class Download(ModelHelper, Base): # noqa __tablename__ = 'download' # noqa __table_args__ = ( Index('idx_download_collection_id', 'collection_id'), + Index('idx_download_last_download_attempt', 'last_download_attempt'), ) id = Column(Integer, primary_key=True) url = Column(String, nullable=False, unique=True) @@ -252,6 +253,7 @@ class Download(ModelHelper, Base): # noqa error = Column(Text) # traceback from an error during downloading. frequency = Column(Integer) # seconds between re-downloading. info_json = Column(JSONB) # information retrieved WHILE downloading (from yt-dlp) + last_download_attempt = Column(TZDateTime) # last time this Download was dispatched (success or failure) last_successful_download = Column(TZDateTime) location = Column(Text) # Relative App URL where the downloaded item can be viewed. next_download = Column(TZDateTime) @@ -280,6 +282,7 @@ class Download(ModelHelper, Base): # noqa frequency=self.frequency, destination=self.destination, id=self.id, + last_download_attempt=self.last_download_attempt, last_successful_download=self.last_successful_download, location=self.location, next_download=self.next_download, @@ -440,6 +443,14 @@ class Downloader: def __repr__(self): return f'' + def normalize_domain(self, url: str) -> str: + """Return the canonical domain used to group this Download for daily download limits. + + The base implementation strips common subdomains (www, m, mobile) and the port. Subclasses + may override to apply download-type-specific URL normalization before extracting the domain + (e.g. VideoDownloader collapses youtu.be and /shorts/ links to youtube.com).""" + return normalize_domain(url) + def prepare_download(self, session: Session, download: Download) -> Any: """DB-bound prep. Caller (DownloadManager) opens and commits the session. @@ -720,14 +731,57 @@ class DownloadManager: self.disabled.clear() @staticmethod - def _get_local_time(): - """Return the current local time, using configured timezone or system default.""" - from wrolpi.common import get_wrolpi_config + def _get_local_now(): + """Return the current local datetime, using configured timezone or system default.""" config = get_wrolpi_config() if config.timezone: - tz = ZoneInfo(config.timezone) - return now().astimezone(tz).time() - return now().astimezone().time() + return now().astimezone(ZoneInfo(config.timezone)) + return now().astimezone() + + @staticmethod + def _get_local_time(): + """Return the current local time, using configured timezone or system default.""" + return DownloadManager._get_local_now().time() + + @classmethod + def _get_local_day_start(cls): + """Return the most recent local midnight as a timezone-aware datetime. + + Used as the lower bound when counting downloads attempted "today" for daily limits. Comparing + against the (UTC-aware) ``last_download_attempt`` column works because both are timezone-aware.""" + return cls._get_local_now().replace(hour=0, minute=0, second=0, microsecond=0) + + def _normalize_download_domain(self, download: 'Download') -> str: + """Return the normalized domain used to group a Download for daily limits. + + Delegates to the Download's downloader so type-specific URL normalization applies; falls back to + the generic normalizer when no downloader is registered for the Download.""" + downloader = None + if download.downloader: + try: + downloader = self.find_downloader_by_name(download.downloader) + except InvalidDownload: + downloader = None + if downloader: + return downloader.normalize_domain(download.url) + return normalize_domain(download.url) + + def daily_download_counts(self, session: Session) -> Tuple[int, Dict[str, int]]: + """Count non-recurring downloads attempted since local midnight. + + Recurring downloads (those with a ``frequency``, e.g. Channel/RSS catalog updates) never count + against the limits. Returns ``(global_count, {domain: count})`` where domains are normalized.""" + day_start = self._get_local_day_start() + attempted = session.query(Download).filter( + Download.frequency.is_(None), + Download.last_download_attempt.isnot(None), + Download.last_download_attempt >= day_start, + ).all() + domain_counts: Dict[str, int] = {} + for download in attempted: + domain = self._normalize_download_domain(download) + domain_counts[domain] = domain_counts.get(domain, 0) + 1 + return len(attempted), domain_counts def is_within_download_window(self) -> bool: """Returns True if current time is within the configured download window, or no window is configured.""" @@ -1019,11 +1073,33 @@ class DownloadManager: Download.id)) count = 0 download_wait = get_wrolpi_config().download_wait + # Daily download limits. A value of None/0 means unlimited. Recurring downloads (Channel/RSS + # catalog updates) are never gated and never counted; only their non-recurring children and + # one-off downloads count. + limit_per_domain = get_wrolpi_config().download_daily_limit_per_domain or 0 + limit_global = get_wrolpi_config().download_daily_limit_global or 0 + global_count, domain_counts = 0, {} + if limit_per_domain or limit_global: + global_count, domain_counts = self.daily_download_counts(session) rate_limited_domains = {} # Track domains we've already checked for rate limiting + limits_enabled = bool(limit_per_domain or limit_global) for download in new_downloads: domain = download.domain + is_recurring = download.frequency is not None + # Non-recurring downloads with limits enabled are grouped by their normalized domain. + limit_domain = self._normalize_download_domain(download) \ + if (limits_enabled and not is_recurring) else None if domain not in self.processing_domains and len( self.processing_domains) < SIMULTANEOUS_DOWNLOAD_DOMAINS: + # Daily download limits (recurring downloads bypass the gate entirely). + if limit_domain is not None: + if limit_global and global_count >= limit_global: + # Global daily limit reached; skip all remaining counted downloads this cycle. + continue + if limit_per_domain and domain_counts.get(limit_domain, 0) >= limit_per_domain: + # This domain hit its daily limit; other domains may still have room. + continue + # Per-domain rate limiting. if download_wait and download_wait > 0: try: @@ -1037,6 +1113,13 @@ class DownloadManager: except (AttributeError, RuntimeError): pass # shared_ctx not initialized + # Record the attempt time now (in this committed session) so the next dispatch cycle + # counts it even before the download's signal handler marks it started. + download.last_download_attempt = now() + if limit_domain is not None: + global_count += 1 + domain_counts[limit_domain] = domain_counts.get(limit_domain, 0) + 1 + self._add_processing_domain(domain) context = dict(download_id=download.id, download_url=download.url) await api_app.dispatch('wrolpi.download.download', context=context) diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 81c0d1c8..e598bf19 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -156,6 +156,8 @@ def get_settings(_: Request): 'archive_destination': wrolpi_config.archive_destination, 'download_manager_disabled': download_manager.is_disabled, 'download_manager_stopped': download_manager.is_stopped, + 'download_daily_limit_global': wrolpi_config.download_daily_limit_global, + 'download_daily_limit_per_domain': wrolpi_config.download_daily_limit_per_domain, 'download_on_startup': wrolpi_config.download_on_startup, 'download_timeout': wrolpi_config.download_timeout, 'download_wait': wrolpi_config.download_wait, @@ -266,6 +268,12 @@ async def update_settings(_: Request, body: schema.SettingsRequest): except (KeyError, Exception): raise InvalidConfig(f'Invalid timezone: {body.timezone}') + # Daily download limits must be non-negative integers. 0 (or null) means unlimited. + for field_name in ('download_daily_limit_global', 'download_daily_limit_per_domain'): + value = getattr(body, field_name, None) + if value is not None and value < 0: + raise InvalidConfig(f'{field_name} must be a non-negative integer') + # Handle download window fields: empty string clears the value. for field_name in ('download_window_start', 'download_window_end'): value = getattr(body, field_name, None) diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 3eaeb092..5bfef337 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -16,6 +16,8 @@ class SettingsObject: class SettingsResponse: download_manager_disabled: bool download_manager_stopped: bool + download_daily_limit_global: Optional[int] + download_daily_limit_per_domain: Optional[int] download_on_startup: bool download_timeout: int download_wait: int @@ -63,6 +65,8 @@ class SemanticUIColors(enum.StrEnum): @dataclass class SettingsRequest: archive_destination: Optional[str] = None + download_daily_limit_global: Optional[int] = None + download_daily_limit_per_domain: Optional[int] = None download_on_startup: Optional[bool] = None download_timeout: Optional[int] = None download_wait: Optional[int] = None diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index d41803a2..3172b222 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -2,7 +2,7 @@ import asyncio import json import pathlib from abc import ABC -from datetime import datetime, time, timezone +from datetime import datetime, time, timezone, timedelta from http import HTTPStatus from itertools import zip_longest from unittest import mock @@ -11,8 +11,9 @@ import pytest import pytz import yaml -from wrolpi.common import get_wrolpi_config -from wrolpi.dates import Seconds +from wrolpi.api_utils import api_app +from wrolpi.common import get_wrolpi_config, normalize_domain +from wrolpi.dates import Seconds, now from wrolpi.db import get_db_context from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \ get_download_manager_config, RSSDownloader, parse_aria2c_progress, _parse_size, \ @@ -1586,3 +1587,225 @@ def test_require_media_mounted_gate_opt_out(test_download_manager, test_wrolpi_c with mock.patch('wrolpi.downloader.PYTEST', False), \ mock.patch('wrolpi.downloader.wrol_mode_enabled', return_value=False): assert test_download_manager.can_download is True + + +# =================================================================================================== +# Daily download limits (per-domain and global) +# =================================================================================================== + +def test_normalize_domain_helper(): + """The generic domain normalizer strips port, www, and other common subdomains.""" + assert normalize_domain('https://www.example.com/foo') == 'example.com' + assert normalize_domain('https://m.example.com/foo') == 'example.com' + assert normalize_domain('https://mobile.example.com/foo') == 'example.com' + assert normalize_domain('https://example.com:443/foo') == 'example.com' + assert normalize_domain('https://example.com/foo') == 'example.com' + + +def test_base_downloader_normalize_domain(test_downloader): + """The base Downloader.normalize_domain strips common subdomains.""" + assert test_downloader.normalize_domain('https://m.rumble.com/abc') == 'rumble.com' + assert test_downloader.normalize_domain('https://www.rumble.com/abc') == 'rumble.com' + + +def test_video_downloader_normalize_domain(): + """VideoDownloader/ChannelDownloader collapse youtu.be, shorts, and tracking params to youtube.com.""" + from modules.videos.downloader import VideoDownloader, ChannelDownloader + # Build instances without __init__ to avoid re-registering with the global manager. + vd = object.__new__(VideoDownloader) + cd = object.__new__(ChannelDownloader) + for downloader in (vd, cd): + assert downloader.normalize_domain('https://youtu.be/abc12345') == 'youtube.com' + assert downloader.normalize_domain('https://www.youtube.com/shorts/abc12345') == 'youtube.com' + assert downloader.normalize_domain('https://m.youtube.com/watch?v=abc12345') == 'youtube.com' + assert downloader.normalize_domain( + 'https://www.youtube.com/watch?v=abc12345&list=PL123&si=foo') == 'youtube.com' + + +def _make_processed_download(session, downloader_name, url, *, when=None, frequency=None, status='complete'): + """Create a Download that looks like it was already attempted (for daily-limit counting).""" + download = Download(url=url, downloader=downloader_name, frequency=frequency, status=status) + download.last_download_attempt = when if when is not None else now() + session.add(download) + session.flush() + return download + + +def _dispatched_urls(dispatch_mock) -> set: + return {c.kwargs['context']['download_url'] for c in dispatch_mock.call_args_list} + + +@pytest.mark.asyncio +async def test_daily_download_counts(test_session, test_download_manager, test_downloader): + """daily_download_counts tallies non-recurring downloads attempted since local midnight, by domain.""" + name = test_downloader.name + # Counted today: two example.com (one via m. subdomain), one rumble.com. + _make_processed_download(test_session, name, 'https://example.com/1') + _make_processed_download(test_session, name, 'https://m.example.com/2') + _make_processed_download(test_session, name, 'https://rumble.com/3') + # Not counted: recurring (has frequency), and one attempted two days ago. + _make_processed_download(test_session, name, 'https://example.com/recurring', + frequency=DownloadFrequency.daily) + _make_processed_download(test_session, name, 'https://example.com/yesterday', + when=now() - timedelta(days=2)) + test_session.commit() + + global_count, domain_counts = test_download_manager.daily_download_counts(test_session) + assert global_count == 3 + assert domain_counts == {'example.com': 2, 'rumble.com': 1} + + +@pytest.mark.asyncio +async def test_daily_limit_per_domain_blocks(test_session, test_download_manager, test_downloader): + """A new download is not dispatched once its domain hits the per-domain daily limit.""" + config = get_wrolpi_config() + config.download_daily_limit_per_domain = 3 + name = test_downloader.name + for i in range(3): + _make_processed_download(test_session, name, f'https://example.com/done{i}') + # A different domain under its own limit must still be allowed. + new_blocked = Download(url='https://example.com/new', downloader=name, status='new') + new_allowed = Download(url='https://rumble.com/new', downloader=name, status='new') + test_session.add_all([new_blocked, new_allowed]) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + dispatched = _dispatched_urls(dispatch) + assert 'https://example.com/new' not in dispatched + assert 'https://rumble.com/new' in dispatched + test_session.refresh(new_blocked) + assert new_blocked.last_download_attempt is None + assert new_blocked.status == 'new' + + +@pytest.mark.asyncio +async def test_daily_limit_per_domain_allows_under_limit(test_session, test_download_manager, test_downloader): + """A new download is dispatched while its domain is under the per-domain daily limit.""" + config = get_wrolpi_config() + config.download_daily_limit_per_domain = 3 + name = test_downloader.name + for i in range(2): + _make_processed_download(test_session, name, f'https://example.com/done{i}') + new = Download(url='https://example.com/new', downloader=name, status='new') + test_session.add(new) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + assert 'https://example.com/new' in _dispatched_urls(dispatch) + test_session.refresh(new) + assert new.last_download_attempt is not None + + +@pytest.mark.asyncio +async def test_daily_limit_resets_next_day(test_session, test_download_manager, test_downloader): + """Downloads attempted on a previous day do not count against today's limit.""" + config = get_wrolpi_config() + config.download_daily_limit_per_domain = 3 + name = test_downloader.name + # Three were processed two days ago; they should not count today. + for i in range(3): + _make_processed_download(test_session, name, f'https://example.com/old{i}', + when=now() - timedelta(days=2)) + new = Download(url='https://example.com/new', downloader=name, status='new') + test_session.add(new) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + assert 'https://example.com/new' in _dispatched_urls(dispatch) + + +@pytest.mark.asyncio +async def test_daily_limit_counts_failed_attempts(test_session, test_download_manager, test_downloader): + """A failed/deferred download attempted today still consumes a daily slot.""" + config = get_wrolpi_config() + config.download_daily_limit_per_domain = 2 + name = test_downloader.name + # One complete, one deferred (failed) today => 2 attempts, domain is at its limit. + _make_processed_download(test_session, name, 'https://example.com/ok', status='complete') + _make_processed_download(test_session, name, 'https://example.com/failed', status='deferred') + new = Download(url='https://example.com/new', downloader=name, status='new') + test_session.add(new) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + assert 'https://example.com/new' not in _dispatched_urls(dispatch) + + +@pytest.mark.asyncio +async def test_daily_limit_global_blocks(test_session, test_download_manager, test_downloader): + """The global daily limit caps downloads across all domains.""" + config = get_wrolpi_config() + config.download_daily_limit_global = 3 + name = test_downloader.name + # 3 processed today across domains (global limit reached). + _make_processed_download(test_session, name, 'https://example.com/1') + _make_processed_download(test_session, name, 'https://example.com/2') + _make_processed_download(test_session, name, 'https://rumble.com/3') + # A brand-new domain with no prior downloads is still blocked by the global cap. + new = Download(url='https://wikipedia.org/new', downloader=name, status='new') + test_session.add(new) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + assert _dispatched_urls(dispatch) == set() + test_session.refresh(new) + assert new.last_download_attempt is None + + +@pytest.mark.asyncio +async def test_daily_limit_global_allows_under_limit(test_session, test_download_manager, test_downloader): + """Downloads dispatch while the global daily total is under the limit.""" + config = get_wrolpi_config() + config.download_daily_limit_global = 5 + name = test_downloader.name + for i in range(3): + _make_processed_download(test_session, name, f'https://example.com/{i}') + new = Download(url='https://wikipedia.org/new', downloader=name, status='new') + test_session.add(new) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + assert 'https://wikipedia.org/new' in _dispatched_urls(dispatch) + + +@pytest.mark.asyncio +async def test_recurring_downloads_bypass_daily_limits(test_session, test_download_manager, test_downloader): + """Recurring (Channel/RSS) downloads are never gated and never consume a daily slot.""" + config = get_wrolpi_config() + # Both limits would otherwise block everything past the first download. + config.download_daily_limit_global = 2 + config.download_daily_limit_per_domain = 2 + name = test_downloader.name + # One non-recurring already processed today (global count = 1). + _make_processed_download(test_session, name, 'https://example.com/done') + # A recurring download on a different domain, and a new non-recurring download. + recurring = Download(url='https://rumble.com/channel', downloader=name, status='new', + frequency=DownloadFrequency.daily) + non_recurring = Download(url='https://example.com/new', downloader=name, status='new') + test_session.add_all([recurring, non_recurring]) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + dispatched = _dispatched_urls(dispatch) + # Recurring is dispatched despite the limits. The non-recurring is also dispatched, which proves the + # recurring download did not consume a global slot (else the global count would have reached 2 first). + assert 'https://rumble.com/channel' in dispatched + assert 'https://example.com/new' in dispatched + + # Recurring downloads are excluded from the daily count entirely. + global_count, domain_counts = test_download_manager.daily_download_counts(test_session) + assert 'rumble.com' not in domain_counts diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index fd938686..a608ff75 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -55,6 +55,35 @@ async def test_get_settings(test_session, async_client): assert response.status_code == HTTPStatus.OK +@pytest.mark.asyncio +async def test_daily_download_limit_settings(test_session, async_client): + """The daily download limits round-trip through the settings API and reject negative values.""" + # Defaults are unlimited (None). + request, response = await async_client.get('/api/settings') + assert response.json['download_daily_limit_global'] is None + assert response.json['download_daily_limit_per_domain'] is None + + data = {'download_daily_limit_global': 100, 'download_daily_limit_per_domain': 50} + request, response = await async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.NO_CONTENT + + request, response = await async_client.get('/api/settings') + assert response.json['download_daily_limit_global'] == 100 + assert response.json['download_daily_limit_per_domain'] == 50 + assert get_wrolpi_config().download_daily_limit_global == 100 + assert get_wrolpi_config().download_daily_limit_per_domain == 50 + + # 0 means unlimited and is accepted. + data = {'download_daily_limit_per_domain': 0} + request, response = await async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.NO_CONTENT + + # Negative values are rejected. + data = {'download_daily_limit_global': -1} + request, response = await async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.BAD_REQUEST + + @pytest.mark.asyncio async def test_log_level(async_client): """Log level must be between 0 and 40.""" From 706e22bd8ff533eb0dea6f191b615ad2d7b6fdb2 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 28 Jun 2026 17:11:45 -0600 Subject: [PATCH 04/15] Address review: clear daily limits via 0; document soft-cap semantics - Settings.js: send 0 (not null) when a daily-limit field is cleared, so the settings API actually clears a previously-saved limit instead of dropping the null and keeping the old value. - downloader.py: document that the daily limit is a best-effort soft cap; minor bounded overshoot is possible when create_downloads' background dispatch overlaps the perpetual cycle, and self-corrects the next cycle. A strict cross-process cap is intentionally not implemented. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/src/components/admin/Settings.js | 8 +++++--- wrolpi/downloader.py | 9 +++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js index 85e232b8..921d73eb 100644 --- a/app/src/components/admin/Settings.js +++ b/app/src/components/admin/Settings.js @@ -367,11 +367,13 @@ export function SettingsPage() { setPendingSave(true); newSettings.download_timeout = parseInt(newSettings.download_timeout); newSettings.download_wait = parseInt(newSettings.download_wait); - // Empty daily-limit fields become null (no change / unlimited); a number is sent as-is. + // Empty daily-limit fields become 0 (unlimited); a number is sent as-is. Sending 0 rather than + // null matters because the settings API drops null fields, which would leave a previously-saved + // limit active when the user clears the field. newSettings.download_daily_limit_per_domain = newSettings.download_daily_limit_per_domain === '' - ? null : parseInt(newSettings.download_daily_limit_per_domain); + ? 0 : parseInt(newSettings.download_daily_limit_per_domain); newSettings.download_daily_limit_global = newSettings.download_daily_limit_global === '' - ? null : parseInt(newSettings.download_daily_limit_global); + ? 0 : parseInt(newSettings.download_daily_limit_global); newSettings.log_level = toApiLogLevel(newSettings.log_level); try { await saveSettings(newSettings); diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index ef5b5356..1c3130fb 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1076,6 +1076,15 @@ class DownloadManager: # Daily download limits. A value of None/0 means unlimited. Recurring downloads (Channel/RSS # catalog updates) are never gated and never counted; only their non-recurring children and # one-off downloads count. + # + # This is a best-effort soft cap, not a hard quota. The count is read from the DB and the slot is + # claimed (last_download_attempt set) inside this dispatch transaction. The steady-state dispatcher + # (perpetual_download_worker) is single-process and self-serializing, so it never races itself; but + # create_downloads fires a background dispatch_downloads() to "start ASAP", which can overlap this + # cycle in another worker and read the same pre-commit count. At a limit boundary that can leak a + # small, bounded overshoot (paced at one download per domain per cycle); it self-corrects the next + # cycle once last_download_attempt is committed and visible. Enforcing a strict cap would require + # coordinating the count in shared_ctx with cross-process locking, which is not warranted here. limit_per_domain = get_wrolpi_config().download_daily_limit_per_domain or 0 limit_global = get_wrolpi_config().download_daily_limit_global or 0 global_count, domain_counts = 0, {} From e513e364961fa3d4ab5d7ee1f4c4eeaaa7af5053 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 28 Jun 2026 17:30:33 -0600 Subject: [PATCH 05/15] Fix daily download limits not displaying in Settings UI getSettings() in api.js remaps the /settings response through a fixed field whitelist, which dropped download_daily_limit_global/per_domain before they reached the SettingsContext. The values saved correctly but always rendered empty. Add both fields to the mapping. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/src/api.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/src/api.js b/app/src/api.js index f357fffb..80d70ec7 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -315,6 +315,8 @@ export async function getSettings() { archive_destination: content.archive_destination, download_manager_disabled: content.download_manager_disabled, download_manager_stopped: content.download_manager_stopped, + download_daily_limit_global: content.download_daily_limit_global, + download_daily_limit_per_domain: content.download_daily_limit_per_domain, download_on_startup: content.download_on_startup, download_timeout: content.download_timeout, download_wait: content.download_wait, From fcb64fed136a1e5544500eafa95fec547ad04fca Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 28 Jun 2026 17:58:55 -0600 Subject: [PATCH 06/15] Deprecate "Most Views" download order YouTube's new listing layout means yt-dlp no longer returns `view_count` in flat channel/playlist entries, so sorting by views raised KeyError and the download failed and retried forever. Sorting by views now requires per-video extraction (hundreds of requests per channel), so the feature is deprecated rather than degraded: - downloader: the 'views' branch raises UnrecoverableDownloadError so the download fails cleanly (no retry loop) with a message to switch to Newest/Oldest. May be revisited via the YouTube Data API. - schema: download_order validation drops 'views' (newest/oldest/null only). - UI: removed "Most Views" from the download-order dropdown. Co-Authored-By: Claude Opus 4.8 (1M context) --- app/src/components/Vars.js | 2 +- modules/videos/downloader.py | 9 ++++++--- modules/videos/test/test_downloader.py | 21 +++++++++++++++++++++ wrolpi/schema.py | 5 +++-- wrolpi/test/test_downloader.py | 14 ++++++++++++++ 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/app/src/components/Vars.js b/app/src/components/Vars.js index 99071eb9..3642f907 100644 --- a/app/src/components/Vars.js +++ b/app/src/components/Vars.js @@ -101,7 +101,7 @@ export const validUrlRegex = /^(http|https):\/\/[^ "]+$/; export const downloadOrderOptions = [ {key: 'newest', text: 'Newest', value: 'newest'}, {key: 'oldest', text: 'Oldest', value: 'oldest'}, - {key: 'views', text: 'Most Views', value: 'views'}, + // 'views'/'Most Views' was removed: YouTube no longer provides view counts in channel/playlist listings. ]; export const downloadResolutionOptions = [ diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index ed0a2a26..111e51cc 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -720,9 +720,12 @@ class ChannelDownloader(Downloader, ABC): logger.debug(f'Downloading oldest videos from {download.url}') downloads = downloads.reverse() elif sort_key == 'views': - # Download videos with most views. - logger.debug(f'Downloading most viewed videos from {download.url}') - downloads = sorted(downloads, key=lambda i: i['view_count'], reverse=True) + # YouTube no longer exposes `view_count` in flat channel/playlist listings, so we can no longer + # sort by views without fetching every video individually. This may become possible again if we + # integrate the YouTube Data API. + raise UnrecoverableDownloadError( + 'Downloading by "Most Views" is no longer supported because YouTube no longer provides view ' + 'counts in channel/playlist listings. Please change this download to "Newest" or "Oldest".') # Limit the videos that will be downloads (this allows the user to download the "top 100" videos of a Channel) if video_count_limit := settings.get('video_count_limit'): diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index e15fca7c..83dd5421 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -706,6 +706,27 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac } +def test_get_missing_videos_views_order_deprecated(test_session): + """The 'views' download order was deprecated. YouTube no longer provides `view_count` in flat + channel/playlist listings, so a download configured for 'views' raises an unrecoverable error rather + than crashing with a KeyError or silently downloading in the wrong order.""" + from modules.videos.downloader import channel_downloader + # Flat entries no longer contain a `view_count` key (matches current yt-dlp behavior). + download = Download( + url='https://www.youtube.com/@example/videos', + settings={'download_order': 'views'}, + info_json={'entries': [ + {'_type': 'url', 'id': 'a', 'title': 'a', 'url': 'https://youtube.com/watch?v=a'}, + {'_type': 'url', 'id': 'b', 'title': 'b', 'url': 'https://youtube.com/watch?v=b'}, + ]}, + ) + test_session.add(download) + test_session.commit() + + with pytest.raises(UnrecoverableDownloadError): + channel_downloader.get_missing_videos(test_session, download) + + @pytest.mark.asyncio async def test_channel_download_crud(test_session, async_client, assert_downloads, tag_factory): """Test creating more complex Channel Download.""" diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 5bfef337..6d13e4ac 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -301,8 +301,9 @@ class DownloadSettings: if not self.download_metadata_only: del self.download_metadata_only - if self.download_order not in (None, 'newest', 'oldest', 'views'): - raise ValidationError(f'Download order must be one of newest, oldest, views, or null.') + # 'views' was deprecated: YouTube no longer provides view counts in channel/playlist listings. + if self.download_order not in (None, 'newest', 'oldest'): + raise ValidationError(f'Download order must be one of newest, oldest, or null.') if self.video_format not in (None, 'mp4', 'mkv', 'ogg', 'webm'): raise ValidationError(f'Video format must be one of mp4, mkv, ogg, webm, or null.') diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 3172b222..c4d060f0 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -914,6 +914,20 @@ def test_download_settings_coerces_numeric_strings(): assert 'max_pages' not in req.settings +@pytest.mark.parametrize('order', ['newest', 'oldest', None]) +def test_download_settings_accepts_valid_download_order(order): + from wrolpi.schema import DownloadSettings + DownloadSettings(download_order=order) + + +def test_download_settings_rejects_deprecated_views_order(): + """'views' download order was deprecated; YouTube no longer provides view counts in listings.""" + from wrolpi.schema import DownloadSettings + from wrolpi.errors import ValidationError + with pytest.raises(ValidationError): + DownloadSettings(download_order='views') + + @pytest.mark.asyncio async def test_downloads_config(test_session, test_download_manager, test_download_manager_config, test_downloader, assert_downloads, tag_factory, await_switches): From c95afc7f62d131f29ed694d7d55515c2c384330e Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 1 Jul 2026 18:24:21 -0600 Subject: [PATCH 07/15] Add ESP32 firmware flasher Browser-based ESP32/ESP8266 flasher (esptool-js + Web Serial); nothing is uploaded to WROLPi, the flash is written straight to the device over USB. Firmware sources (themed, tabbed): a saved set, the WROLPi media directory, or a local file. Local files can be dropped anywhere on the page, which jumps to the "Add from computer" dropzone. The media picker searches .bin files by path and can detect the connected chip to filter firmware to matching images. - Indexed file_group.suffix + path filter for fast .bin search (alembic migration backfills via split_path_stem_and_suffix) - modules/flasher: chip detection from ESP image headers, /search endpoint, and flasher.yaml saved configurations (/configs CRUD) - Flash progress modal with ETA, connect timeout, boot-mode hint, auto- scrolling logs Co-Authored-By: Claude Opus 4.8 (1M context) --- .../d4e6f8a0c2b5_file_group_suffix.py | 69 ++ app/package-lock.json | 140 ++- app/package.json | 1 + app/src/App.js | 2 + app/src/api.js | 53 +- app/src/components/Flasher.js | 866 ++++++++++++++++++ app/src/components/Flasher.test.js | 299 ++++++ app/src/components/Nav.js | 1 + modules/flasher/__init__.py | 0 modules/flasher/api.py | 55 ++ modules/flasher/config.py | 86 ++ modules/flasher/conftest.py | 71 ++ modules/flasher/lib.py | 129 +++ modules/flasher/schema.py | 20 + modules/flasher/test/__init__.py | 0 modules/flasher/test/test_flasher.py | 156 ++++ wrolpi/common.py | 14 + wrolpi/contexts.py | 7 + wrolpi/files/api.py | 2 +- wrolpi/files/lib.py | 25 +- wrolpi/files/models.py | 6 +- wrolpi/files/schema.py | 2 + wrolpi/files/test/test_api.py | 44 + wrolpi/files/test/test_lib.py | 27 + wrolpi/root_api.py | 2 + 25 files changed, 2063 insertions(+), 14 deletions(-) create mode 100644 alembic/versions/d4e6f8a0c2b5_file_group_suffix.py create mode 100644 app/src/components/Flasher.js create mode 100644 app/src/components/Flasher.test.js create mode 100644 modules/flasher/__init__.py create mode 100644 modules/flasher/api.py create mode 100644 modules/flasher/config.py create mode 100644 modules/flasher/conftest.py create mode 100644 modules/flasher/lib.py create mode 100644 modules/flasher/schema.py create mode 100644 modules/flasher/test/__init__.py create mode 100644 modules/flasher/test/test_flasher.py diff --git a/alembic/versions/d4e6f8a0c2b5_file_group_suffix.py b/alembic/versions/d4e6f8a0c2b5_file_group_suffix.py new file mode 100644 index 00000000..72526929 --- /dev/null +++ b/alembic/versions/d4e6f8a0c2b5_file_group_suffix.py @@ -0,0 +1,69 @@ +"""file_group_suffix + +Add the `suffix` column to the `file_group` table. This stores the lowercased suffix of the primary file +(e.g. ".bin") so searches can filter by file type using an index, rather than relying on full-text tokens. + +`suffix` is populated at ingest time alongside `mimetype` (wrolpi.files.lib._upsert_files and +FileGroup.from_paths). Existing rows are backfilled here using the same `split_path_stem_and_suffix` logic so +special multi-part suffixes like ".info.json" are stored correctly (not a naive ".json"). + +Revision ID: d4e6f8a0c2b5 +Revises: c3d5f7a9b1e2 +Create Date: 2026-07-01 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'd4e6f8a0c2b5' +down_revision = 'c3d5f7a9b1e2' +branch_labels = None +depends_on = None + +# Backfill in batches to bound memory on low-RAM devices (e.g. Raspberry Pi). +BATCH_SIZE = 5000 + + +def upgrade(): + op.add_column('file_group', sa.Column('suffix', sa.String(), nullable=True)) + op.create_index('ix_file_group_suffix', 'file_group', ['suffix']) + + # Backfill existing rows using the same suffix logic used at ingest time so the stored value matches what + # future refreshes would produce (including WROLPi special suffixes like ".info.json"). + from wrolpi.files.lib import split_path_stem_and_suffix + conn = op.get_bind() + last_id = 0 + while True: + rows = conn.execute( + sa.text('SELECT id, primary_path FROM file_group WHERE id > :last_id ORDER BY id LIMIT :limit'), + {'last_id': last_id, 'limit': BATCH_SIZE}, + ).fetchall() + if not rows: + break + + ids, suffixes = [], [] + for row_id, primary_path in rows: + suffix = (split_path_stem_and_suffix(primary_path)[1] or '').lower() or None + if suffix: + ids.append(row_id) + suffixes.append(suffix) + + if ids: + conn.execute( + sa.text(''' + UPDATE file_group AS fg + SET suffix = v.suffix + FROM (SELECT unnest(cast(:ids AS integer[])) AS id, + unnest(cast(:suffixes AS text[])) AS suffix) AS v + WHERE fg.id = v.id + '''), + {'ids': ids, 'suffixes': suffixes}, + ) + + last_id = rows[-1][0] + + +def downgrade(): + op.drop_index('ix_file_group_suffix', table_name='file_group') + op.drop_column('file_group', 'suffix') diff --git a/app/package-lock.json b/app/package-lock.json index 80c8e8f5..9e8a8597 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -14,6 +14,7 @@ "@testing-library/react": "^13.4.0", "@testing-library/user-event": "^13.5.0", "cypress": "^13.15.2", + "esptool-js": "^0.6.0", "lodash": "^4.18.1", "maplibre-gl": "^5.24.0", "mathjs": "^15.2.0", @@ -106,6 +107,7 @@ "version": "7.20.12", "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz", "integrity": "sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==", + "peer": true, "dependencies": { "@ampproject/remapping": "^2.1.0", "@babel/code-frame": "^7.18.6", @@ -902,6 +904,7 @@ "version": "7.18.6", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz", "integrity": "sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.18.6" }, @@ -1463,6 +1466,7 @@ "version": "7.20.13", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.20.13.tgz", "integrity": "sha512-MmTZx/bkUrfJhhYAYt3Urjm+h8DQGrPrnKQ94jLo7NLuOU+T89a7IByhKmrb8SKhrIYIQ0FN0CHMbnFRen4qNw==", + "peer": true, "dependencies": { "@babel/helper-annotate-as-pure": "^7.18.6", "@babel/helper-module-imports": "^7.18.6", @@ -3377,6 +3381,7 @@ "version": "2.11.6", "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.6.tgz", "integrity": "sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==", + "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/popperjs" @@ -4164,6 +4169,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.27.tgz", "integrity": "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w==", "license": "MIT", + "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -4174,6 +4180,7 @@ "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^18.0.0" } @@ -4297,6 +4304,7 @@ "version": "5.50.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.50.0.tgz", "integrity": "sha512-vwksQWSFZiUhgq3Kv7o1Jcj0DUNylwnIlGvKvLLYsq8pAWha6/WCnXUeaSoNNha/K7QSf2+jvmkxggC1u3pIwQ==", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.50.0", "@typescript-eslint/type-utils": "5.50.0", @@ -4348,6 +4356,7 @@ "version": "5.50.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.50.0.tgz", "integrity": "sha512-KCcSyNaogUDftK2G9RXfQyOCt51uB5yqC6pkUYqhYh8Kgt+DwR5M0EwEAxGPy/+DH6hnmKeGsNhiZRQxjH71uQ==", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.50.0", "@typescript-eslint/types": "5.50.0", @@ -4678,6 +4687,7 @@ "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4800,6 +4810,7 @@ "version": "6.14.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -5106,6 +5117,12 @@ "node": ">= 4.0.0" } }, + "node_modules/atob-lite": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/atob-lite/-/atob-lite-2.0.0.tgz", + "integrity": "sha512-LEeSAWeh2Gfa2FtlQE1shxQ8zi5F9GHarrGKz08TMdODD5T4eH6BMsvtnhbWZ+XQn+Gb6om/917ucvRu7l7ukw==", + "license": "MIT" + }, "node_modules/attr-accept": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.2.tgz", @@ -5657,6 +5674,7 @@ "url": "https://github.com/sponsors/ai" } ], + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -6454,6 +6472,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -6740,6 +6759,7 @@ "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.15.2.tgz", "integrity": "sha512-ARbnUorjcCM3XiPwgHKuqsyr5W9Qn+pIIBPaoilnoBkLdSC2oLQjV1BUpnmc7KR+b7Avah3Ly2RMFnfxr96E/A==", "hasInstallScript": true, + "peer": true, "dependencies": { "@cypress/request": "^3.0.6", "@cypress/xvfb": "^1.2.4", @@ -7506,6 +7526,7 @@ "version": "2.4.1", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", "integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==", + "peer": true, "dependencies": { "ansi-colors": "^4.1.1", "strip-ansi": "^6.0.1" @@ -7785,6 +7806,7 @@ "version": "8.33.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.33.0.tgz", "integrity": "sha512-WjOpFQgKK8VrCnAtl8We0SUOy/oVZ5NHykyMiagV1M9r8IFpIJX7DduK6n1mpfhlG7T1NLWm2SuD8QB7KFySaA==", + "peer": true, "dependencies": { "@eslint/eslintrc": "^1.4.1", "@humanwhocodes/config-array": "^0.11.8", @@ -8198,6 +8220,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -8422,6 +8445,17 @@ "node": ">=4" } }, + "node_modules/esptool-js": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/esptool-js/-/esptool-js-0.6.0.tgz", + "integrity": "sha512-ZyuSBXvmS+4DRKM6GrDHfCscnq3YSNlRfM+Zw0DoS7uC3SdkyC262s/ji4MjDYVmMH8SE2BiYx3/dZ1J23Xiew==", + "license": "Apache-2.0", + "dependencies": { + "atob-lite": "^2.0.0", + "pako": "^2.1.0", + "tslib": "^2.4.1" + } + }, "node_modules/esquery": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", @@ -10689,6 +10723,7 @@ "version": "27.5.1", "resolved": "https://registry.npmjs.org/jest/-/jest-27.5.1.tgz", "integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==", + "peer": true, "dependencies": { "@jest/core": "^27.5.1", "import-local": "^3.0.2", @@ -13383,6 +13418,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -13913,6 +13949,22 @@ "node": ">=6" } }, + "node_modules/pako": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.2.0.tgz", + "integrity": "sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "(MIT AND Zlib)" + }, "node_modules/param-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", @@ -14223,6 +14275,7 @@ "url": "https://tidelift.com/funding/github/npm/postcss" } ], + "peer": true, "dependencies": { "nanoid": "^3.3.4", "picocolors": "^1.0.0", @@ -15281,6 +15334,7 @@ "version": "6.0.11", "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz", "integrity": "sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g==", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -15470,6 +15524,7 @@ "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "peer": true, "dependencies": { "loose-envify": "^1.4.0", "object-assign": "^4.1.1", @@ -15677,6 +15732,7 @@ "version": "18.2.0", "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -15830,6 +15886,7 @@ "version": "18.2.0", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==", + "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.0" @@ -15931,6 +15988,7 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.11.0.tgz", "integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -16445,6 +16503,7 @@ "version": "2.80.0", "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.80.0.tgz", "integrity": "sha512-cIFJOD1DESzpjOBl763Kp1AH7UE/0fcdHe6rZXUdQ9c50uvgigvW97u3IcSeBwOkgqL/PXPBktBCh0KEu5L8XQ==", + "peer": true, "bin": { "rollup": "dist/bin/rollup" }, @@ -16720,6 +16779,7 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.1.5.tgz", "integrity": "sha512-nIqmmUNpFHfovEb+RI2w3E2/maZQutd8UIWyRjf1SLse+XF51hI559xbz/sLN3O6RpLjr/echLOOXwKCirPy3Q==", + "peer": true, "dependencies": { "@babel/runtime": "^7.10.5", "@fluentui/react-component-event-listener": "~0.63.0", @@ -17807,6 +17867,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -17878,7 +17939,8 @@ "node_modules/three": { "version": "0.184.0", "resolved": "https://registry.npmjs.org/three/-/three-0.184.0.tgz", - "integrity": "sha512-wtTRjG92pM5eUg/KuUnHsqSAlPM296brTOcLgMRqEeylYTh/CdtvKUvCyyCQTzFuStieWxvZb8mVTMvdPyUpxg==" + "integrity": "sha512-wtTRjG92pM5eUg/KuUnHsqSAlPM296brTOcLgMRqEeylYTh/CdtvKUvCyyCQTzFuStieWxvZb8mVTMvdPyUpxg==", + "peer": true }, "node_modules/three-stdlib": { "version": "2.17.2", @@ -18126,6 +18188,7 @@ "version": "0.21.3", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "peer": true, "engines": { "node": ">=10" }, @@ -18179,6 +18242,7 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -18489,6 +18553,7 @@ "version": "5.105.0", "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz", "integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==", + "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -18558,6 +18623,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -18607,6 +18673,7 @@ "version": "4.11.1", "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.11.1.tgz", "integrity": "sha512-lILVz9tAUy1zGFwieuaQtYiadImb5M3d+H+L1zDYalYoDl0cksAB1UNyuE5MMWJrG6zR1tXkCP2fitl7yoUJiw==", + "peer": true, "dependencies": { "@types/bonjour": "^3.5.9", "@types/connect-history-api-fallback": "^1.3.5", @@ -18661,6 +18728,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -18773,6 +18841,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -19055,6 +19124,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -19499,6 +19569,7 @@ "version": "7.20.12", "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.20.12.tgz", "integrity": "sha512-XsMfHovsUYHFMdrIHkZphTN/2Hzzi78R08NuHfDBehym2VsPDL6Zn/JAD/JQdnRvbSsbQc4mVaU1m6JgtTEElg==", + "peer": true, "requires": { "@ampproject/remapping": "^2.1.0", "@babel/code-frame": "^7.18.6", @@ -20041,6 +20112,7 @@ "version": "7.18.6", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz", "integrity": "sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==", + "peer": true, "requires": { "@babel/helper-plugin-utils": "^7.18.6" } @@ -20386,6 +20458,7 @@ "version": "7.20.13", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.20.13.tgz", "integrity": "sha512-MmTZx/bkUrfJhhYAYt3Urjm+h8DQGrPrnKQ94jLo7NLuOU+T89a7IByhKmrb8SKhrIYIQ0FN0CHMbnFRen4qNw==", + "peer": true, "requires": { "@babel/helper-annotate-as-pure": "^7.18.6", "@babel/helper-module-imports": "^7.18.6", @@ -21733,7 +21806,8 @@ "@popperjs/core": { "version": "2.11.6", "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.6.tgz", - "integrity": "sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==" + "integrity": "sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==", + "peer": true }, "@react-three/fiber": { "version": "8.16.5", @@ -22311,6 +22385,7 @@ "version": "18.3.27", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.27.tgz", "integrity": "sha512-cisd7gxkzjBKU2GgdYrTdtQx1SORymWyaAFhaxQPK9bYO9ot3Y5OikQRvY0VYQtvwjeQnizCINJAenh/V7MK2w==", + "peer": true, "requires": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -22320,6 +22395,7 @@ "version": "18.3.7", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "peer": true, "requires": {} }, "@types/react-reconciler": { @@ -22440,6 +22516,7 @@ "version": "5.50.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.50.0.tgz", "integrity": "sha512-vwksQWSFZiUhgq3Kv7o1Jcj0DUNylwnIlGvKvLLYsq8pAWha6/WCnXUeaSoNNha/K7QSf2+jvmkxggC1u3pIwQ==", + "peer": true, "requires": { "@typescript-eslint/scope-manager": "5.50.0", "@typescript-eslint/type-utils": "5.50.0", @@ -22465,6 +22542,7 @@ "version": "5.50.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.50.0.tgz", "integrity": "sha512-KCcSyNaogUDftK2G9RXfQyOCt51uB5yqC6pkUYqhYh8Kgt+DwR5M0EwEAxGPy/+DH6hnmKeGsNhiZRQxjH71uQ==", + "peer": true, "requires": { "@typescript-eslint/scope-manager": "5.50.0", "@typescript-eslint/types": "5.50.0", @@ -22714,7 +22792,8 @@ "acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==" + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "peer": true }, "acorn-globals": { "version": "6.0.0", @@ -22801,6 +22880,7 @@ "version": "6.14.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -23017,6 +23097,11 @@ "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==" }, + "atob-lite": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/atob-lite/-/atob-lite-2.0.0.tgz", + "integrity": "sha512-LEeSAWeh2Gfa2FtlQE1shxQ8zi5F9GHarrGKz08TMdODD5T4eH6BMsvtnhbWZ+XQn+Gb6om/917ucvRu7l7ukw==" + }, "attr-accept": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.2.tgz", @@ -23418,6 +23503,7 @@ "version": "4.28.1", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "peer": true, "requires": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -23959,6 +24045,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -24167,6 +24254,7 @@ "version": "13.15.2", "resolved": "https://registry.npmjs.org/cypress/-/cypress-13.15.2.tgz", "integrity": "sha512-ARbnUorjcCM3XiPwgHKuqsyr5W9Qn+pIIBPaoilnoBkLdSC2oLQjV1BUpnmc7KR+b7Avah3Ly2RMFnfxr96E/A==", + "peer": true, "requires": { "@cypress/request": "^3.0.6", "@cypress/xvfb": "^1.2.4", @@ -24730,6 +24818,7 @@ "version": "2.4.1", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", "integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==", + "peer": true, "requires": { "ansi-colors": "^4.1.1", "strip-ansi": "^6.0.1" @@ -24948,6 +25037,7 @@ "version": "8.33.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.33.0.tgz", "integrity": "sha512-WjOpFQgKK8VrCnAtl8We0SUOy/oVZ5NHykyMiagV1M9r8IFpIJX7DduK6n1mpfhlG7T1NLWm2SuD8QB7KFySaA==", + "peer": true, "requires": { "@eslint/eslintrc": "^1.4.1", "@humanwhocodes/config-array": "^0.11.8", @@ -25323,6 +25413,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -25394,6 +25485,16 @@ "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" }, + "esptool-js": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/esptool-js/-/esptool-js-0.6.0.tgz", + "integrity": "sha512-ZyuSBXvmS+4DRKM6GrDHfCscnq3YSNlRfM+Zw0DoS7uC3SdkyC262s/ji4MjDYVmMH8SE2BiYx3/dZ1J23Xiew==", + "requires": { + "atob-lite": "^2.0.0", + "pako": "^2.1.0", + "tslib": "^2.4.1" + } + }, "esquery": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", @@ -26993,6 +27094,7 @@ "version": "27.5.1", "resolved": "https://registry.npmjs.org/jest/-/jest-27.5.1.tgz", "integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==", + "peer": true, "requires": { "@jest/core": "^27.5.1", "import-local": "^3.0.2", @@ -28964,6 +29066,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -29331,6 +29434,11 @@ "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" }, + "pako": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.2.0.tgz", + "integrity": "sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==" + }, "param-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", @@ -29556,6 +29664,7 @@ "version": "8.4.21", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.21.tgz", "integrity": "sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==", + "peer": true, "requires": { "nanoid": "^3.3.4", "picocolors": "^1.0.0", @@ -30139,6 +30248,7 @@ "version": "6.0.11", "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz", "integrity": "sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g==", + "peer": true, "requires": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -30278,6 +30388,7 @@ "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "peer": true, "requires": { "loose-envify": "^1.4.0", "object-assign": "^4.1.1", @@ -30435,6 +30546,7 @@ "version": "18.2.0", "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==", + "peer": true, "requires": { "loose-envify": "^1.1.0" } @@ -30548,6 +30660,7 @@ "version": "18.2.0", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz", "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==", + "peer": true, "requires": { "loose-envify": "^1.1.0", "scheduler": "^0.23.0" @@ -30624,7 +30737,8 @@ "react-refresh": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.11.0.tgz", - "integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==" + "integrity": "sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A==", + "peer": true }, "react-router": { "version": "7.16.0", @@ -30993,6 +31107,7 @@ "version": "2.80.0", "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.80.0.tgz", "integrity": "sha512-cIFJOD1DESzpjOBl763Kp1AH7UE/0fcdHe6rZXUdQ9c50uvgigvW97u3IcSeBwOkgqL/PXPBktBCh0KEu5L8XQ==", + "peer": true, "requires": { "fsevents": "~2.3.2" } @@ -31178,6 +31293,7 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/semantic-ui-react/-/semantic-ui-react-2.1.5.tgz", "integrity": "sha512-nIqmmUNpFHfovEb+RI2w3E2/maZQutd8UIWyRjf1SLse+XF51hI559xbz/sLN3O6RpLjr/echLOOXwKCirPy3Q==", + "peer": true, "requires": { "@babel/runtime": "^7.10.5", "@fluentui/react-component-event-listener": "~0.63.0", @@ -32007,6 +32123,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -32058,7 +32175,8 @@ "three": { "version": "0.184.0", "resolved": "https://registry.npmjs.org/three/-/three-0.184.0.tgz", - "integrity": "sha512-wtTRjG92pM5eUg/KuUnHsqSAlPM296brTOcLgMRqEeylYTh/CdtvKUvCyyCQTzFuStieWxvZb8mVTMvdPyUpxg==" + "integrity": "sha512-wtTRjG92pM5eUg/KuUnHsqSAlPM296brTOcLgMRqEeylYTh/CdtvKUvCyyCQTzFuStieWxvZb8mVTMvdPyUpxg==", + "peer": true }, "three-stdlib": { "version": "2.17.2", @@ -32260,7 +32378,8 @@ "type-fest": { "version": "0.21.3", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==" + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "peer": true }, "type-is": { "version": "1.6.18", @@ -32297,7 +32416,8 @@ "typescript": { "version": "4.9.5", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", - "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==" + "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", + "peer": true }, "unbox-primitive": { "version": "1.0.2", @@ -32524,6 +32644,7 @@ "version": "5.105.0", "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz", "integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==", + "peer": true, "requires": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -32556,6 +32677,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -32619,6 +32741,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -32656,6 +32779,7 @@ "version": "4.11.1", "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-4.11.1.tgz", "integrity": "sha512-lILVz9tAUy1zGFwieuaQtYiadImb5M3d+H+L1zDYalYoDl0cksAB1UNyuE5MMWJrG6zR1tXkCP2fitl7yoUJiw==", + "peer": true, "requires": { "@types/bonjour": "^3.5.9", "@types/connect-history-api-fallback": "^1.3.5", @@ -32692,6 +32816,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -32938,6 +33063,7 @@ "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "peer": true, "requires": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", diff --git a/app/package.json b/app/package.json index 2e171d27..be4b3373 100644 --- a/app/package.json +++ b/app/package.json @@ -9,6 +9,7 @@ "@testing-library/react": "^13.4.0", "@testing-library/user-event": "^13.5.0", "cypress": "^13.15.2", + "esptool-js": "^0.6.0", "lodash": "^4.18.1", "maplibre-gl": "^5.24.0", "mathjs": "^15.2.0", diff --git a/app/src/App.js b/app/src/App.js index d82b6980..f1ed37db 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -25,6 +25,7 @@ import {TagsProvider} from "./Tags"; import {ZimRoute} from "./components/Zim"; import {PlaylistsRoute} from "./components/Playlists"; import {DocsRoute} from "./components/Docs"; +import {FlasherRoute} from "./components/Flasher"; import ErrorBoundary from "./components/ErrorBoundary"; import {KeyboardShortcutsProvider} from "./components/KeyboardShortcutsProvider"; @@ -107,6 +108,7 @@ const router = createBrowserRouter(createRoutesFromElements(}/> }/> }/> + }/> )); export default function App() { diff --git a/app/src/api.js b/app/src/api.js index 80d70ec7..74efc88d 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -1212,8 +1212,14 @@ export async function deleteDownload(downloadId) { } } -export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag, order) { +export async function filesSearch(offset, limit, searchStr, mimetypes, model, tagNames, headline, months, fromYear, toYear, anyTag, order, suffix, path) { const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit), any_tag: anyTag}; + if (suffix) { + body['suffix'] = suffix; + } + if (path) { + body['path'] = path; + } if (mimetypes) { body['mimetypes'] = mimetypes; } @@ -1257,6 +1263,51 @@ export async function filesSearch(offset, limit, searchStr, mimetypes, model, ta } } +export async function flasherSearch(chip, path) { + // Search .bin firmware, filtered to ESP images for the given chip (from connecting to the device). + const body = {}; + if (chip) body['chip'] = chip; + if (path) body['path'] = path; + const response = await apiPost(`${API_URI}/flasher/search`, body); + if (response.ok) { + const data = await response.json(); + return [data['file_groups'], data['totals']['file_groups']]; + } + const message = await getErrorMessage(response, 'Cannot search firmware. See server logs.'); + toast({type: 'error', title: 'Unable to search firmware', description: message, time: 5000}); + return [null, null]; +} + +export async function getFlasherConfigs() { + // Return the user's saved firmware flashing configurations. + const response = await apiGet(`${API_URI}/flasher/configs`); + if (response.ok) { + const data = await response.json(); + return data['configurations'] || []; + } + return []; +} + +export async function saveFlasherConfig(name, files, eraseAll) { + const response = await apiPost(`${API_URI}/flasher/configs`, {name, files, erase_all: eraseAll}); + if (!response.ok) { + const message = await getErrorMessage(response, 'Cannot save configuration. See server logs.'); + toast({type: 'error', title: 'Unable to save configuration', description: message, time: 5000}); + return false; + } + return true; +} + +export async function deleteFlasherConfig(name) { + const response = await apiDelete(`${API_URI}/flasher/configs/${encodeURIComponent(name)}`); + if (!response.ok) { + const message = await getErrorMessage(response, 'Cannot delete configuration. See server logs.'); + toast({type: 'error', title: 'Unable to delete configuration', description: message, time: 5000}); + return false; + } + return true; +} + export async function refreshFiles(paths) { let response; if (Array.isArray(paths) && paths.length > 0) { diff --git a/app/src/components/Flasher.js b/app/src/components/Flasher.js new file mode 100644 index 00000000..cdcdb2a4 --- /dev/null +++ b/app/src/components/Flasher.js @@ -0,0 +1,866 @@ +import React, {useContext, useEffect, useLayoutEffect, useRef, useState} from 'react'; +import {Route, Routes} from "react-router"; +import {Checkbox, Label, Message, Select, Button as SButton, Icon as SIcon} from "semantic-ui-react"; +import {ESPLoader, Transport} from "esptool-js"; +import {useDropzone} from "react-dropzone"; +import _ from "lodash"; +import {encodeMediaPath, humanFileSize, PageContainer, useTitle} from "./Common"; +import { + Button, + darkTheme, + Divider, + Form, + Header, + Icon, + List, + Loader, + Modal, + Progress, + Segment, + Tab, + Table, + TextArea +} from "./Theme"; +import {ThemeContext} from "../contexts/contexts"; +import {deleteFlasherConfig, filesSearch, flasherSearch, getFlasherConfigs, saveFlasherConfig} from "../api"; + +// The suffix used to find flashable ESP32 firmware in the media directory. +const FIRMWARE_SUFFIX = '.bin'; + +// esptool-js reports a chip description like "ESP32-S2 (revision v0.0)"; take the family before " (". +export function chipFamily(description) { + return (description || '').split(' (')[0].trim() || null; +} + +// Flashing happens entirely in the browser using the Web Serial API; nothing is uploaded to WROLPi. esptool-js is +// bundled into the frontend build, so this works fully offline once the page has loaded. + +const BAUD_OPTIONS = [ + {key: '115200', value: 115200, text: '115200 (safe)'}, + {key: '230400', value: 230400, text: '230400'}, + {key: '460800', value: 460800, text: '460800'}, + {key: '921600', value: 921600, text: '921600 (fast)'}, +]; + +// Web Serial is only available in a secure context (HTTPS or localhost) on Chromium-based browsers. +export function webSerialSupported() { + return typeof navigator !== 'undefined' && 'serial' in navigator; +} + +function readFileAsUint8Array(file) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(new Uint8Array(reader.result)); + reader.onerror = () => reject(reader.error); + reader.readAsArrayBuffer(file); + }); +} + +// Load a firmware entry's bytes. Local files are read from disk; media entries are fetched from the WROLPi +// media directory over HTTP. Both return a Uint8Array as esptool-js expects. +async function readEntryBytes(item) { + if (item.file) { + return readFileAsUint8Array(item.file); + } + const resp = await fetch(`/media/${encodeMediaPath(item.mediaPath)}`); + if (!resp.ok) { + throw new Error(`Failed to fetch ${item.name} from the media directory (${resp.status})`); + } + const buffer = await resp.arrayBuffer(); + return new Uint8Array(buffer); +} + +// A flash offset must be a 0x-prefixed hex string, e.g. "0x0" or "0x10000". +const HEX_OFFSET_RE = /^0x[0-9a-f]+$/i; + +export function isValidHexOffset(value) { + return HEX_OFFSET_RE.test((value || '').trim()); +} + +// Parse a validated hex flash offset like "0x10000" or "0x0" into a number. +export function parseAddress(value) { + const trimmed = (value || '').trim(); + if (trimmed === '') { + throw new Error('Flash offset is required'); + } + if (!isValidHexOffset(trimmed)) { + throw new Error(`Invalid flash offset (expected a hex value like 0x10000): ${value}`); + } + return parseInt(trimmed, 16); +} + +// Rough estimate of how long flashing will take, in seconds. Serial is 8N1 (10 bits per byte), so the link +// carries ~baud/10 bytes/sec; esptool-js compresses the image (~0.6x), which roughly offsets SLIP framing and +// flash-write overhead, making raw bytes / (baud/10) a reasonable ballpark. +export function estimateFlashSeconds(totalBytes, baudrate) { + if (!totalBytes || !baudrate) { + return 0; + } + return totalBytes / (baudrate / 10); +} + +// Format a duration in seconds as e.g. "45s" or "2m 5s". +export function humanDuration(seconds) { + if (!seconds || !isFinite(seconds) || seconds <= 0) { + return null; + } + const total = Math.round(seconds); + const minutes = Math.floor(total / 60); + const secs = total % 60; + return minutes > 0 ? `${minutes}m ${secs}s` : `${secs}s`; +} + +// Reject if `promise` doesn't settle within `ms`. esptool-js's connect can hang indefinitely when a board is +// not in download mode; this guarantees the caller always gets a result so the UI can recover. +export function withTimeout(promise, ms, message) { + let timer; + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(message)), ms); + }); + // Swallow a late rejection from the raced promise so it doesn't surface as an unhandled rejection. + promise.catch(() => { + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); +} + +// How long to wait for a device to connect before giving up (esptool-js retries can otherwise stall the UI). +const CONNECT_TIMEOUT_MS = 30_000; + +// A read-only log TextArea that auto-scrolls to the newest output — but only when the user is already at the +// bottom, so scrolling up to read earlier lines isn't yanked back down. Starts at the bottom on mount (e.g. when +// a modal opens). +function AutoScrollLog({value, ...props}) { + const containerRef = useRef(null); + const atBottomRef = useRef(true); + + const getTextArea = () => containerRef.current && containerRef.current.querySelector('textarea'); + + const handleScroll = (e) => { + const el = e.target; + // Treat "within a few px of the bottom" as at-bottom so it re-sticks when the user scrolls back down. + atBottomRef.current = el.scrollHeight - el.scrollTop - el.clientHeight < 20; + }; + + useLayoutEffect(() => { + const el = getTextArea(); + if (el && atBottomRef.current) { + el.scrollTop = el.scrollHeight; + } + }, [value]); + + return
+