diff --git a/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py b/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py new file mode 100644 index 00000000..1ad2c86a --- /dev/null +++ b/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py @@ -0,0 +1,29 @@ +"""download_last_download_attempt + +Add the `last_download_attempt` column to the `download` table. This records the last time a Download was +dispatched (whether the attempt succeeds or fails) and is used to enforce per-domain and global daily download +limits. An index supports the "attempted since local midnight" count query. + +Revision ID: c3d5f7a9b1e2 +Revises: b2e4f6a8c1d3 +Create Date: 2026-06-28 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'c3d5f7a9b1e2' +down_revision = 'b2e4f6a8c1d3' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('download', sa.Column('last_download_attempt', sa.DateTime(timezone=True), nullable=True)) + op.create_index('idx_download_last_download_attempt', 'download', ['last_download_attempt']) + + +def downgrade(): + op.drop_index('idx_download_last_download_attempt', table_name='download') + op.drop_column('download', 'last_download_attempt') diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js index c73b1548..85e232b8 100644 --- a/app/src/components/admin/Settings.js +++ b/app/src/components/admin/Settings.js @@ -367,6 +367,11 @@ export function SettingsPage() { setPendingSave(true); newSettings.download_timeout = parseInt(newSettings.download_timeout); newSettings.download_wait = parseInt(newSettings.download_wait); + // Empty daily-limit fields become null (no change / unlimited); a number is sent as-is. + newSettings.download_daily_limit_per_domain = newSettings.download_daily_limit_per_domain === '' + ? null : parseInt(newSettings.download_daily_limit_per_domain); + newSettings.download_daily_limit_global = newSettings.download_daily_limit_global === '' + ? null : parseInt(newSettings.download_daily_limit_global); newSettings.log_level = toApiLogLevel(newSettings.log_level); try { await saveSettings(newSettings); @@ -456,6 +461,8 @@ export function SettingsPage() { require_media_mounted: settings.require_media_mounted, download_timeout: settings.download_timeout, download_wait: settings.download_wait, + download_daily_limit_per_domain: settings.download_daily_limit_per_domain ?? '', + download_daily_limit_global: settings.download_daily_limit_global ?? '', download_window_start: settings.download_window_start || '', download_window_end: settings.download_window_end || '', timezone: settings.timezone || '', @@ -726,6 +733,34 @@ export function SettingsPage() { /> + + + Daily Download Limit (per domain) + + } + value={state.download_daily_limit_per_domain} + disabled={disabled} + onChange={(e, i) => handleTimeoutChange(e, 'download_daily_limit_per_domain', i.value)} + /> + + + + Daily Download Limit (total) + + } + value={state.download_daily_limit_global} + disabled={disabled} + onChange={(e, i) => handleTimeoutChange(e, 'download_daily_limit_global', i.value)} + /> + + diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index ede0e364..ed0a2a26 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -23,7 +23,7 @@ from yt_dlp.extractor import YoutubeTabIE # noqa from wrolpi.cmd import YT_DLP_BIN from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \ - trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post + trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post, normalize_domain from wrolpi.dates import now from wrolpi.db import get_db_session from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, \ @@ -529,6 +529,10 @@ class ChannelDownloader(Downloader, ABC): def __repr__(self): return f'' + def normalize_domain(self, url: str) -> str: + # Collapse youtu.be / youtube.com/shorts / tracking params to the canonical youtube.com domain. + return normalize_domain(normalize_video_url(url)) + @staticmethod def is_a_playlist(info: dict): # A playlist may have an id different from its channel. @@ -793,6 +797,10 @@ class VideoDownloader(Downloader, ABC): def __repr__(self): return f'' + def normalize_domain(self, url: str) -> str: + # Collapse youtu.be / youtube.com/shorts / tracking params to the canonical youtube.com domain. + return normalize_domain(normalize_video_url(url)) + def already_downloaded(self, session: Session, *urls: str) -> List: # We only consider a video record with a video file as "downloaded". file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video')) diff --git a/wrolpi/common.py b/wrolpi/common.py index 48c4989b..2148e158 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -1068,6 +1068,8 @@ class MultiFileConfig: class WROLPiConfigValidator: archive_destination: str = None check_for_upgrades: bool = None + download_daily_limit_global: int = None + download_daily_limit_per_domain: int = None download_on_startup: bool = None download_timeout: int = None download_wait: int = None @@ -1251,6 +1253,8 @@ class WROLPiConfig(ConfigFile): file_name = 'wrolpi.yaml' default_config = dict( archive_destination='archive/%(domain_tag)s/%(domain)s', + download_daily_limit_global=None, + download_daily_limit_per_domain=None, download_on_startup=True, download_timeout=0, download_wait=20, @@ -1371,6 +1375,24 @@ class WROLPiConfig(ConfigFile): def download_wait(self, value: int): self.update({'download_wait': value}) + @property + def download_daily_limit_per_domain(self) -> Optional[int]: + """Maximum non-recurring downloads allowed per domain per day. ``None``/``0`` means unlimited.""" + return self._config.get('download_daily_limit_per_domain') + + @download_daily_limit_per_domain.setter + def download_daily_limit_per_domain(self, value: Optional[int]): + self.update({'download_daily_limit_per_domain': value}) + + @property + def download_daily_limit_global(self) -> Optional[int]: + """Maximum non-recurring downloads allowed across all domains per day. ``None``/``0`` means unlimited.""" + return self._config.get('download_daily_limit_global') + + @download_daily_limit_global.setter + def download_daily_limit_global(self, value: Optional[int]): + self.update({'download_daily_limit_global': value}) + @staticmethod def _normalize_time_value(value) -> str: """YAML may parse HH:MM values as sexagesimal integers (e.g. 21:00 -> 1260). Convert back to HH:MM.""" @@ -1825,6 +1847,32 @@ def extract_domain(url: str) -> str: return domain +# Common subdomains stripped when grouping downloads by domain (e.g. for daily download limits). +COMMON_DOWNLOAD_SUBDOMAINS = ('m.', 'mobile.') + + +def normalize_domain(url: str) -> str: + """Return the canonical domain used to group downloads (e.g. for daily download limits). + + Strips the port, a leading ``www.`` (via :func:`extract_domain`), and other common subdomains + (``m.``, ``mobile.``) so that, for example, ``m.example.com`` and ``www.example.com`` share one bucket. + + Downloaders may apply type-specific URL normalization before calling this (see + ``Downloader.normalize_domain``). + + >>> normalize_domain('https://m.example.com/foo') + 'example.com' + >>> normalize_domain('https://www.example.com:443/foo') + 'example.com' + """ + domain = extract_domain(url) + for subdomain in COMMON_DOWNLOAD_SUBDOMAINS: + if domain.startswith(subdomain): + domain = domain[len(subdomain):] + break + return domain + + def api_param_limiter(maximum: int, default: int = 20) -> callable: """Create a function which restricts the maximum number that can be returned. Useful for restricting API limit params. diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index d49b88f2..ef5b5356 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -33,7 +33,7 @@ from wrolpi.api_utils import api_app, perpetual_signal from wrolpi.cmd import which, run_command, CommandResult from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \ wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \ - get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL + get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL, normalize_domain from wrolpi.dates import TZDateTime, now, Seconds from wrolpi.db import get_db_session, get_db_curs from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, BotBlockedDownloadError, UnknownDownload, \ @@ -241,6 +241,7 @@ class Download(ModelHelper, Base): # noqa __tablename__ = 'download' # noqa __table_args__ = ( Index('idx_download_collection_id', 'collection_id'), + Index('idx_download_last_download_attempt', 'last_download_attempt'), ) id = Column(Integer, primary_key=True) url = Column(String, nullable=False, unique=True) @@ -252,6 +253,7 @@ class Download(ModelHelper, Base): # noqa error = Column(Text) # traceback from an error during downloading. frequency = Column(Integer) # seconds between re-downloading. info_json = Column(JSONB) # information retrieved WHILE downloading (from yt-dlp) + last_download_attempt = Column(TZDateTime) # last time this Download was dispatched (success or failure) last_successful_download = Column(TZDateTime) location = Column(Text) # Relative App URL where the downloaded item can be viewed. next_download = Column(TZDateTime) @@ -280,6 +282,7 @@ class Download(ModelHelper, Base): # noqa frequency=self.frequency, destination=self.destination, id=self.id, + last_download_attempt=self.last_download_attempt, last_successful_download=self.last_successful_download, location=self.location, next_download=self.next_download, @@ -440,6 +443,14 @@ class Downloader: def __repr__(self): return f'' + def normalize_domain(self, url: str) -> str: + """Return the canonical domain used to group this Download for daily download limits. + + The base implementation strips common subdomains (www, m, mobile) and the port. Subclasses + may override to apply download-type-specific URL normalization before extracting the domain + (e.g. VideoDownloader collapses youtu.be and /shorts/ links to youtube.com).""" + return normalize_domain(url) + def prepare_download(self, session: Session, download: Download) -> Any: """DB-bound prep. Caller (DownloadManager) opens and commits the session. @@ -720,14 +731,57 @@ class DownloadManager: self.disabled.clear() @staticmethod - def _get_local_time(): - """Return the current local time, using configured timezone or system default.""" - from wrolpi.common import get_wrolpi_config + def _get_local_now(): + """Return the current local datetime, using configured timezone or system default.""" config = get_wrolpi_config() if config.timezone: - tz = ZoneInfo(config.timezone) - return now().astimezone(tz).time() - return now().astimezone().time() + return now().astimezone(ZoneInfo(config.timezone)) + return now().astimezone() + + @staticmethod + def _get_local_time(): + """Return the current local time, using configured timezone or system default.""" + return DownloadManager._get_local_now().time() + + @classmethod + def _get_local_day_start(cls): + """Return the most recent local midnight as a timezone-aware datetime. + + Used as the lower bound when counting downloads attempted "today" for daily limits. Comparing + against the (UTC-aware) ``last_download_attempt`` column works because both are timezone-aware.""" + return cls._get_local_now().replace(hour=0, minute=0, second=0, microsecond=0) + + def _normalize_download_domain(self, download: 'Download') -> str: + """Return the normalized domain used to group a Download for daily limits. + + Delegates to the Download's downloader so type-specific URL normalization applies; falls back to + the generic normalizer when no downloader is registered for the Download.""" + downloader = None + if download.downloader: + try: + downloader = self.find_downloader_by_name(download.downloader) + except InvalidDownload: + downloader = None + if downloader: + return downloader.normalize_domain(download.url) + return normalize_domain(download.url) + + def daily_download_counts(self, session: Session) -> Tuple[int, Dict[str, int]]: + """Count non-recurring downloads attempted since local midnight. + + Recurring downloads (those with a ``frequency``, e.g. Channel/RSS catalog updates) never count + against the limits. Returns ``(global_count, {domain: count})`` where domains are normalized.""" + day_start = self._get_local_day_start() + attempted = session.query(Download).filter( + Download.frequency.is_(None), + Download.last_download_attempt.isnot(None), + Download.last_download_attempt >= day_start, + ).all() + domain_counts: Dict[str, int] = {} + for download in attempted: + domain = self._normalize_download_domain(download) + domain_counts[domain] = domain_counts.get(domain, 0) + 1 + return len(attempted), domain_counts def is_within_download_window(self) -> bool: """Returns True if current time is within the configured download window, or no window is configured.""" @@ -1019,11 +1073,33 @@ class DownloadManager: Download.id)) count = 0 download_wait = get_wrolpi_config().download_wait + # Daily download limits. A value of None/0 means unlimited. Recurring downloads (Channel/RSS + # catalog updates) are never gated and never counted; only their non-recurring children and + # one-off downloads count. + limit_per_domain = get_wrolpi_config().download_daily_limit_per_domain or 0 + limit_global = get_wrolpi_config().download_daily_limit_global or 0 + global_count, domain_counts = 0, {} + if limit_per_domain or limit_global: + global_count, domain_counts = self.daily_download_counts(session) rate_limited_domains = {} # Track domains we've already checked for rate limiting + limits_enabled = bool(limit_per_domain or limit_global) for download in new_downloads: domain = download.domain + is_recurring = download.frequency is not None + # Non-recurring downloads with limits enabled are grouped by their normalized domain. + limit_domain = self._normalize_download_domain(download) \ + if (limits_enabled and not is_recurring) else None if domain not in self.processing_domains and len( self.processing_domains) < SIMULTANEOUS_DOWNLOAD_DOMAINS: + # Daily download limits (recurring downloads bypass the gate entirely). + if limit_domain is not None: + if limit_global and global_count >= limit_global: + # Global daily limit reached; skip all remaining counted downloads this cycle. + continue + if limit_per_domain and domain_counts.get(limit_domain, 0) >= limit_per_domain: + # This domain hit its daily limit; other domains may still have room. + continue + # Per-domain rate limiting. if download_wait and download_wait > 0: try: @@ -1037,6 +1113,13 @@ class DownloadManager: except (AttributeError, RuntimeError): pass # shared_ctx not initialized + # Record the attempt time now (in this committed session) so the next dispatch cycle + # counts it even before the download's signal handler marks it started. + download.last_download_attempt = now() + if limit_domain is not None: + global_count += 1 + domain_counts[limit_domain] = domain_counts.get(limit_domain, 0) + 1 + self._add_processing_domain(domain) context = dict(download_id=download.id, download_url=download.url) await api_app.dispatch('wrolpi.download.download', context=context) diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 81c0d1c8..e598bf19 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -156,6 +156,8 @@ def get_settings(_: Request): 'archive_destination': wrolpi_config.archive_destination, 'download_manager_disabled': download_manager.is_disabled, 'download_manager_stopped': download_manager.is_stopped, + 'download_daily_limit_global': wrolpi_config.download_daily_limit_global, + 'download_daily_limit_per_domain': wrolpi_config.download_daily_limit_per_domain, 'download_on_startup': wrolpi_config.download_on_startup, 'download_timeout': wrolpi_config.download_timeout, 'download_wait': wrolpi_config.download_wait, @@ -266,6 +268,12 @@ async def update_settings(_: Request, body: schema.SettingsRequest): except (KeyError, Exception): raise InvalidConfig(f'Invalid timezone: {body.timezone}') + # Daily download limits must be non-negative integers. 0 (or null) means unlimited. + for field_name in ('download_daily_limit_global', 'download_daily_limit_per_domain'): + value = getattr(body, field_name, None) + if value is not None and value < 0: + raise InvalidConfig(f'{field_name} must be a non-negative integer') + # Handle download window fields: empty string clears the value. for field_name in ('download_window_start', 'download_window_end'): value = getattr(body, field_name, None) diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 3eaeb092..5bfef337 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -16,6 +16,8 @@ class SettingsObject: class SettingsResponse: download_manager_disabled: bool download_manager_stopped: bool + download_daily_limit_global: Optional[int] + download_daily_limit_per_domain: Optional[int] download_on_startup: bool download_timeout: int download_wait: int @@ -63,6 +65,8 @@ class SemanticUIColors(enum.StrEnum): @dataclass class SettingsRequest: archive_destination: Optional[str] = None + download_daily_limit_global: Optional[int] = None + download_daily_limit_per_domain: Optional[int] = None download_on_startup: Optional[bool] = None download_timeout: Optional[int] = None download_wait: Optional[int] = None diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index d41803a2..3172b222 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -2,7 +2,7 @@ import asyncio import json import pathlib from abc import ABC -from datetime import datetime, time, timezone +from datetime import datetime, time, timezone, timedelta from http import HTTPStatus from itertools import zip_longest from unittest import mock @@ -11,8 +11,9 @@ import pytest import pytz import yaml -from wrolpi.common import get_wrolpi_config -from wrolpi.dates import Seconds +from wrolpi.api_utils import api_app +from wrolpi.common import get_wrolpi_config, normalize_domain +from wrolpi.dates import Seconds, now from wrolpi.db import get_db_context from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \ get_download_manager_config, RSSDownloader, parse_aria2c_progress, _parse_size, \ @@ -1586,3 +1587,225 @@ def test_require_media_mounted_gate_opt_out(test_download_manager, test_wrolpi_c with mock.patch('wrolpi.downloader.PYTEST', False), \ mock.patch('wrolpi.downloader.wrol_mode_enabled', return_value=False): assert test_download_manager.can_download is True + + +# =================================================================================================== +# Daily download limits (per-domain and global) +# =================================================================================================== + +def test_normalize_domain_helper(): + """The generic domain normalizer strips port, www, and other common subdomains.""" + assert normalize_domain('https://www.example.com/foo') == 'example.com' + assert normalize_domain('https://m.example.com/foo') == 'example.com' + assert normalize_domain('https://mobile.example.com/foo') == 'example.com' + assert normalize_domain('https://example.com:443/foo') == 'example.com' + assert normalize_domain('https://example.com/foo') == 'example.com' + + +def test_base_downloader_normalize_domain(test_downloader): + """The base Downloader.normalize_domain strips common subdomains.""" + assert test_downloader.normalize_domain('https://m.rumble.com/abc') == 'rumble.com' + assert test_downloader.normalize_domain('https://www.rumble.com/abc') == 'rumble.com' + + +def test_video_downloader_normalize_domain(): + """VideoDownloader/ChannelDownloader collapse youtu.be, shorts, and tracking params to youtube.com.""" + from modules.videos.downloader import VideoDownloader, ChannelDownloader + # Build instances without __init__ to avoid re-registering with the global manager. + vd = object.__new__(VideoDownloader) + cd = object.__new__(ChannelDownloader) + for downloader in (vd, cd): + assert downloader.normalize_domain('https://youtu.be/abc12345') == 'youtube.com' + assert downloader.normalize_domain('https://www.youtube.com/shorts/abc12345') == 'youtube.com' + assert downloader.normalize_domain('https://m.youtube.com/watch?v=abc12345') == 'youtube.com' + assert downloader.normalize_domain( + 'https://www.youtube.com/watch?v=abc12345&list=PL123&si=foo') == 'youtube.com' + + +def _make_processed_download(session, downloader_name, url, *, when=None, frequency=None, status='complete'): + """Create a Download that looks like it was already attempted (for daily-limit counting).""" + download = Download(url=url, downloader=downloader_name, frequency=frequency, status=status) + download.last_download_attempt = when if when is not None else now() + session.add(download) + session.flush() + return download + + +def _dispatched_urls(dispatch_mock) -> set: + return {c.kwargs['context']['download_url'] for c in dispatch_mock.call_args_list} + + +@pytest.mark.asyncio +async def test_daily_download_counts(test_session, test_download_manager, test_downloader): + """daily_download_counts tallies non-recurring downloads attempted since local midnight, by domain.""" + name = test_downloader.name + # Counted today: two example.com (one via m. subdomain), one rumble.com. + _make_processed_download(test_session, name, 'https://example.com/1') + _make_processed_download(test_session, name, 'https://m.example.com/2') + _make_processed_download(test_session, name, 'https://rumble.com/3') + # Not counted: recurring (has frequency), and one attempted two days ago. + _make_processed_download(test_session, name, 'https://example.com/recurring', + frequency=DownloadFrequency.daily) + _make_processed_download(test_session, name, 'https://example.com/yesterday', + when=now() - timedelta(days=2)) + test_session.commit() + + global_count, domain_counts = test_download_manager.daily_download_counts(test_session) + assert global_count == 3 + assert domain_counts == {'example.com': 2, 'rumble.com': 1} + + +@pytest.mark.asyncio +async def test_daily_limit_per_domain_blocks(test_session, test_download_manager, test_downloader): + """A new download is not dispatched once its domain hits the per-domain daily limit.""" + config = get_wrolpi_config() + config.download_daily_limit_per_domain = 3 + name = test_downloader.name + for i in range(3): + _make_processed_download(test_session, name, f'https://example.com/done{i}') + # A different domain under its own limit must still be allowed. + new_blocked = Download(url='https://example.com/new', downloader=name, status='new') + new_allowed = Download(url='https://rumble.com/new', downloader=name, status='new') + test_session.add_all([new_blocked, new_allowed]) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + dispatched = _dispatched_urls(dispatch) + assert 'https://example.com/new' not in dispatched + assert 'https://rumble.com/new' in dispatched + test_session.refresh(new_blocked) + assert new_blocked.last_download_attempt is None + assert new_blocked.status == 'new' + + +@pytest.mark.asyncio +async def test_daily_limit_per_domain_allows_under_limit(test_session, test_download_manager, test_downloader): + """A new download is dispatched while its domain is under the per-domain daily limit.""" + config = get_wrolpi_config() + config.download_daily_limit_per_domain = 3 + name = test_downloader.name + for i in range(2): + _make_processed_download(test_session, name, f'https://example.com/done{i}') + new = Download(url='https://example.com/new', downloader=name, status='new') + test_session.add(new) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + assert 'https://example.com/new' in _dispatched_urls(dispatch) + test_session.refresh(new) + assert new.last_download_attempt is not None + + +@pytest.mark.asyncio +async def test_daily_limit_resets_next_day(test_session, test_download_manager, test_downloader): + """Downloads attempted on a previous day do not count against today's limit.""" + config = get_wrolpi_config() + config.download_daily_limit_per_domain = 3 + name = test_downloader.name + # Three were processed two days ago; they should not count today. + for i in range(3): + _make_processed_download(test_session, name, f'https://example.com/old{i}', + when=now() - timedelta(days=2)) + new = Download(url='https://example.com/new', downloader=name, status='new') + test_session.add(new) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + assert 'https://example.com/new' in _dispatched_urls(dispatch) + + +@pytest.mark.asyncio +async def test_daily_limit_counts_failed_attempts(test_session, test_download_manager, test_downloader): + """A failed/deferred download attempted today still consumes a daily slot.""" + config = get_wrolpi_config() + config.download_daily_limit_per_domain = 2 + name = test_downloader.name + # One complete, one deferred (failed) today => 2 attempts, domain is at its limit. + _make_processed_download(test_session, name, 'https://example.com/ok', status='complete') + _make_processed_download(test_session, name, 'https://example.com/failed', status='deferred') + new = Download(url='https://example.com/new', downloader=name, status='new') + test_session.add(new) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + assert 'https://example.com/new' not in _dispatched_urls(dispatch) + + +@pytest.mark.asyncio +async def test_daily_limit_global_blocks(test_session, test_download_manager, test_downloader): + """The global daily limit caps downloads across all domains.""" + config = get_wrolpi_config() + config.download_daily_limit_global = 3 + name = test_downloader.name + # 3 processed today across domains (global limit reached). + _make_processed_download(test_session, name, 'https://example.com/1') + _make_processed_download(test_session, name, 'https://example.com/2') + _make_processed_download(test_session, name, 'https://rumble.com/3') + # A brand-new domain with no prior downloads is still blocked by the global cap. + new = Download(url='https://wikipedia.org/new', downloader=name, status='new') + test_session.add(new) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + assert _dispatched_urls(dispatch) == set() + test_session.refresh(new) + assert new.last_download_attempt is None + + +@pytest.mark.asyncio +async def test_daily_limit_global_allows_under_limit(test_session, test_download_manager, test_downloader): + """Downloads dispatch while the global daily total is under the limit.""" + config = get_wrolpi_config() + config.download_daily_limit_global = 5 + name = test_downloader.name + for i in range(3): + _make_processed_download(test_session, name, f'https://example.com/{i}') + new = Download(url='https://wikipedia.org/new', downloader=name, status='new') + test_session.add(new) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + assert 'https://wikipedia.org/new' in _dispatched_urls(dispatch) + + +@pytest.mark.asyncio +async def test_recurring_downloads_bypass_daily_limits(test_session, test_download_manager, test_downloader): + """Recurring (Channel/RSS) downloads are never gated and never consume a daily slot.""" + config = get_wrolpi_config() + # Both limits would otherwise block everything past the first download. + config.download_daily_limit_global = 2 + config.download_daily_limit_per_domain = 2 + name = test_downloader.name + # One non-recurring already processed today (global count = 1). + _make_processed_download(test_session, name, 'https://example.com/done') + # A recurring download on a different domain, and a new non-recurring download. + recurring = Download(url='https://rumble.com/channel', downloader=name, status='new', + frequency=DownloadFrequency.daily) + non_recurring = Download(url='https://example.com/new', downloader=name, status='new') + test_session.add_all([recurring, non_recurring]) + test_session.commit() + + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + await test_download_manager.dispatch_downloads() + + dispatched = _dispatched_urls(dispatch) + # Recurring is dispatched despite the limits. The non-recurring is also dispatched, which proves the + # recurring download did not consume a global slot (else the global count would have reached 2 first). + assert 'https://rumble.com/channel' in dispatched + assert 'https://example.com/new' in dispatched + + # Recurring downloads are excluded from the daily count entirely. + global_count, domain_counts = test_download_manager.daily_download_counts(test_session) + assert 'rumble.com' not in domain_counts diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index fd938686..a608ff75 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -55,6 +55,35 @@ async def test_get_settings(test_session, async_client): assert response.status_code == HTTPStatus.OK +@pytest.mark.asyncio +async def test_daily_download_limit_settings(test_session, async_client): + """The daily download limits round-trip through the settings API and reject negative values.""" + # Defaults are unlimited (None). + request, response = await async_client.get('/api/settings') + assert response.json['download_daily_limit_global'] is None + assert response.json['download_daily_limit_per_domain'] is None + + data = {'download_daily_limit_global': 100, 'download_daily_limit_per_domain': 50} + request, response = await async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.NO_CONTENT + + request, response = await async_client.get('/api/settings') + assert response.json['download_daily_limit_global'] == 100 + assert response.json['download_daily_limit_per_domain'] == 50 + assert get_wrolpi_config().download_daily_limit_global == 100 + assert get_wrolpi_config().download_daily_limit_per_domain == 50 + + # 0 means unlimited and is accepted. + data = {'download_daily_limit_per_domain': 0} + request, response = await async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.NO_CONTENT + + # Negative values are rejected. + data = {'download_daily_limit_global': -1} + request, response = await async_client.patch('/api/settings', content=json.dumps(data)) + assert response.status_code == HTTPStatus.BAD_REQUEST + + @pytest.mark.asyncio async def test_log_level(async_client): """Log level must be between 0 and 40."""