diff --git a/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py b/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py
new file mode 100644
index 00000000..1ad2c86a
--- /dev/null
+++ b/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py
@@ -0,0 +1,29 @@
+"""download_last_download_attempt
+
+Add the `last_download_attempt` column to the `download` table. This records the last time a Download was
+dispatched (whether the attempt succeeds or fails) and is used to enforce per-domain and global daily download
+limits. An index supports the "attempted since local midnight" count query.
+
+Revision ID: c3d5f7a9b1e2
+Revises: b2e4f6a8c1d3
+Create Date: 2026-06-28
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'c3d5f7a9b1e2'
+down_revision = 'b2e4f6a8c1d3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.add_column('download', sa.Column('last_download_attempt', sa.DateTime(timezone=True), nullable=True))
+ op.create_index('idx_download_last_download_attempt', 'download', ['last_download_attempt'])
+
+
+def downgrade():
+ op.drop_index('idx_download_last_download_attempt', table_name='download')
+ op.drop_column('download', 'last_download_attempt')
diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js
index c73b1548..85e232b8 100644
--- a/app/src/components/admin/Settings.js
+++ b/app/src/components/admin/Settings.js
@@ -367,6 +367,11 @@ export function SettingsPage() {
setPendingSave(true);
newSettings.download_timeout = parseInt(newSettings.download_timeout);
newSettings.download_wait = parseInt(newSettings.download_wait);
+ // Empty daily-limit fields become null (no change / unlimited); a number is sent as-is.
+ newSettings.download_daily_limit_per_domain = newSettings.download_daily_limit_per_domain === ''
+ ? null : parseInt(newSettings.download_daily_limit_per_domain);
+ newSettings.download_daily_limit_global = newSettings.download_daily_limit_global === ''
+ ? null : parseInt(newSettings.download_daily_limit_global);
newSettings.log_level = toApiLogLevel(newSettings.log_level);
try {
await saveSettings(newSettings);
@@ -456,6 +461,8 @@ export function SettingsPage() {
require_media_mounted: settings.require_media_mounted,
download_timeout: settings.download_timeout,
download_wait: settings.download_wait,
+ download_daily_limit_per_domain: settings.download_daily_limit_per_domain ?? '',
+ download_daily_limit_global: settings.download_daily_limit_global ?? '',
download_window_start: settings.download_window_start || '',
download_window_end: settings.download_window_end || '',
timezone: settings.timezone || '',
@@ -726,6 +733,34 @@ export function SettingsPage() {
/>
+
+
+ Daily Download Limit (per domain)
+
+ >}
+ value={state.download_daily_limit_per_domain}
+ disabled={disabled}
+ onChange={(e, i) => handleTimeoutChange(e, 'download_daily_limit_per_domain', i.value)}
+ />
+
+
+
+ Daily Download Limit (total)
+
+ >}
+ value={state.download_daily_limit_global}
+ disabled={disabled}
+ onChange={(e, i) => handleTimeoutChange(e, 'download_daily_limit_global', i.value)}
+ />
+
+
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index ede0e364..ed0a2a26 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -23,7 +23,7 @@ from yt_dlp.extractor import YoutubeTabIE # noqa
from wrolpi.cmd import YT_DLP_BIN
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \
- trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post
+ trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post, normalize_domain
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, \
@@ -529,6 +529,10 @@ class ChannelDownloader(Downloader, ABC):
def __repr__(self):
return f''
+ def normalize_domain(self, url: str) -> str:
+ # Collapse youtu.be / youtube.com/shorts / tracking params to the canonical youtube.com domain.
+ return normalize_domain(normalize_video_url(url))
+
@staticmethod
def is_a_playlist(info: dict):
# A playlist may have an id different from its channel.
@@ -793,6 +797,10 @@ class VideoDownloader(Downloader, ABC):
def __repr__(self):
return f''
+ def normalize_domain(self, url: str) -> str:
+ # Collapse youtu.be / youtube.com/shorts / tracking params to the canonical youtube.com domain.
+ return normalize_domain(normalize_video_url(url))
+
def already_downloaded(self, session: Session, *urls: str) -> List:
# We only consider a video record with a video file as "downloaded".
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video'))
diff --git a/wrolpi/common.py b/wrolpi/common.py
index 48c4989b..2148e158 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -1068,6 +1068,8 @@ class MultiFileConfig:
class WROLPiConfigValidator:
archive_destination: str = None
check_for_upgrades: bool = None
+ download_daily_limit_global: int = None
+ download_daily_limit_per_domain: int = None
download_on_startup: bool = None
download_timeout: int = None
download_wait: int = None
@@ -1251,6 +1253,8 @@ class WROLPiConfig(ConfigFile):
file_name = 'wrolpi.yaml'
default_config = dict(
archive_destination='archive/%(domain_tag)s/%(domain)s',
+ download_daily_limit_global=None,
+ download_daily_limit_per_domain=None,
download_on_startup=True,
download_timeout=0,
download_wait=20,
@@ -1371,6 +1375,24 @@ class WROLPiConfig(ConfigFile):
def download_wait(self, value: int):
self.update({'download_wait': value})
+ @property
+ def download_daily_limit_per_domain(self) -> Optional[int]:
+ """Maximum non-recurring downloads allowed per domain per day. ``None``/``0`` means unlimited."""
+ return self._config.get('download_daily_limit_per_domain')
+
+ @download_daily_limit_per_domain.setter
+ def download_daily_limit_per_domain(self, value: Optional[int]):
+ self.update({'download_daily_limit_per_domain': value})
+
+ @property
+ def download_daily_limit_global(self) -> Optional[int]:
+ """Maximum non-recurring downloads allowed across all domains per day. ``None``/``0`` means unlimited."""
+ return self._config.get('download_daily_limit_global')
+
+ @download_daily_limit_global.setter
+ def download_daily_limit_global(self, value: Optional[int]):
+ self.update({'download_daily_limit_global': value})
+
@staticmethod
def _normalize_time_value(value) -> str:
"""YAML may parse HH:MM values as sexagesimal integers (e.g. 21:00 -> 1260). Convert back to HH:MM."""
@@ -1825,6 +1847,32 @@ def extract_domain(url: str) -> str:
return domain
+# Common subdomains stripped when grouping downloads by domain (e.g. for daily download limits).
+COMMON_DOWNLOAD_SUBDOMAINS = ('m.', 'mobile.')
+
+
+def normalize_domain(url: str) -> str:
+ """Return the canonical domain used to group downloads (e.g. for daily download limits).
+
+ Strips the port, a leading ``www.`` (via :func:`extract_domain`), and other common subdomains
+ (``m.``, ``mobile.``) so that, for example, ``m.example.com`` and ``www.example.com`` share one bucket.
+
+ Downloaders may apply type-specific URL normalization before calling this (see
+ ``Downloader.normalize_domain``).
+
+ >>> normalize_domain('https://m.example.com/foo')
+ 'example.com'
+ >>> normalize_domain('https://www.example.com:443/foo')
+ 'example.com'
+ """
+ domain = extract_domain(url)
+ for subdomain in COMMON_DOWNLOAD_SUBDOMAINS:
+ if domain.startswith(subdomain):
+ domain = domain[len(subdomain):]
+ break
+ return domain
+
+
def api_param_limiter(maximum: int, default: int = 20) -> callable:
"""Create a function which restricts the maximum number that can be returned.
Useful for restricting API limit params.
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index d49b88f2..ef5b5356 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -33,7 +33,7 @@ from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import which, run_command, CommandResult
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
- get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL
+ get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL, normalize_domain
from wrolpi.dates import TZDateTime, now, Seconds
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, BotBlockedDownloadError, UnknownDownload, \
@@ -241,6 +241,7 @@ class Download(ModelHelper, Base): # noqa
__tablename__ = 'download' # noqa
__table_args__ = (
Index('idx_download_collection_id', 'collection_id'),
+ Index('idx_download_last_download_attempt', 'last_download_attempt'),
)
id = Column(Integer, primary_key=True)
url = Column(String, nullable=False, unique=True)
@@ -252,6 +253,7 @@ class Download(ModelHelper, Base): # noqa
error = Column(Text) # traceback from an error during downloading.
frequency = Column(Integer) # seconds between re-downloading.
info_json = Column(JSONB) # information retrieved WHILE downloading (from yt-dlp)
+ last_download_attempt = Column(TZDateTime) # last time this Download was dispatched (success or failure)
last_successful_download = Column(TZDateTime)
location = Column(Text) # Relative App URL where the downloaded item can be viewed.
next_download = Column(TZDateTime)
@@ -280,6 +282,7 @@ class Download(ModelHelper, Base): # noqa
frequency=self.frequency,
destination=self.destination,
id=self.id,
+ last_download_attempt=self.last_download_attempt,
last_successful_download=self.last_successful_download,
location=self.location,
next_download=self.next_download,
@@ -440,6 +443,14 @@ class Downloader:
def __repr__(self):
return f''
+ def normalize_domain(self, url: str) -> str:
+ """Return the canonical domain used to group this Download for daily download limits.
+
+ The base implementation strips common subdomains (www, m, mobile) and the port. Subclasses
+ may override to apply download-type-specific URL normalization before extracting the domain
+ (e.g. VideoDownloader collapses youtu.be and /shorts/ links to youtube.com)."""
+ return normalize_domain(url)
+
def prepare_download(self, session: Session, download: Download) -> Any:
"""DB-bound prep. Caller (DownloadManager) opens and commits the session.
@@ -720,14 +731,57 @@ class DownloadManager:
self.disabled.clear()
@staticmethod
- def _get_local_time():
- """Return the current local time, using configured timezone or system default."""
- from wrolpi.common import get_wrolpi_config
+ def _get_local_now():
+ """Return the current local datetime, using configured timezone or system default."""
config = get_wrolpi_config()
if config.timezone:
- tz = ZoneInfo(config.timezone)
- return now().astimezone(tz).time()
- return now().astimezone().time()
+ return now().astimezone(ZoneInfo(config.timezone))
+ return now().astimezone()
+
+ @staticmethod
+ def _get_local_time():
+ """Return the current local time, using configured timezone or system default."""
+ return DownloadManager._get_local_now().time()
+
+ @classmethod
+ def _get_local_day_start(cls):
+ """Return the most recent local midnight as a timezone-aware datetime.
+
+ Used as the lower bound when counting downloads attempted "today" for daily limits. Comparing
+ against the (UTC-aware) ``last_download_attempt`` column works because both are timezone-aware."""
+ return cls._get_local_now().replace(hour=0, minute=0, second=0, microsecond=0)
+
+ def _normalize_download_domain(self, download: 'Download') -> str:
+ """Return the normalized domain used to group a Download for daily limits.
+
+ Delegates to the Download's downloader so type-specific URL normalization applies; falls back to
+ the generic normalizer when no downloader is registered for the Download."""
+ downloader = None
+ if download.downloader:
+ try:
+ downloader = self.find_downloader_by_name(download.downloader)
+ except InvalidDownload:
+ downloader = None
+ if downloader:
+ return downloader.normalize_domain(download.url)
+ return normalize_domain(download.url)
+
+ def daily_download_counts(self, session: Session) -> Tuple[int, Dict[str, int]]:
+ """Count non-recurring downloads attempted since local midnight.
+
+ Recurring downloads (those with a ``frequency``, e.g. Channel/RSS catalog updates) never count
+ against the limits. Returns ``(global_count, {domain: count})`` where domains are normalized."""
+ day_start = self._get_local_day_start()
+ attempted = session.query(Download).filter(
+ Download.frequency.is_(None),
+ Download.last_download_attempt.isnot(None),
+ Download.last_download_attempt >= day_start,
+ ).all()
+ domain_counts: Dict[str, int] = {}
+ for download in attempted:
+ domain = self._normalize_download_domain(download)
+ domain_counts[domain] = domain_counts.get(domain, 0) + 1
+ return len(attempted), domain_counts
def is_within_download_window(self) -> bool:
"""Returns True if current time is within the configured download window, or no window is configured."""
@@ -1019,11 +1073,33 @@ class DownloadManager:
Download.id))
count = 0
download_wait = get_wrolpi_config().download_wait
+ # Daily download limits. A value of None/0 means unlimited. Recurring downloads (Channel/RSS
+ # catalog updates) are never gated and never counted; only their non-recurring children and
+ # one-off downloads count.
+ limit_per_domain = get_wrolpi_config().download_daily_limit_per_domain or 0
+ limit_global = get_wrolpi_config().download_daily_limit_global or 0
+ global_count, domain_counts = 0, {}
+ if limit_per_domain or limit_global:
+ global_count, domain_counts = self.daily_download_counts(session)
rate_limited_domains = {} # Track domains we've already checked for rate limiting
+ limits_enabled = bool(limit_per_domain or limit_global)
for download in new_downloads:
domain = download.domain
+ is_recurring = download.frequency is not None
+ # Non-recurring downloads with limits enabled are grouped by their normalized domain.
+ limit_domain = self._normalize_download_domain(download) \
+ if (limits_enabled and not is_recurring) else None
if domain not in self.processing_domains and len(
self.processing_domains) < SIMULTANEOUS_DOWNLOAD_DOMAINS:
+ # Daily download limits (recurring downloads bypass the gate entirely).
+ if limit_domain is not None:
+ if limit_global and global_count >= limit_global:
+ # Global daily limit reached; skip all remaining counted downloads this cycle.
+ continue
+ if limit_per_domain and domain_counts.get(limit_domain, 0) >= limit_per_domain:
+ # This domain hit its daily limit; other domains may still have room.
+ continue
+
# Per-domain rate limiting.
if download_wait and download_wait > 0:
try:
@@ -1037,6 +1113,13 @@ class DownloadManager:
except (AttributeError, RuntimeError):
pass # shared_ctx not initialized
+ # Record the attempt time now (in this committed session) so the next dispatch cycle
+ # counts it even before the download's signal handler marks it started.
+ download.last_download_attempt = now()
+ if limit_domain is not None:
+ global_count += 1
+ domain_counts[limit_domain] = domain_counts.get(limit_domain, 0) + 1
+
self._add_processing_domain(domain)
context = dict(download_id=download.id, download_url=download.url)
await api_app.dispatch('wrolpi.download.download', context=context)
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index 81c0d1c8..e598bf19 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -156,6 +156,8 @@ def get_settings(_: Request):
'archive_destination': wrolpi_config.archive_destination,
'download_manager_disabled': download_manager.is_disabled,
'download_manager_stopped': download_manager.is_stopped,
+ 'download_daily_limit_global': wrolpi_config.download_daily_limit_global,
+ 'download_daily_limit_per_domain': wrolpi_config.download_daily_limit_per_domain,
'download_on_startup': wrolpi_config.download_on_startup,
'download_timeout': wrolpi_config.download_timeout,
'download_wait': wrolpi_config.download_wait,
@@ -266,6 +268,12 @@ async def update_settings(_: Request, body: schema.SettingsRequest):
except (KeyError, Exception):
raise InvalidConfig(f'Invalid timezone: {body.timezone}')
+ # Daily download limits must be non-negative integers. 0 (or null) means unlimited.
+ for field_name in ('download_daily_limit_global', 'download_daily_limit_per_domain'):
+ value = getattr(body, field_name, None)
+ if value is not None and value < 0:
+ raise InvalidConfig(f'{field_name} must be a non-negative integer')
+
# Handle download window fields: empty string clears the value.
for field_name in ('download_window_start', 'download_window_end'):
value = getattr(body, field_name, None)
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index 3eaeb092..5bfef337 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -16,6 +16,8 @@ class SettingsObject:
class SettingsResponse:
download_manager_disabled: bool
download_manager_stopped: bool
+ download_daily_limit_global: Optional[int]
+ download_daily_limit_per_domain: Optional[int]
download_on_startup: bool
download_timeout: int
download_wait: int
@@ -63,6 +65,8 @@ class SemanticUIColors(enum.StrEnum):
@dataclass
class SettingsRequest:
archive_destination: Optional[str] = None
+ download_daily_limit_global: Optional[int] = None
+ download_daily_limit_per_domain: Optional[int] = None
download_on_startup: Optional[bool] = None
download_timeout: Optional[int] = None
download_wait: Optional[int] = None
diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py
index d41803a2..3172b222 100644
--- a/wrolpi/test/test_downloader.py
+++ b/wrolpi/test/test_downloader.py
@@ -2,7 +2,7 @@ import asyncio
import json
import pathlib
from abc import ABC
-from datetime import datetime, time, timezone
+from datetime import datetime, time, timezone, timedelta
from http import HTTPStatus
from itertools import zip_longest
from unittest import mock
@@ -11,8 +11,9 @@ import pytest
import pytz
import yaml
-from wrolpi.common import get_wrolpi_config
-from wrolpi.dates import Seconds
+from wrolpi.api_utils import api_app
+from wrolpi.common import get_wrolpi_config, normalize_domain
+from wrolpi.dates import Seconds, now
from wrolpi.db import get_db_context
from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \
get_download_manager_config, RSSDownloader, parse_aria2c_progress, _parse_size, \
@@ -1586,3 +1587,225 @@ def test_require_media_mounted_gate_opt_out(test_download_manager, test_wrolpi_c
with mock.patch('wrolpi.downloader.PYTEST', False), \
mock.patch('wrolpi.downloader.wrol_mode_enabled', return_value=False):
assert test_download_manager.can_download is True
+
+
+# ===================================================================================================
+# Daily download limits (per-domain and global)
+# ===================================================================================================
+
+def test_normalize_domain_helper():
+ """The generic domain normalizer strips port, www, and other common subdomains."""
+ assert normalize_domain('https://www.example.com/foo') == 'example.com'
+ assert normalize_domain('https://m.example.com/foo') == 'example.com'
+ assert normalize_domain('https://mobile.example.com/foo') == 'example.com'
+ assert normalize_domain('https://example.com:443/foo') == 'example.com'
+ assert normalize_domain('https://example.com/foo') == 'example.com'
+
+
+def test_base_downloader_normalize_domain(test_downloader):
+ """The base Downloader.normalize_domain strips common subdomains."""
+ assert test_downloader.normalize_domain('https://m.rumble.com/abc') == 'rumble.com'
+ assert test_downloader.normalize_domain('https://www.rumble.com/abc') == 'rumble.com'
+
+
+def test_video_downloader_normalize_domain():
+ """VideoDownloader/ChannelDownloader collapse youtu.be, shorts, and tracking params to youtube.com."""
+ from modules.videos.downloader import VideoDownloader, ChannelDownloader
+ # Build instances without __init__ to avoid re-registering with the global manager.
+ vd = object.__new__(VideoDownloader)
+ cd = object.__new__(ChannelDownloader)
+ for downloader in (vd, cd):
+ assert downloader.normalize_domain('https://youtu.be/abc12345') == 'youtube.com'
+ assert downloader.normalize_domain('https://www.youtube.com/shorts/abc12345') == 'youtube.com'
+ assert downloader.normalize_domain('https://m.youtube.com/watch?v=abc12345') == 'youtube.com'
+ assert downloader.normalize_domain(
+ 'https://www.youtube.com/watch?v=abc12345&list=PL123&si=foo') == 'youtube.com'
+
+
+def _make_processed_download(session, downloader_name, url, *, when=None, frequency=None, status='complete'):
+ """Create a Download that looks like it was already attempted (for daily-limit counting)."""
+ download = Download(url=url, downloader=downloader_name, frequency=frequency, status=status)
+ download.last_download_attempt = when if when is not None else now()
+ session.add(download)
+ session.flush()
+ return download
+
+
+def _dispatched_urls(dispatch_mock) -> set:
+ return {c.kwargs['context']['download_url'] for c in dispatch_mock.call_args_list}
+
+
+@pytest.mark.asyncio
+async def test_daily_download_counts(test_session, test_download_manager, test_downloader):
+ """daily_download_counts tallies non-recurring downloads attempted since local midnight, by domain."""
+ name = test_downloader.name
+ # Counted today: two example.com (one via m. subdomain), one rumble.com.
+ _make_processed_download(test_session, name, 'https://example.com/1')
+ _make_processed_download(test_session, name, 'https://m.example.com/2')
+ _make_processed_download(test_session, name, 'https://rumble.com/3')
+ # Not counted: recurring (has frequency), and one attempted two days ago.
+ _make_processed_download(test_session, name, 'https://example.com/recurring',
+ frequency=DownloadFrequency.daily)
+ _make_processed_download(test_session, name, 'https://example.com/yesterday',
+ when=now() - timedelta(days=2))
+ test_session.commit()
+
+ global_count, domain_counts = test_download_manager.daily_download_counts(test_session)
+ assert global_count == 3
+ assert domain_counts == {'example.com': 2, 'rumble.com': 1}
+
+
+@pytest.mark.asyncio
+async def test_daily_limit_per_domain_blocks(test_session, test_download_manager, test_downloader):
+ """A new download is not dispatched once its domain hits the per-domain daily limit."""
+ config = get_wrolpi_config()
+ config.download_daily_limit_per_domain = 3
+ name = test_downloader.name
+ for i in range(3):
+ _make_processed_download(test_session, name, f'https://example.com/done{i}')
+ # A different domain under its own limit must still be allowed.
+ new_blocked = Download(url='https://example.com/new', downloader=name, status='new')
+ new_allowed = Download(url='https://rumble.com/new', downloader=name, status='new')
+ test_session.add_all([new_blocked, new_allowed])
+ test_session.commit()
+
+ with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
+ await test_download_manager.dispatch_downloads()
+
+ dispatched = _dispatched_urls(dispatch)
+ assert 'https://example.com/new' not in dispatched
+ assert 'https://rumble.com/new' in dispatched
+ test_session.refresh(new_blocked)
+ assert new_blocked.last_download_attempt is None
+ assert new_blocked.status == 'new'
+
+
+@pytest.mark.asyncio
+async def test_daily_limit_per_domain_allows_under_limit(test_session, test_download_manager, test_downloader):
+ """A new download is dispatched while its domain is under the per-domain daily limit."""
+ config = get_wrolpi_config()
+ config.download_daily_limit_per_domain = 3
+ name = test_downloader.name
+ for i in range(2):
+ _make_processed_download(test_session, name, f'https://example.com/done{i}')
+ new = Download(url='https://example.com/new', downloader=name, status='new')
+ test_session.add(new)
+ test_session.commit()
+
+ with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
+ await test_download_manager.dispatch_downloads()
+
+ assert 'https://example.com/new' in _dispatched_urls(dispatch)
+ test_session.refresh(new)
+ assert new.last_download_attempt is not None
+
+
+@pytest.mark.asyncio
+async def test_daily_limit_resets_next_day(test_session, test_download_manager, test_downloader):
+ """Downloads attempted on a previous day do not count against today's limit."""
+ config = get_wrolpi_config()
+ config.download_daily_limit_per_domain = 3
+ name = test_downloader.name
+ # Three were processed two days ago; they should not count today.
+ for i in range(3):
+ _make_processed_download(test_session, name, f'https://example.com/old{i}',
+ when=now() - timedelta(days=2))
+ new = Download(url='https://example.com/new', downloader=name, status='new')
+ test_session.add(new)
+ test_session.commit()
+
+ with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
+ await test_download_manager.dispatch_downloads()
+
+ assert 'https://example.com/new' in _dispatched_urls(dispatch)
+
+
+@pytest.mark.asyncio
+async def test_daily_limit_counts_failed_attempts(test_session, test_download_manager, test_downloader):
+ """A failed/deferred download attempted today still consumes a daily slot."""
+ config = get_wrolpi_config()
+ config.download_daily_limit_per_domain = 2
+ name = test_downloader.name
+ # One complete, one deferred (failed) today => 2 attempts, domain is at its limit.
+ _make_processed_download(test_session, name, 'https://example.com/ok', status='complete')
+ _make_processed_download(test_session, name, 'https://example.com/failed', status='deferred')
+ new = Download(url='https://example.com/new', downloader=name, status='new')
+ test_session.add(new)
+ test_session.commit()
+
+ with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
+ await test_download_manager.dispatch_downloads()
+
+ assert 'https://example.com/new' not in _dispatched_urls(dispatch)
+
+
+@pytest.mark.asyncio
+async def test_daily_limit_global_blocks(test_session, test_download_manager, test_downloader):
+ """The global daily limit caps downloads across all domains."""
+ config = get_wrolpi_config()
+ config.download_daily_limit_global = 3
+ name = test_downloader.name
+ # 3 processed today across domains (global limit reached).
+ _make_processed_download(test_session, name, 'https://example.com/1')
+ _make_processed_download(test_session, name, 'https://example.com/2')
+ _make_processed_download(test_session, name, 'https://rumble.com/3')
+ # A brand-new domain with no prior downloads is still blocked by the global cap.
+ new = Download(url='https://wikipedia.org/new', downloader=name, status='new')
+ test_session.add(new)
+ test_session.commit()
+
+ with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
+ await test_download_manager.dispatch_downloads()
+
+ assert _dispatched_urls(dispatch) == set()
+ test_session.refresh(new)
+ assert new.last_download_attempt is None
+
+
+@pytest.mark.asyncio
+async def test_daily_limit_global_allows_under_limit(test_session, test_download_manager, test_downloader):
+ """Downloads dispatch while the global daily total is under the limit."""
+ config = get_wrolpi_config()
+ config.download_daily_limit_global = 5
+ name = test_downloader.name
+ for i in range(3):
+ _make_processed_download(test_session, name, f'https://example.com/{i}')
+ new = Download(url='https://wikipedia.org/new', downloader=name, status='new')
+ test_session.add(new)
+ test_session.commit()
+
+ with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
+ await test_download_manager.dispatch_downloads()
+
+ assert 'https://wikipedia.org/new' in _dispatched_urls(dispatch)
+
+
+@pytest.mark.asyncio
+async def test_recurring_downloads_bypass_daily_limits(test_session, test_download_manager, test_downloader):
+ """Recurring (Channel/RSS) downloads are never gated and never consume a daily slot."""
+ config = get_wrolpi_config()
+ # Both limits would otherwise block everything past the first download.
+ config.download_daily_limit_global = 2
+ config.download_daily_limit_per_domain = 2
+ name = test_downloader.name
+ # One non-recurring already processed today (global count = 1).
+ _make_processed_download(test_session, name, 'https://example.com/done')
+ # A recurring download on a different domain, and a new non-recurring download.
+ recurring = Download(url='https://rumble.com/channel', downloader=name, status='new',
+ frequency=DownloadFrequency.daily)
+ non_recurring = Download(url='https://example.com/new', downloader=name, status='new')
+ test_session.add_all([recurring, non_recurring])
+ test_session.commit()
+
+ with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
+ await test_download_manager.dispatch_downloads()
+
+ dispatched = _dispatched_urls(dispatch)
+ # Recurring is dispatched despite the limits. The non-recurring is also dispatched, which proves the
+ # recurring download did not consume a global slot (else the global count would have reached 2 first).
+ assert 'https://rumble.com/channel' in dispatched
+ assert 'https://example.com/new' in dispatched
+
+ # Recurring downloads are excluded from the daily count entirely.
+ global_count, domain_counts = test_download_manager.daily_download_counts(test_session)
+ assert 'rumble.com' not in domain_counts
diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py
index fd938686..a608ff75 100644
--- a/wrolpi/test/test_root_api.py
+++ b/wrolpi/test/test_root_api.py
@@ -55,6 +55,35 @@ async def test_get_settings(test_session, async_client):
assert response.status_code == HTTPStatus.OK
+@pytest.mark.asyncio
+async def test_daily_download_limit_settings(test_session, async_client):
+ """The daily download limits round-trip through the settings API and reject negative values."""
+ # Defaults are unlimited (None).
+ request, response = await async_client.get('/api/settings')
+ assert response.json['download_daily_limit_global'] is None
+ assert response.json['download_daily_limit_per_domain'] is None
+
+ data = {'download_daily_limit_global': 100, 'download_daily_limit_per_domain': 50}
+ request, response = await async_client.patch('/api/settings', content=json.dumps(data))
+ assert response.status_code == HTTPStatus.NO_CONTENT
+
+ request, response = await async_client.get('/api/settings')
+ assert response.json['download_daily_limit_global'] == 100
+ assert response.json['download_daily_limit_per_domain'] == 50
+ assert get_wrolpi_config().download_daily_limit_global == 100
+ assert get_wrolpi_config().download_daily_limit_per_domain == 50
+
+ # 0 means unlimited and is accepted.
+ data = {'download_daily_limit_per_domain': 0}
+ request, response = await async_client.patch('/api/settings', content=json.dumps(data))
+ assert response.status_code == HTTPStatus.NO_CONTENT
+
+ # Negative values are rejected.
+ data = {'download_daily_limit_global': -1}
+ request, response = await async_client.patch('/api/settings', content=json.dumps(data))
+ assert response.status_code == HTTPStatus.BAD_REQUEST
+
+
@pytest.mark.asyncio
async def test_log_level(async_client):
"""Log level must be between 0 and 40."""