Add per-domain and global daily download limits

Adds two optional DownloadManager settings:
- download_daily_limit_per_domain: max non-recurring downloads per domain per day
- download_daily_limit_global: max non-recurring downloads across all domains per day

Recurring Channel/RSS catalog updates are never gated and never counted; only the
videos/archives they spawn and one-off downloads count. Both successful and failed
attempts consume a slot (counted once per download per day). Limits reset at local
midnight (configured timezone). Over-limit downloads stay "new" and resume the next day.

Implementation:
- New Download.last_download_attempt column (+ index, migration) set at dispatch time so
  the count survives restarts and is correct across workers.
- dispatch_downloads gates non-recurring downloads: per-domain skips that domain, global
  stops dispatching counted downloads for the cycle. daily_download_counts derives the
  day's totals from the DB.
- Domain grouping uses normalized URLs via a new Downloader.normalize_domain method
  (base strips www/m/mobile + port); VideoDownloader/ChannelDownloader override it to
  collapse youtu.be, /shorts/, and tracking params to youtube.com. ArchiveDownloader is
  the seam for future archive URL normalization.
- Settings wired through WROLPiConfig, the /settings API (GET/PATCH, non-negative
  validation, 0 = unlimited), and the React Settings page.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Roland 2026-06-28 16:51:21 -06:00 • committed by lrnselfreliance
parent d6862ecd2d
commit d95249218b
9 changed files with 478 additions and 11 deletions

View file

@ -0,0 +1,29 @@
"""download_last_download_attempt
Add the `last_download_attempt` column to the `download` table. This records the last time a Download was
dispatched (whether the attempt succeeds or fails) and is used to enforce per-domain and global daily download
limits. An index supports the "attempted since local midnight" count query.
Revision ID: c3d5f7a9b1e2
Revises: b2e4f6a8c1d3
Create Date: 2026-06-28
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'c3d5f7a9b1e2'
down_revision = 'b2e4f6a8c1d3'
branch_labels = None
depends_on = None
def upgrade():
op.add_column('download', sa.Column('last_download_attempt', sa.DateTime(timezone=True), nullable=True))
op.create_index('idx_download_last_download_attempt', 'download', ['last_download_attempt'])
def downgrade():
op.drop_index('idx_download_last_download_attempt', table_name='download')
op.drop_column('download', 'last_download_attempt')

View file

@ -367,6 +367,11 @@ export function SettingsPage() {
setPendingSave(true);
newSettings.download_timeout = parseInt(newSettings.download_timeout);
newSettings.download_wait = parseInt(newSettings.download_wait);
// Empty daily-limit fields become null (no change / unlimited); a number is sent as-is.
newSettings.download_daily_limit_per_domain = newSettings.download_daily_limit_per_domain === ''
? null : parseInt(newSettings.download_daily_limit_per_domain);
newSettings.download_daily_limit_global = newSettings.download_daily_limit_global === ''
? null : parseInt(newSettings.download_daily_limit_global);
newSettings.log_level = toApiLogLevel(newSettings.log_level);
try {
await saveSettings(newSettings);
@ -456,6 +461,8 @@ export function SettingsPage() {
require_media_mounted: settings.require_media_mounted,
download_timeout: settings.download_timeout,
download_wait: settings.download_wait,
download_daily_limit_per_domain: settings.download_daily_limit_per_domain ?? '',
download_daily_limit_global: settings.download_daily_limit_global ?? '',
download_window_start: settings.download_window_start || '',
download_window_end: settings.download_window_end || '',
timezone: settings.timezone || '',
@ -726,6 +733,34 @@ export function SettingsPage() {
/>
</Form.Group>
<Form.Group inline>
<Form.Input
label={<>
<b>Daily Download Limit (per domain)</b>
<InfoPopup
content='Maximum number of downloads from any single domain per day. Recurring
Channel/RSS catalog updates do not count; only the videos/archives they create
and one-off downloads count. Leave empty or set to 0 for no limit.'/>
</>}
value={state.download_daily_limit_per_domain}
disabled={disabled}
onChange={(e, i) => handleTimeoutChange(e, 'download_daily_limit_per_domain', i.value)}
/>
</Form.Group>
<Form.Group inline>
<Form.Input
label={<>
<b>Daily Download Limit (total)</b>
<InfoPopup
content='Maximum number of downloads across all domains per day. Recurring
Channel/RSS catalog updates do not count. Leave empty or set to 0 for no limit.'/>
</>}
value={state.download_daily_limit_global}
disabled={disabled}
onChange={(e, i) => handleTimeoutChange(e, 'download_daily_limit_global', i.value)}
/>
</Form.Group>
<Form.Group inline>
<Form.Input
label={<>

View file

@ -23,7 +23,7 @@ from yt_dlp.extractor import YoutubeTabIE # noqa
from wrolpi.cmd import YT_DLP_BIN
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \
trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post
trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post, normalize_domain
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, \
@ -529,6 +529,10 @@ class ChannelDownloader(Downloader, ABC):
def __repr__(self):
return f'<ChannelDownloader>'
def normalize_domain(self, url: str) -> str:
# Collapse youtu.be / youtube.com/shorts / tracking params to the canonical youtube.com domain.
return normalize_domain(normalize_video_url(url))
@staticmethod
def is_a_playlist(info: dict):
# A playlist may have an id different from its channel.
@ -793,6 +797,10 @@ class VideoDownloader(Downloader, ABC):
def __repr__(self):
return f'<VideoDownloader>'
def normalize_domain(self, url: str) -> str:
# Collapse youtu.be / youtube.com/shorts / tracking params to the canonical youtube.com domain.
return normalize_domain(normalize_video_url(url))
def already_downloaded(self, session: Session, *urls: str) -> List:
# We only consider a video record with a video file as "downloaded".
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video'))

View file

@ -1068,6 +1068,8 @@ class MultiFileConfig:
class WROLPiConfigValidator:
archive_destination: str = None
check_for_upgrades: bool = None
download_daily_limit_global: int = None
download_daily_limit_per_domain: int = None
download_on_startup: bool = None
download_timeout: int = None
download_wait: int = None
@ -1251,6 +1253,8 @@ class WROLPiConfig(ConfigFile):
file_name = 'wrolpi.yaml'
default_config = dict(
archive_destination='archive/%(domain_tag)s/%(domain)s',
download_daily_limit_global=None,
download_daily_limit_per_domain=None,
download_on_startup=True,
download_timeout=0,
download_wait=20,
@ -1371,6 +1375,24 @@ class WROLPiConfig(ConfigFile):
def download_wait(self, value: int):
self.update({'download_wait': value})
@property
def download_daily_limit_per_domain(self) -> Optional[int]:
"""Maximum non-recurring downloads allowed per domain per day. ``None``/``0`` means unlimited."""
return self._config.get('download_daily_limit_per_domain')
@download_daily_limit_per_domain.setter
def download_daily_limit_per_domain(self, value: Optional[int]):
self.update({'download_daily_limit_per_domain': value})
@property
def download_daily_limit_global(self) -> Optional[int]:
"""Maximum non-recurring downloads allowed across all domains per day. ``None``/``0`` means unlimited."""
return self._config.get('download_daily_limit_global')
@download_daily_limit_global.setter
def download_daily_limit_global(self, value: Optional[int]):
self.update({'download_daily_limit_global': value})
@staticmethod
def _normalize_time_value(value) -> str:
"""YAML may parse HH:MM values as sexagesimal integers (e.g. 21:00 -> 1260). Convert back to HH:MM."""
@ -1825,6 +1847,32 @@ def extract_domain(url: str) -> str:
return domain
# Common subdomains stripped when grouping downloads by domain (e.g. for daily download limits).
COMMON_DOWNLOAD_SUBDOMAINS = ('m.', 'mobile.')
def normalize_domain(url: str) -> str:
"""Return the canonical domain used to group downloads (e.g. for daily download limits).
Strips the port, a leading ``www.`` (via :func:`extract_domain`), and other common subdomains
(``m.``, ``mobile.``) so that, for example, ``m.example.com`` and ``www.example.com`` share one bucket.
Downloaders may apply type-specific URL normalization before calling this (see
``Downloader.normalize_domain``).
>>> normalize_domain('https://m.example.com/foo')
'example.com'
>>> normalize_domain('https://www.example.com:443/foo')
'example.com'
"""
domain = extract_domain(url)
for subdomain in COMMON_DOWNLOAD_SUBDOMAINS:
if domain.startswith(subdomain):
domain = domain[len(subdomain):]
break
return domain
def api_param_limiter(maximum: int, default: int = 20) -> callable:
"""Create a function which restricts the maximum number that can be returned.
Useful for restricting API limit params.

View file

@ -33,7 +33,7 @@ from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import which, run_command, CommandResult
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL
get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL, normalize_domain
from wrolpi.dates import TZDateTime, now, Seconds
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, BotBlockedDownloadError, UnknownDownload, \
@ -241,6 +241,7 @@ class Download(ModelHelper, Base): # noqa
__tablename__ = 'download' # noqa
__table_args__ = (
Index('idx_download_collection_id', 'collection_id'),
Index('idx_download_last_download_attempt', 'last_download_attempt'),
)
id = Column(Integer, primary_key=True)
url = Column(String, nullable=False, unique=True)
@ -252,6 +253,7 @@ class Download(ModelHelper, Base): # noqa
error = Column(Text) # traceback from an error during downloading.
frequency = Column(Integer) # seconds between re-downloading.
info_json = Column(JSONB) # information retrieved WHILE downloading (from yt-dlp)
last_download_attempt = Column(TZDateTime) # last time this Download was dispatched (success or failure)
last_successful_download = Column(TZDateTime)
location = Column(Text) # Relative App URL where the downloaded item can be viewed.
next_download = Column(TZDateTime)
@ -280,6 +282,7 @@ class Download(ModelHelper, Base): # noqa
frequency=self.frequency,
destination=self.destination,
id=self.id,
last_download_attempt=self.last_download_attempt,
last_successful_download=self.last_successful_download,
location=self.location,
next_download=self.next_download,
@ -440,6 +443,14 @@ class Downloader:
def __repr__(self):
return f'<Downloader name={self.name}>'
def normalize_domain(self, url: str) -> str:
"""Return the canonical domain used to group this Download for daily download limits.
The base implementation strips common subdomains (www, m, mobile) and the port. Subclasses
may override to apply download-type-specific URL normalization before extracting the domain
(e.g. VideoDownloader collapses youtu.be and /shorts/ links to youtube.com)."""
return normalize_domain(url)
def prepare_download(self, session: Session, download: Download) -> Any:
"""DB-bound prep. Caller (DownloadManager) opens and commits the session.
@ -720,14 +731,57 @@ class DownloadManager:
self.disabled.clear()
@staticmethod
def _get_local_time():
"""Return the current local time, using configured timezone or system default."""
from wrolpi.common import get_wrolpi_config
def _get_local_now():
"""Return the current local datetime, using configured timezone or system default."""
config = get_wrolpi_config()
if config.timezone:
tz = ZoneInfo(config.timezone)
return now().astimezone(tz).time()
return now().astimezone().time()
return now().astimezone(ZoneInfo(config.timezone))
return now().astimezone()
@staticmethod
def _get_local_time():
"""Return the current local time, using configured timezone or system default."""
return DownloadManager._get_local_now().time()
@classmethod
def _get_local_day_start(cls):
"""Return the most recent local midnight as a timezone-aware datetime.
Used as the lower bound when counting downloads attempted "today" for daily limits. Comparing
against the (UTC-aware) ``last_download_attempt`` column works because both are timezone-aware."""
return cls._get_local_now().replace(hour=0, minute=0, second=0, microsecond=0)
def _normalize_download_domain(self, download: 'Download') -> str:
"""Return the normalized domain used to group a Download for daily limits.
Delegates to the Download's downloader so type-specific URL normalization applies; falls back to
the generic normalizer when no downloader is registered for the Download."""
downloader = None
if download.downloader:
try:
downloader = self.find_downloader_by_name(download.downloader)
except InvalidDownload:
downloader = None
if downloader:
return downloader.normalize_domain(download.url)
return normalize_domain(download.url)
def daily_download_counts(self, session: Session) -> Tuple[int, Dict[str, int]]:
"""Count non-recurring downloads attempted since local midnight.
Recurring downloads (those with a ``frequency``, e.g. Channel/RSS catalog updates) never count
against the limits. Returns ``(global_count, {domain: count})`` where domains are normalized."""
day_start = self._get_local_day_start()
attempted = session.query(Download).filter(
Download.frequency.is_(None),
Download.last_download_attempt.isnot(None),
Download.last_download_attempt >= day_start,
).all()
domain_counts: Dict[str, int] = {}
for download in attempted:
domain = self._normalize_download_domain(download)
domain_counts[domain] = domain_counts.get(domain, 0) + 1
return len(attempted), domain_counts
def is_within_download_window(self) -> bool:
"""Returns True if current time is within the configured download window, or no window is configured."""
@ -1019,11 +1073,33 @@ class DownloadManager:
Download.id))
count = 0
download_wait = get_wrolpi_config().download_wait
# Daily download limits. A value of None/0 means unlimited. Recurring downloads (Channel/RSS
# catalog updates) are never gated and never counted; only their non-recurring children and
# one-off downloads count.
limit_per_domain = get_wrolpi_config().download_daily_limit_per_domain or 0
limit_global = get_wrolpi_config().download_daily_limit_global or 0
global_count, domain_counts = 0, {}
if limit_per_domain or limit_global:
global_count, domain_counts = self.daily_download_counts(session)
rate_limited_domains = {} # Track domains we've already checked for rate limiting
limits_enabled = bool(limit_per_domain or limit_global)
for download in new_downloads:
domain = download.domain
is_recurring = download.frequency is not None
# Non-recurring downloads with limits enabled are grouped by their normalized domain.
limit_domain = self._normalize_download_domain(download) \
if (limits_enabled and not is_recurring) else None
if domain not in self.processing_domains and len(
self.processing_domains) < SIMULTANEOUS_DOWNLOAD_DOMAINS:
# Daily download limits (recurring downloads bypass the gate entirely).
if limit_domain is not None:
if limit_global and global_count >= limit_global:
# Global daily limit reached; skip all remaining counted downloads this cycle.
continue
if limit_per_domain and domain_counts.get(limit_domain, 0) >= limit_per_domain:
# This domain hit its daily limit; other domains may still have room.
continue
# Per-domain rate limiting.
if download_wait and download_wait > 0:
try:
@ -1037,6 +1113,13 @@ class DownloadManager:
except (AttributeError, RuntimeError):
pass # shared_ctx not initialized
# Record the attempt time now (in this committed session) so the next dispatch cycle
# counts it even before the download's signal handler marks it started.
download.last_download_attempt = now()
if limit_domain is not None:
global_count += 1
domain_counts[limit_domain] = domain_counts.get(limit_domain, 0) + 1
self._add_processing_domain(domain)
context = dict(download_id=download.id, download_url=download.url)
await api_app.dispatch('wrolpi.download.download', context=context)

View file

@ -156,6 +156,8 @@ def get_settings(_: Request):
'archive_destination': wrolpi_config.archive_destination,
'download_manager_disabled': download_manager.is_disabled,
'download_manager_stopped': download_manager.is_stopped,
'download_daily_limit_global': wrolpi_config.download_daily_limit_global,
'download_daily_limit_per_domain': wrolpi_config.download_daily_limit_per_domain,
'download_on_startup': wrolpi_config.download_on_startup,
'download_timeout': wrolpi_config.download_timeout,
'download_wait': wrolpi_config.download_wait,
@ -266,6 +268,12 @@ async def update_settings(_: Request, body: schema.SettingsRequest):
except (KeyError, Exception):
raise InvalidConfig(f'Invalid timezone: {body.timezone}')
# Daily download limits must be non-negative integers. 0 (or null) means unlimited.
for field_name in ('download_daily_limit_global', 'download_daily_limit_per_domain'):
value = getattr(body, field_name, None)
if value is not None and value < 0:
raise InvalidConfig(f'{field_name} must be a non-negative integer')
# Handle download window fields: empty string clears the value.
for field_name in ('download_window_start', 'download_window_end'):
value = getattr(body, field_name, None)

View file

@ -16,6 +16,8 @@ class SettingsObject:
class SettingsResponse:
download_manager_disabled: bool
download_manager_stopped: bool
download_daily_limit_global: Optional[int]
download_daily_limit_per_domain: Optional[int]
download_on_startup: bool
download_timeout: int
download_wait: int
@ -63,6 +65,8 @@ class SemanticUIColors(enum.StrEnum):
@dataclass
class SettingsRequest:
archive_destination: Optional[str] = None
download_daily_limit_global: Optional[int] = None
download_daily_limit_per_domain: Optional[int] = None
download_on_startup: Optional[bool] = None
download_timeout: Optional[int] = None
download_wait: Optional[int] = None

View file

@ -2,7 +2,7 @@ import asyncio
import json
import pathlib
from abc import ABC
from datetime import datetime, time, timezone
from datetime import datetime, time, timezone, timedelta
from http import HTTPStatus
from itertools import zip_longest
from unittest import mock
@ -11,8 +11,9 @@ import pytest
import pytz
import yaml
from wrolpi.common import get_wrolpi_config
from wrolpi.dates import Seconds
from wrolpi.api_utils import api_app
from wrolpi.common import get_wrolpi_config, normalize_domain
from wrolpi.dates import Seconds, now
from wrolpi.db import get_db_context
from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \
get_download_manager_config, RSSDownloader, parse_aria2c_progress, _parse_size, \
@ -1586,3 +1587,225 @@ def test_require_media_mounted_gate_opt_out(test_download_manager, test_wrolpi_c
with mock.patch('wrolpi.downloader.PYTEST', False), \
mock.patch('wrolpi.downloader.wrol_mode_enabled', return_value=False):
assert test_download_manager.can_download is True
# ===================================================================================================
# Daily download limits (per-domain and global)
# ===================================================================================================
def test_normalize_domain_helper():
"""The generic domain normalizer strips port, www, and other common subdomains."""
assert normalize_domain('https://www.example.com/foo') == 'example.com'
assert normalize_domain('https://m.example.com/foo') == 'example.com'
assert normalize_domain('https://mobile.example.com/foo') == 'example.com'
assert normalize_domain('https://example.com:443/foo') == 'example.com'
assert normalize_domain('https://example.com/foo') == 'example.com'
def test_base_downloader_normalize_domain(test_downloader):
"""The base Downloader.normalize_domain strips common subdomains."""
assert test_downloader.normalize_domain('https://m.rumble.com/abc') == 'rumble.com'
assert test_downloader.normalize_domain('https://www.rumble.com/abc') == 'rumble.com'
def test_video_downloader_normalize_domain():
"""VideoDownloader/ChannelDownloader collapse youtu.be, shorts, and tracking params to youtube.com."""
from modules.videos.downloader import VideoDownloader, ChannelDownloader
# Build instances without __init__ to avoid re-registering with the global manager.
vd = object.__new__(VideoDownloader)
cd = object.__new__(ChannelDownloader)
for downloader in (vd, cd):
assert downloader.normalize_domain('https://youtu.be/abc12345') == 'youtube.com'
assert downloader.normalize_domain('https://www.youtube.com/shorts/abc12345') == 'youtube.com'
assert downloader.normalize_domain('https://m.youtube.com/watch?v=abc12345') == 'youtube.com'
assert downloader.normalize_domain(
'https://www.youtube.com/watch?v=abc12345&list=PL123&si=foo') == 'youtube.com'
def _make_processed_download(session, downloader_name, url, *, when=None, frequency=None, status='complete'):
"""Create a Download that looks like it was already attempted (for daily-limit counting)."""
download = Download(url=url, downloader=downloader_name, frequency=frequency, status=status)
download.last_download_attempt = when if when is not None else now()
session.add(download)
session.flush()
return download
def _dispatched_urls(dispatch_mock) -> set:
return {c.kwargs['context']['download_url'] for c in dispatch_mock.call_args_list}
@pytest.mark.asyncio
async def test_daily_download_counts(test_session, test_download_manager, test_downloader):
"""daily_download_counts tallies non-recurring downloads attempted since local midnight, by domain."""
name = test_downloader.name
# Counted today: two example.com (one via m. subdomain), one rumble.com.
_make_processed_download(test_session, name, 'https://example.com/1')
_make_processed_download(test_session, name, 'https://m.example.com/2')
_make_processed_download(test_session, name, 'https://rumble.com/3')
# Not counted: recurring (has frequency), and one attempted two days ago.
_make_processed_download(test_session, name, 'https://example.com/recurring',
frequency=DownloadFrequency.daily)
_make_processed_download(test_session, name, 'https://example.com/yesterday',
when=now() - timedelta(days=2))
test_session.commit()
global_count, domain_counts = test_download_manager.daily_download_counts(test_session)
assert global_count == 3
assert domain_counts == {'example.com': 2, 'rumble.com': 1}
@pytest.mark.asyncio
async def test_daily_limit_per_domain_blocks(test_session, test_download_manager, test_downloader):
"""A new download is not dispatched once its domain hits the per-domain daily limit."""
config = get_wrolpi_config()
config.download_daily_limit_per_domain = 3
name = test_downloader.name
for i in range(3):
_make_processed_download(test_session, name, f'https://example.com/done{i}')
# A different domain under its own limit must still be allowed.
new_blocked = Download(url='https://example.com/new', downloader=name, status='new')
new_allowed = Download(url='https://rumble.com/new', downloader=name, status='new')
test_session.add_all([new_blocked, new_allowed])
test_session.commit()
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
await test_download_manager.dispatch_downloads()
dispatched = _dispatched_urls(dispatch)
assert 'https://example.com/new' not in dispatched
assert 'https://rumble.com/new' in dispatched
test_session.refresh(new_blocked)
assert new_blocked.last_download_attempt is None
assert new_blocked.status == 'new'
@pytest.mark.asyncio
async def test_daily_limit_per_domain_allows_under_limit(test_session, test_download_manager, test_downloader):
"""A new download is dispatched while its domain is under the per-domain daily limit."""
config = get_wrolpi_config()
config.download_daily_limit_per_domain = 3
name = test_downloader.name
for i in range(2):
_make_processed_download(test_session, name, f'https://example.com/done{i}')
new = Download(url='https://example.com/new', downloader=name, status='new')
test_session.add(new)
test_session.commit()
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
await test_download_manager.dispatch_downloads()
assert 'https://example.com/new' in _dispatched_urls(dispatch)
test_session.refresh(new)
assert new.last_download_attempt is not None
@pytest.mark.asyncio
async def test_daily_limit_resets_next_day(test_session, test_download_manager, test_downloader):
"""Downloads attempted on a previous day do not count against today's limit."""
config = get_wrolpi_config()
config.download_daily_limit_per_domain = 3
name = test_downloader.name
# Three were processed two days ago; they should not count today.
for i in range(3):
_make_processed_download(test_session, name, f'https://example.com/old{i}',
when=now() - timedelta(days=2))
new = Download(url='https://example.com/new', downloader=name, status='new')
test_session.add(new)
test_session.commit()
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
await test_download_manager.dispatch_downloads()
assert 'https://example.com/new' in _dispatched_urls(dispatch)
@pytest.mark.asyncio
async def test_daily_limit_counts_failed_attempts(test_session, test_download_manager, test_downloader):
"""A failed/deferred download attempted today still consumes a daily slot."""
config = get_wrolpi_config()
config.download_daily_limit_per_domain = 2
name = test_downloader.name
# One complete, one deferred (failed) today => 2 attempts, domain is at its limit.
_make_processed_download(test_session, name, 'https://example.com/ok', status='complete')
_make_processed_download(test_session, name, 'https://example.com/failed', status='deferred')
new = Download(url='https://example.com/new', downloader=name, status='new')
test_session.add(new)
test_session.commit()
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
await test_download_manager.dispatch_downloads()
assert 'https://example.com/new' not in _dispatched_urls(dispatch)
@pytest.mark.asyncio
async def test_daily_limit_global_blocks(test_session, test_download_manager, test_downloader):
"""The global daily limit caps downloads across all domains."""
config = get_wrolpi_config()
config.download_daily_limit_global = 3
name = test_downloader.name
# 3 processed today across domains (global limit reached).
_make_processed_download(test_session, name, 'https://example.com/1')
_make_processed_download(test_session, name, 'https://example.com/2')
_make_processed_download(test_session, name, 'https://rumble.com/3')
# A brand-new domain with no prior downloads is still blocked by the global cap.
new = Download(url='https://wikipedia.org/new', downloader=name, status='new')
test_session.add(new)
test_session.commit()
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
await test_download_manager.dispatch_downloads()
assert _dispatched_urls(dispatch) == set()
test_session.refresh(new)
assert new.last_download_attempt is None
@pytest.mark.asyncio
async def test_daily_limit_global_allows_under_limit(test_session, test_download_manager, test_downloader):
"""Downloads dispatch while the global daily total is under the limit."""
config = get_wrolpi_config()
config.download_daily_limit_global = 5
name = test_downloader.name
for i in range(3):
_make_processed_download(test_session, name, f'https://example.com/{i}')
new = Download(url='https://wikipedia.org/new', downloader=name, status='new')
test_session.add(new)
test_session.commit()
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
await test_download_manager.dispatch_downloads()
assert 'https://wikipedia.org/new' in _dispatched_urls(dispatch)
@pytest.mark.asyncio
async def test_recurring_downloads_bypass_daily_limits(test_session, test_download_manager, test_downloader):
"""Recurring (Channel/RSS) downloads are never gated and never consume a daily slot."""
config = get_wrolpi_config()
# Both limits would otherwise block everything past the first download.
config.download_daily_limit_global = 2
config.download_daily_limit_per_domain = 2
name = test_downloader.name
# One non-recurring already processed today (global count = 1).
_make_processed_download(test_session, name, 'https://example.com/done')
# A recurring download on a different domain, and a new non-recurring download.
recurring = Download(url='https://rumble.com/channel', downloader=name, status='new',
frequency=DownloadFrequency.daily)
non_recurring = Download(url='https://example.com/new', downloader=name, status='new')
test_session.add_all([recurring, non_recurring])
test_session.commit()
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
await test_download_manager.dispatch_downloads()
dispatched = _dispatched_urls(dispatch)
# Recurring is dispatched despite the limits. The non-recurring is also dispatched, which proves the
# recurring download did not consume a global slot (else the global count would have reached 2 first).
assert 'https://rumble.com/channel' in dispatched
assert 'https://example.com/new' in dispatched
# Recurring downloads are excluded from the daily count entirely.
global_count, domain_counts = test_download_manager.daily_download_counts(test_session)
assert 'rumble.com' not in domain_counts

View file

@ -55,6 +55,35 @@ async def test_get_settings(test_session, async_client):
assert response.status_code == HTTPStatus.OK
@pytest.mark.asyncio
async def test_daily_download_limit_settings(test_session, async_client):
"""The daily download limits round-trip through the settings API and reject negative values."""
# Defaults are unlimited (None).
request, response = await async_client.get('/api/settings')
assert response.json['download_daily_limit_global'] is None
assert response.json['download_daily_limit_per_domain'] is None
data = {'download_daily_limit_global': 100, 'download_daily_limit_per_domain': 50}
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = await async_client.get('/api/settings')
assert response.json['download_daily_limit_global'] == 100
assert response.json['download_daily_limit_per_domain'] == 50
assert get_wrolpi_config().download_daily_limit_global == 100
assert get_wrolpi_config().download_daily_limit_per_domain == 50
# 0 means unlimited and is accepted.
data = {'download_daily_limit_per_domain': 0}
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
# Negative values are rejected.
data = {'download_daily_limit_global': -1}
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_log_level(async_client):
"""Log level must be between 0 and 40."""