mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
Add per-domain and global daily download limits
Adds two optional DownloadManager settings: - download_daily_limit_per_domain: max non-recurring downloads per domain per day - download_daily_limit_global: max non-recurring downloads across all domains per day Recurring Channel/RSS catalog updates are never gated and never counted; only the videos/archives they spawn and one-off downloads count. Both successful and failed attempts consume a slot (counted once per download per day). Limits reset at local midnight (configured timezone). Over-limit downloads stay "new" and resume the next day. Implementation: - New Download.last_download_attempt column (+ index, migration) set at dispatch time so the count survives restarts and is correct across workers. - dispatch_downloads gates non-recurring downloads: per-domain skips that domain, global stops dispatching counted downloads for the cycle. daily_download_counts derives the day's totals from the DB. - Domain grouping uses normalized URLs via a new Downloader.normalize_domain method (base strips www/m/mobile + port); VideoDownloader/ChannelDownloader override it to collapse youtu.be, /shorts/, and tracking params to youtube.com. ArchiveDownloader is the seam for future archive URL normalization. - Settings wired through WROLPiConfig, the /settings API (GET/PATCH, non-negative validation, 0 = unlimited), and the React Settings page. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
d6862ecd2d
commit
d95249218b
9 changed files with 478 additions and 11 deletions
|
|
@ -0,0 +1,29 @@
|
|||
"""download_last_download_attempt
|
||||
|
||||
Add the `last_download_attempt` column to the `download` table. This records the last time a Download was
|
||||
dispatched (whether the attempt succeeds or fails) and is used to enforce per-domain and global daily download
|
||||
limits. An index supports the "attempted since local midnight" count query.
|
||||
|
||||
Revision ID: c3d5f7a9b1e2
|
||||
Revises: b2e4f6a8c1d3
|
||||
Create Date: 2026-06-28
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'c3d5f7a9b1e2'
|
||||
down_revision = 'b2e4f6a8c1d3'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column('download', sa.Column('last_download_attempt', sa.DateTime(timezone=True), nullable=True))
|
||||
op.create_index('idx_download_last_download_attempt', 'download', ['last_download_attempt'])
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index('idx_download_last_download_attempt', table_name='download')
|
||||
op.drop_column('download', 'last_download_attempt')
|
||||
|
|
@ -367,6 +367,11 @@ export function SettingsPage() {
|
|||
setPendingSave(true);
|
||||
newSettings.download_timeout = parseInt(newSettings.download_timeout);
|
||||
newSettings.download_wait = parseInt(newSettings.download_wait);
|
||||
// Empty daily-limit fields become null (no change / unlimited); a number is sent as-is.
|
||||
newSettings.download_daily_limit_per_domain = newSettings.download_daily_limit_per_domain === ''
|
||||
? null : parseInt(newSettings.download_daily_limit_per_domain);
|
||||
newSettings.download_daily_limit_global = newSettings.download_daily_limit_global === ''
|
||||
? null : parseInt(newSettings.download_daily_limit_global);
|
||||
newSettings.log_level = toApiLogLevel(newSettings.log_level);
|
||||
try {
|
||||
await saveSettings(newSettings);
|
||||
|
|
@ -456,6 +461,8 @@ export function SettingsPage() {
|
|||
require_media_mounted: settings.require_media_mounted,
|
||||
download_timeout: settings.download_timeout,
|
||||
download_wait: settings.download_wait,
|
||||
download_daily_limit_per_domain: settings.download_daily_limit_per_domain ?? '',
|
||||
download_daily_limit_global: settings.download_daily_limit_global ?? '',
|
||||
download_window_start: settings.download_window_start || '',
|
||||
download_window_end: settings.download_window_end || '',
|
||||
timezone: settings.timezone || '',
|
||||
|
|
@ -726,6 +733,34 @@ export function SettingsPage() {
|
|||
/>
|
||||
</Form.Group>
|
||||
|
||||
<Form.Group inline>
|
||||
<Form.Input
|
||||
label={<>
|
||||
<b>Daily Download Limit (per domain)</b>
|
||||
<InfoPopup
|
||||
content='Maximum number of downloads from any single domain per day. Recurring
|
||||
Channel/RSS catalog updates do not count; only the videos/archives they create
|
||||
and one-off downloads count. Leave empty or set to 0 for no limit.'/>
|
||||
</>}
|
||||
value={state.download_daily_limit_per_domain}
|
||||
disabled={disabled}
|
||||
onChange={(e, i) => handleTimeoutChange(e, 'download_daily_limit_per_domain', i.value)}
|
||||
/>
|
||||
</Form.Group>
|
||||
<Form.Group inline>
|
||||
<Form.Input
|
||||
label={<>
|
||||
<b>Daily Download Limit (total)</b>
|
||||
<InfoPopup
|
||||
content='Maximum number of downloads across all domains per day. Recurring
|
||||
Channel/RSS catalog updates do not count. Leave empty or set to 0 for no limit.'/>
|
||||
</>}
|
||||
value={state.download_daily_limit_global}
|
||||
disabled={disabled}
|
||||
onChange={(e, i) => handleTimeoutChange(e, 'download_daily_limit_global', i.value)}
|
||||
/>
|
||||
</Form.Group>
|
||||
|
||||
<Form.Group inline>
|
||||
<Form.Input
|
||||
label={<>
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from yt_dlp.extractor import YoutubeTabIE # noqa
|
|||
|
||||
from wrolpi.cmd import YT_DLP_BIN
|
||||
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \
|
||||
trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post
|
||||
trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post, normalize_domain
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, \
|
||||
|
|
@ -529,6 +529,10 @@ class ChannelDownloader(Downloader, ABC):
|
|||
def __repr__(self):
|
||||
return f'<ChannelDownloader>'
|
||||
|
||||
def normalize_domain(self, url: str) -> str:
|
||||
# Collapse youtu.be / youtube.com/shorts / tracking params to the canonical youtube.com domain.
|
||||
return normalize_domain(normalize_video_url(url))
|
||||
|
||||
@staticmethod
|
||||
def is_a_playlist(info: dict):
|
||||
# A playlist may have an id different from its channel.
|
||||
|
|
@ -793,6 +797,10 @@ class VideoDownloader(Downloader, ABC):
|
|||
def __repr__(self):
|
||||
return f'<VideoDownloader>'
|
||||
|
||||
def normalize_domain(self, url: str) -> str:
|
||||
# Collapse youtu.be / youtube.com/shorts / tracking params to the canonical youtube.com domain.
|
||||
return normalize_domain(normalize_video_url(url))
|
||||
|
||||
def already_downloaded(self, session: Session, *urls: str) -> List:
|
||||
# We only consider a video record with a video file as "downloaded".
|
||||
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video'))
|
||||
|
|
|
|||
|
|
@ -1068,6 +1068,8 @@ class MultiFileConfig:
|
|||
class WROLPiConfigValidator:
|
||||
archive_destination: str = None
|
||||
check_for_upgrades: bool = None
|
||||
download_daily_limit_global: int = None
|
||||
download_daily_limit_per_domain: int = None
|
||||
download_on_startup: bool = None
|
||||
download_timeout: int = None
|
||||
download_wait: int = None
|
||||
|
|
@ -1251,6 +1253,8 @@ class WROLPiConfig(ConfigFile):
|
|||
file_name = 'wrolpi.yaml'
|
||||
default_config = dict(
|
||||
archive_destination='archive/%(domain_tag)s/%(domain)s',
|
||||
download_daily_limit_global=None,
|
||||
download_daily_limit_per_domain=None,
|
||||
download_on_startup=True,
|
||||
download_timeout=0,
|
||||
download_wait=20,
|
||||
|
|
@ -1371,6 +1375,24 @@ class WROLPiConfig(ConfigFile):
|
|||
def download_wait(self, value: int):
|
||||
self.update({'download_wait': value})
|
||||
|
||||
@property
|
||||
def download_daily_limit_per_domain(self) -> Optional[int]:
|
||||
"""Maximum non-recurring downloads allowed per domain per day. ``None``/``0`` means unlimited."""
|
||||
return self._config.get('download_daily_limit_per_domain')
|
||||
|
||||
@download_daily_limit_per_domain.setter
|
||||
def download_daily_limit_per_domain(self, value: Optional[int]):
|
||||
self.update({'download_daily_limit_per_domain': value})
|
||||
|
||||
@property
|
||||
def download_daily_limit_global(self) -> Optional[int]:
|
||||
"""Maximum non-recurring downloads allowed across all domains per day. ``None``/``0`` means unlimited."""
|
||||
return self._config.get('download_daily_limit_global')
|
||||
|
||||
@download_daily_limit_global.setter
|
||||
def download_daily_limit_global(self, value: Optional[int]):
|
||||
self.update({'download_daily_limit_global': value})
|
||||
|
||||
@staticmethod
|
||||
def _normalize_time_value(value) -> str:
|
||||
"""YAML may parse HH:MM values as sexagesimal integers (e.g. 21:00 -> 1260). Convert back to HH:MM."""
|
||||
|
|
@ -1825,6 +1847,32 @@ def extract_domain(url: str) -> str:
|
|||
return domain
|
||||
|
||||
|
||||
# Common subdomains stripped when grouping downloads by domain (e.g. for daily download limits).
|
||||
COMMON_DOWNLOAD_SUBDOMAINS = ('m.', 'mobile.')
|
||||
|
||||
|
||||
def normalize_domain(url: str) -> str:
|
||||
"""Return the canonical domain used to group downloads (e.g. for daily download limits).
|
||||
|
||||
Strips the port, a leading ``www.`` (via :func:`extract_domain`), and other common subdomains
|
||||
(``m.``, ``mobile.``) so that, for example, ``m.example.com`` and ``www.example.com`` share one bucket.
|
||||
|
||||
Downloaders may apply type-specific URL normalization before calling this (see
|
||||
``Downloader.normalize_domain``).
|
||||
|
||||
>>> normalize_domain('https://m.example.com/foo')
|
||||
'example.com'
|
||||
>>> normalize_domain('https://www.example.com:443/foo')
|
||||
'example.com'
|
||||
"""
|
||||
domain = extract_domain(url)
|
||||
for subdomain in COMMON_DOWNLOAD_SUBDOMAINS:
|
||||
if domain.startswith(subdomain):
|
||||
domain = domain[len(subdomain):]
|
||||
break
|
||||
return domain
|
||||
|
||||
|
||||
def api_param_limiter(maximum: int, default: int = 20) -> callable:
|
||||
"""Create a function which restricts the maximum number that can be returned.
|
||||
Useful for restricting API limit params.
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ from wrolpi.api_utils import api_app, perpetual_signal
|
|||
from wrolpi.cmd import which, run_command, CommandResult
|
||||
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
|
||||
wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
|
||||
get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL
|
||||
get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL, normalize_domain
|
||||
from wrolpi.dates import TZDateTime, now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, BotBlockedDownloadError, UnknownDownload, \
|
||||
|
|
@ -241,6 +241,7 @@ class Download(ModelHelper, Base): # noqa
|
|||
__tablename__ = 'download' # noqa
|
||||
__table_args__ = (
|
||||
Index('idx_download_collection_id', 'collection_id'),
|
||||
Index('idx_download_last_download_attempt', 'last_download_attempt'),
|
||||
)
|
||||
id = Column(Integer, primary_key=True)
|
||||
url = Column(String, nullable=False, unique=True)
|
||||
|
|
@ -252,6 +253,7 @@ class Download(ModelHelper, Base): # noqa
|
|||
error = Column(Text) # traceback from an error during downloading.
|
||||
frequency = Column(Integer) # seconds between re-downloading.
|
||||
info_json = Column(JSONB) # information retrieved WHILE downloading (from yt-dlp)
|
||||
last_download_attempt = Column(TZDateTime) # last time this Download was dispatched (success or failure)
|
||||
last_successful_download = Column(TZDateTime)
|
||||
location = Column(Text) # Relative App URL where the downloaded item can be viewed.
|
||||
next_download = Column(TZDateTime)
|
||||
|
|
@ -280,6 +282,7 @@ class Download(ModelHelper, Base): # noqa
|
|||
frequency=self.frequency,
|
||||
destination=self.destination,
|
||||
id=self.id,
|
||||
last_download_attempt=self.last_download_attempt,
|
||||
last_successful_download=self.last_successful_download,
|
||||
location=self.location,
|
||||
next_download=self.next_download,
|
||||
|
|
@ -440,6 +443,14 @@ class Downloader:
|
|||
def __repr__(self):
|
||||
return f'<Downloader name={self.name}>'
|
||||
|
||||
def normalize_domain(self, url: str) -> str:
|
||||
"""Return the canonical domain used to group this Download for daily download limits.
|
||||
|
||||
The base implementation strips common subdomains (www, m, mobile) and the port. Subclasses
|
||||
may override to apply download-type-specific URL normalization before extracting the domain
|
||||
(e.g. VideoDownloader collapses youtu.be and /shorts/ links to youtube.com)."""
|
||||
return normalize_domain(url)
|
||||
|
||||
def prepare_download(self, session: Session, download: Download) -> Any:
|
||||
"""DB-bound prep. Caller (DownloadManager) opens and commits the session.
|
||||
|
||||
|
|
@ -720,14 +731,57 @@ class DownloadManager:
|
|||
self.disabled.clear()
|
||||
|
||||
@staticmethod
|
||||
def _get_local_time():
|
||||
"""Return the current local time, using configured timezone or system default."""
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
def _get_local_now():
|
||||
"""Return the current local datetime, using configured timezone or system default."""
|
||||
config = get_wrolpi_config()
|
||||
if config.timezone:
|
||||
tz = ZoneInfo(config.timezone)
|
||||
return now().astimezone(tz).time()
|
||||
return now().astimezone().time()
|
||||
return now().astimezone(ZoneInfo(config.timezone))
|
||||
return now().astimezone()
|
||||
|
||||
@staticmethod
|
||||
def _get_local_time():
|
||||
"""Return the current local time, using configured timezone or system default."""
|
||||
return DownloadManager._get_local_now().time()
|
||||
|
||||
@classmethod
|
||||
def _get_local_day_start(cls):
|
||||
"""Return the most recent local midnight as a timezone-aware datetime.
|
||||
|
||||
Used as the lower bound when counting downloads attempted "today" for daily limits. Comparing
|
||||
against the (UTC-aware) ``last_download_attempt`` column works because both are timezone-aware."""
|
||||
return cls._get_local_now().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
def _normalize_download_domain(self, download: 'Download') -> str:
|
||||
"""Return the normalized domain used to group a Download for daily limits.
|
||||
|
||||
Delegates to the Download's downloader so type-specific URL normalization applies; falls back to
|
||||
the generic normalizer when no downloader is registered for the Download."""
|
||||
downloader = None
|
||||
if download.downloader:
|
||||
try:
|
||||
downloader = self.find_downloader_by_name(download.downloader)
|
||||
except InvalidDownload:
|
||||
downloader = None
|
||||
if downloader:
|
||||
return downloader.normalize_domain(download.url)
|
||||
return normalize_domain(download.url)
|
||||
|
||||
def daily_download_counts(self, session: Session) -> Tuple[int, Dict[str, int]]:
|
||||
"""Count non-recurring downloads attempted since local midnight.
|
||||
|
||||
Recurring downloads (those with a ``frequency``, e.g. Channel/RSS catalog updates) never count
|
||||
against the limits. Returns ``(global_count, {domain: count})`` where domains are normalized."""
|
||||
day_start = self._get_local_day_start()
|
||||
attempted = session.query(Download).filter(
|
||||
Download.frequency.is_(None),
|
||||
Download.last_download_attempt.isnot(None),
|
||||
Download.last_download_attempt >= day_start,
|
||||
).all()
|
||||
domain_counts: Dict[str, int] = {}
|
||||
for download in attempted:
|
||||
domain = self._normalize_download_domain(download)
|
||||
domain_counts[domain] = domain_counts.get(domain, 0) + 1
|
||||
return len(attempted), domain_counts
|
||||
|
||||
def is_within_download_window(self) -> bool:
|
||||
"""Returns True if current time is within the configured download window, or no window is configured."""
|
||||
|
|
@ -1019,11 +1073,33 @@ class DownloadManager:
|
|||
Download.id))
|
||||
count = 0
|
||||
download_wait = get_wrolpi_config().download_wait
|
||||
# Daily download limits. A value of None/0 means unlimited. Recurring downloads (Channel/RSS
|
||||
# catalog updates) are never gated and never counted; only their non-recurring children and
|
||||
# one-off downloads count.
|
||||
limit_per_domain = get_wrolpi_config().download_daily_limit_per_domain or 0
|
||||
limit_global = get_wrolpi_config().download_daily_limit_global or 0
|
||||
global_count, domain_counts = 0, {}
|
||||
if limit_per_domain or limit_global:
|
||||
global_count, domain_counts = self.daily_download_counts(session)
|
||||
rate_limited_domains = {} # Track domains we've already checked for rate limiting
|
||||
limits_enabled = bool(limit_per_domain or limit_global)
|
||||
for download in new_downloads:
|
||||
domain = download.domain
|
||||
is_recurring = download.frequency is not None
|
||||
# Non-recurring downloads with limits enabled are grouped by their normalized domain.
|
||||
limit_domain = self._normalize_download_domain(download) \
|
||||
if (limits_enabled and not is_recurring) else None
|
||||
if domain not in self.processing_domains and len(
|
||||
self.processing_domains) < SIMULTANEOUS_DOWNLOAD_DOMAINS:
|
||||
# Daily download limits (recurring downloads bypass the gate entirely).
|
||||
if limit_domain is not None:
|
||||
if limit_global and global_count >= limit_global:
|
||||
# Global daily limit reached; skip all remaining counted downloads this cycle.
|
||||
continue
|
||||
if limit_per_domain and domain_counts.get(limit_domain, 0) >= limit_per_domain:
|
||||
# This domain hit its daily limit; other domains may still have room.
|
||||
continue
|
||||
|
||||
# Per-domain rate limiting.
|
||||
if download_wait and download_wait > 0:
|
||||
try:
|
||||
|
|
@ -1037,6 +1113,13 @@ class DownloadManager:
|
|||
except (AttributeError, RuntimeError):
|
||||
pass # shared_ctx not initialized
|
||||
|
||||
# Record the attempt time now (in this committed session) so the next dispatch cycle
|
||||
# counts it even before the download's signal handler marks it started.
|
||||
download.last_download_attempt = now()
|
||||
if limit_domain is not None:
|
||||
global_count += 1
|
||||
domain_counts[limit_domain] = domain_counts.get(limit_domain, 0) + 1
|
||||
|
||||
self._add_processing_domain(domain)
|
||||
context = dict(download_id=download.id, download_url=download.url)
|
||||
await api_app.dispatch('wrolpi.download.download', context=context)
|
||||
|
|
|
|||
|
|
@ -156,6 +156,8 @@ def get_settings(_: Request):
|
|||
'archive_destination': wrolpi_config.archive_destination,
|
||||
'download_manager_disabled': download_manager.is_disabled,
|
||||
'download_manager_stopped': download_manager.is_stopped,
|
||||
'download_daily_limit_global': wrolpi_config.download_daily_limit_global,
|
||||
'download_daily_limit_per_domain': wrolpi_config.download_daily_limit_per_domain,
|
||||
'download_on_startup': wrolpi_config.download_on_startup,
|
||||
'download_timeout': wrolpi_config.download_timeout,
|
||||
'download_wait': wrolpi_config.download_wait,
|
||||
|
|
@ -266,6 +268,12 @@ async def update_settings(_: Request, body: schema.SettingsRequest):
|
|||
except (KeyError, Exception):
|
||||
raise InvalidConfig(f'Invalid timezone: {body.timezone}')
|
||||
|
||||
# Daily download limits must be non-negative integers. 0 (or null) means unlimited.
|
||||
for field_name in ('download_daily_limit_global', 'download_daily_limit_per_domain'):
|
||||
value = getattr(body, field_name, None)
|
||||
if value is not None and value < 0:
|
||||
raise InvalidConfig(f'{field_name} must be a non-negative integer')
|
||||
|
||||
# Handle download window fields: empty string clears the value.
|
||||
for field_name in ('download_window_start', 'download_window_end'):
|
||||
value = getattr(body, field_name, None)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ class SettingsObject:
|
|||
class SettingsResponse:
|
||||
download_manager_disabled: bool
|
||||
download_manager_stopped: bool
|
||||
download_daily_limit_global: Optional[int]
|
||||
download_daily_limit_per_domain: Optional[int]
|
||||
download_on_startup: bool
|
||||
download_timeout: int
|
||||
download_wait: int
|
||||
|
|
@ -63,6 +65,8 @@ class SemanticUIColors(enum.StrEnum):
|
|||
@dataclass
|
||||
class SettingsRequest:
|
||||
archive_destination: Optional[str] = None
|
||||
download_daily_limit_global: Optional[int] = None
|
||||
download_daily_limit_per_domain: Optional[int] = None
|
||||
download_on_startup: Optional[bool] = None
|
||||
download_timeout: Optional[int] = None
|
||||
download_wait: Optional[int] = None
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import asyncio
|
|||
import json
|
||||
import pathlib
|
||||
from abc import ABC
|
||||
from datetime import datetime, time, timezone
|
||||
from datetime import datetime, time, timezone, timedelta
|
||||
from http import HTTPStatus
|
||||
from itertools import zip_longest
|
||||
from unittest import mock
|
||||
|
|
@ -11,8 +11,9 @@ import pytest
|
|||
import pytz
|
||||
import yaml
|
||||
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
from wrolpi.dates import Seconds
|
||||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.common import get_wrolpi_config, normalize_domain
|
||||
from wrolpi.dates import Seconds, now
|
||||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \
|
||||
get_download_manager_config, RSSDownloader, parse_aria2c_progress, _parse_size, \
|
||||
|
|
@ -1586,3 +1587,225 @@ def test_require_media_mounted_gate_opt_out(test_download_manager, test_wrolpi_c
|
|||
with mock.patch('wrolpi.downloader.PYTEST', False), \
|
||||
mock.patch('wrolpi.downloader.wrol_mode_enabled', return_value=False):
|
||||
assert test_download_manager.can_download is True
|
||||
|
||||
|
||||
# ===================================================================================================
|
||||
# Daily download limits (per-domain and global)
|
||||
# ===================================================================================================
|
||||
|
||||
def test_normalize_domain_helper():
|
||||
"""The generic domain normalizer strips port, www, and other common subdomains."""
|
||||
assert normalize_domain('https://www.example.com/foo') == 'example.com'
|
||||
assert normalize_domain('https://m.example.com/foo') == 'example.com'
|
||||
assert normalize_domain('https://mobile.example.com/foo') == 'example.com'
|
||||
assert normalize_domain('https://example.com:443/foo') == 'example.com'
|
||||
assert normalize_domain('https://example.com/foo') == 'example.com'
|
||||
|
||||
|
||||
def test_base_downloader_normalize_domain(test_downloader):
|
||||
"""The base Downloader.normalize_domain strips common subdomains."""
|
||||
assert test_downloader.normalize_domain('https://m.rumble.com/abc') == 'rumble.com'
|
||||
assert test_downloader.normalize_domain('https://www.rumble.com/abc') == 'rumble.com'
|
||||
|
||||
|
||||
def test_video_downloader_normalize_domain():
|
||||
"""VideoDownloader/ChannelDownloader collapse youtu.be, shorts, and tracking params to youtube.com."""
|
||||
from modules.videos.downloader import VideoDownloader, ChannelDownloader
|
||||
# Build instances without __init__ to avoid re-registering with the global manager.
|
||||
vd = object.__new__(VideoDownloader)
|
||||
cd = object.__new__(ChannelDownloader)
|
||||
for downloader in (vd, cd):
|
||||
assert downloader.normalize_domain('https://youtu.be/abc12345') == 'youtube.com'
|
||||
assert downloader.normalize_domain('https://www.youtube.com/shorts/abc12345') == 'youtube.com'
|
||||
assert downloader.normalize_domain('https://m.youtube.com/watch?v=abc12345') == 'youtube.com'
|
||||
assert downloader.normalize_domain(
|
||||
'https://www.youtube.com/watch?v=abc12345&list=PL123&si=foo') == 'youtube.com'
|
||||
|
||||
|
||||
def _make_processed_download(session, downloader_name, url, *, when=None, frequency=None, status='complete'):
|
||||
"""Create a Download that looks like it was already attempted (for daily-limit counting)."""
|
||||
download = Download(url=url, downloader=downloader_name, frequency=frequency, status=status)
|
||||
download.last_download_attempt = when if when is not None else now()
|
||||
session.add(download)
|
||||
session.flush()
|
||||
return download
|
||||
|
||||
|
||||
def _dispatched_urls(dispatch_mock) -> set:
|
||||
return {c.kwargs['context']['download_url'] for c in dispatch_mock.call_args_list}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daily_download_counts(test_session, test_download_manager, test_downloader):
|
||||
"""daily_download_counts tallies non-recurring downloads attempted since local midnight, by domain."""
|
||||
name = test_downloader.name
|
||||
# Counted today: two example.com (one via m. subdomain), one rumble.com.
|
||||
_make_processed_download(test_session, name, 'https://example.com/1')
|
||||
_make_processed_download(test_session, name, 'https://m.example.com/2')
|
||||
_make_processed_download(test_session, name, 'https://rumble.com/3')
|
||||
# Not counted: recurring (has frequency), and one attempted two days ago.
|
||||
_make_processed_download(test_session, name, 'https://example.com/recurring',
|
||||
frequency=DownloadFrequency.daily)
|
||||
_make_processed_download(test_session, name, 'https://example.com/yesterday',
|
||||
when=now() - timedelta(days=2))
|
||||
test_session.commit()
|
||||
|
||||
global_count, domain_counts = test_download_manager.daily_download_counts(test_session)
|
||||
assert global_count == 3
|
||||
assert domain_counts == {'example.com': 2, 'rumble.com': 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daily_limit_per_domain_blocks(test_session, test_download_manager, test_downloader):
|
||||
"""A new download is not dispatched once its domain hits the per-domain daily limit."""
|
||||
config = get_wrolpi_config()
|
||||
config.download_daily_limit_per_domain = 3
|
||||
name = test_downloader.name
|
||||
for i in range(3):
|
||||
_make_processed_download(test_session, name, f'https://example.com/done{i}')
|
||||
# A different domain under its own limit must still be allowed.
|
||||
new_blocked = Download(url='https://example.com/new', downloader=name, status='new')
|
||||
new_allowed = Download(url='https://rumble.com/new', downloader=name, status='new')
|
||||
test_session.add_all([new_blocked, new_allowed])
|
||||
test_session.commit()
|
||||
|
||||
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
|
||||
await test_download_manager.dispatch_downloads()
|
||||
|
||||
dispatched = _dispatched_urls(dispatch)
|
||||
assert 'https://example.com/new' not in dispatched
|
||||
assert 'https://rumble.com/new' in dispatched
|
||||
test_session.refresh(new_blocked)
|
||||
assert new_blocked.last_download_attempt is None
|
||||
assert new_blocked.status == 'new'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daily_limit_per_domain_allows_under_limit(test_session, test_download_manager, test_downloader):
|
||||
"""A new download is dispatched while its domain is under the per-domain daily limit."""
|
||||
config = get_wrolpi_config()
|
||||
config.download_daily_limit_per_domain = 3
|
||||
name = test_downloader.name
|
||||
for i in range(2):
|
||||
_make_processed_download(test_session, name, f'https://example.com/done{i}')
|
||||
new = Download(url='https://example.com/new', downloader=name, status='new')
|
||||
test_session.add(new)
|
||||
test_session.commit()
|
||||
|
||||
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
|
||||
await test_download_manager.dispatch_downloads()
|
||||
|
||||
assert 'https://example.com/new' in _dispatched_urls(dispatch)
|
||||
test_session.refresh(new)
|
||||
assert new.last_download_attempt is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daily_limit_resets_next_day(test_session, test_download_manager, test_downloader):
|
||||
"""Downloads attempted on a previous day do not count against today's limit."""
|
||||
config = get_wrolpi_config()
|
||||
config.download_daily_limit_per_domain = 3
|
||||
name = test_downloader.name
|
||||
# Three were processed two days ago; they should not count today.
|
||||
for i in range(3):
|
||||
_make_processed_download(test_session, name, f'https://example.com/old{i}',
|
||||
when=now() - timedelta(days=2))
|
||||
new = Download(url='https://example.com/new', downloader=name, status='new')
|
||||
test_session.add(new)
|
||||
test_session.commit()
|
||||
|
||||
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
|
||||
await test_download_manager.dispatch_downloads()
|
||||
|
||||
assert 'https://example.com/new' in _dispatched_urls(dispatch)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daily_limit_counts_failed_attempts(test_session, test_download_manager, test_downloader):
|
||||
"""A failed/deferred download attempted today still consumes a daily slot."""
|
||||
config = get_wrolpi_config()
|
||||
config.download_daily_limit_per_domain = 2
|
||||
name = test_downloader.name
|
||||
# One complete, one deferred (failed) today => 2 attempts, domain is at its limit.
|
||||
_make_processed_download(test_session, name, 'https://example.com/ok', status='complete')
|
||||
_make_processed_download(test_session, name, 'https://example.com/failed', status='deferred')
|
||||
new = Download(url='https://example.com/new', downloader=name, status='new')
|
||||
test_session.add(new)
|
||||
test_session.commit()
|
||||
|
||||
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
|
||||
await test_download_manager.dispatch_downloads()
|
||||
|
||||
assert 'https://example.com/new' not in _dispatched_urls(dispatch)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daily_limit_global_blocks(test_session, test_download_manager, test_downloader):
|
||||
"""The global daily limit caps downloads across all domains."""
|
||||
config = get_wrolpi_config()
|
||||
config.download_daily_limit_global = 3
|
||||
name = test_downloader.name
|
||||
# 3 processed today across domains (global limit reached).
|
||||
_make_processed_download(test_session, name, 'https://example.com/1')
|
||||
_make_processed_download(test_session, name, 'https://example.com/2')
|
||||
_make_processed_download(test_session, name, 'https://rumble.com/3')
|
||||
# A brand-new domain with no prior downloads is still blocked by the global cap.
|
||||
new = Download(url='https://wikipedia.org/new', downloader=name, status='new')
|
||||
test_session.add(new)
|
||||
test_session.commit()
|
||||
|
||||
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
|
||||
await test_download_manager.dispatch_downloads()
|
||||
|
||||
assert _dispatched_urls(dispatch) == set()
|
||||
test_session.refresh(new)
|
||||
assert new.last_download_attempt is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daily_limit_global_allows_under_limit(test_session, test_download_manager, test_downloader):
|
||||
"""Downloads dispatch while the global daily total is under the limit."""
|
||||
config = get_wrolpi_config()
|
||||
config.download_daily_limit_global = 5
|
||||
name = test_downloader.name
|
||||
for i in range(3):
|
||||
_make_processed_download(test_session, name, f'https://example.com/{i}')
|
||||
new = Download(url='https://wikipedia.org/new', downloader=name, status='new')
|
||||
test_session.add(new)
|
||||
test_session.commit()
|
||||
|
||||
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
|
||||
await test_download_manager.dispatch_downloads()
|
||||
|
||||
assert 'https://wikipedia.org/new' in _dispatched_urls(dispatch)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recurring_downloads_bypass_daily_limits(test_session, test_download_manager, test_downloader):
|
||||
"""Recurring (Channel/RSS) downloads are never gated and never consume a daily slot."""
|
||||
config = get_wrolpi_config()
|
||||
# Both limits would otherwise block everything past the first download.
|
||||
config.download_daily_limit_global = 2
|
||||
config.download_daily_limit_per_domain = 2
|
||||
name = test_downloader.name
|
||||
# One non-recurring already processed today (global count = 1).
|
||||
_make_processed_download(test_session, name, 'https://example.com/done')
|
||||
# A recurring download on a different domain, and a new non-recurring download.
|
||||
recurring = Download(url='https://rumble.com/channel', downloader=name, status='new',
|
||||
frequency=DownloadFrequency.daily)
|
||||
non_recurring = Download(url='https://example.com/new', downloader=name, status='new')
|
||||
test_session.add_all([recurring, non_recurring])
|
||||
test_session.commit()
|
||||
|
||||
with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch:
|
||||
await test_download_manager.dispatch_downloads()
|
||||
|
||||
dispatched = _dispatched_urls(dispatch)
|
||||
# Recurring is dispatched despite the limits. The non-recurring is also dispatched, which proves the
|
||||
# recurring download did not consume a global slot (else the global count would have reached 2 first).
|
||||
assert 'https://rumble.com/channel' in dispatched
|
||||
assert 'https://example.com/new' in dispatched
|
||||
|
||||
# Recurring downloads are excluded from the daily count entirely.
|
||||
global_count, domain_counts = test_download_manager.daily_download_counts(test_session)
|
||||
assert 'rumble.com' not in domain_counts
|
||||
|
|
|
|||
|
|
@ -55,6 +55,35 @@ async def test_get_settings(test_session, async_client):
|
|||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daily_download_limit_settings(test_session, async_client):
|
||||
"""The daily download limits round-trip through the settings API and reject negative values."""
|
||||
# Defaults are unlimited (None).
|
||||
request, response = await async_client.get('/api/settings')
|
||||
assert response.json['download_daily_limit_global'] is None
|
||||
assert response.json['download_daily_limit_per_domain'] is None
|
||||
|
||||
data = {'download_daily_limit_global': 100, 'download_daily_limit_per_domain': 50}
|
||||
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
request, response = await async_client.get('/api/settings')
|
||||
assert response.json['download_daily_limit_global'] == 100
|
||||
assert response.json['download_daily_limit_per_domain'] == 50
|
||||
assert get_wrolpi_config().download_daily_limit_global == 100
|
||||
assert get_wrolpi_config().download_daily_limit_per_domain == 50
|
||||
|
||||
# 0 means unlimited and is accepted.
|
||||
data = {'download_daily_limit_per_domain': 0}
|
||||
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# Negative values are rejected.
|
||||
data = {'download_daily_limit_global': -1}
|
||||
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_level(async_client):
|
||||
"""Log level must be between 0 and 40."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue