diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 54bcfc83..6a77368e 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -611,6 +611,17 @@ export function DownloadWindowMessage() { return null; } +export function DailyLimitMessage() { + const {status} = React.useContext(StatusContext); + const downloads = status ? status.downloads : null; + + if (downloads && downloads.daily_limit_reached && !downloads.disabled && !downloads.stopped) { + return + } + return null; +} + export function CookiesUnlockModal({open, onClose, onSuccess}) { const [password, setPassword] = useState(''); const [submitting, setSubmitting] = useState(false); diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js index 8388a7c6..70ee7c5c 100644 --- a/app/src/components/admin/Downloads.js +++ b/app/src/components/admin/Downloads.js @@ -15,6 +15,7 @@ import { APIButton, CookiesLockedMessage, DisableDownloadsToggle, + DailyLimitMessage, DownloadWindowMessage, ErrorMessage, formatFrequency, @@ -684,6 +685,7 @@ export function DownloadsPage() { return <> + diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 57ff5617..d1b3f114 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1543,15 +1543,41 @@ class DownloadManager: func.count(Download.id).filter(Download.status == DownloadStatus.pending), func.count(Download.id).filter(Download.frequency != None), # noqa ).one() + daily_limit_reached = self.daily_limit_reached(session) summary = dict( pending=pending_downloads, recurring=recurring_downloads, disabled=self.is_disabled, stopped=self.is_stopped, outside_download_window=self.outside_download_window, + daily_limit_reached=daily_limit_reached, ) return summary + def daily_limit_reached(self, session: Session) -> bool: + """True when due (`new`, non-recurring) downloads exist and every one of them is blocked by the + daily download limits. This is what the UI uses to explain a stalled-looking queue; partial + blockage (some domains capped, others still downloading) does not count.""" + limit_per_domain = get_wrolpi_config().download_daily_limit_per_domain or 0 + limit_global = get_wrolpi_config().download_daily_limit_global or 0 + if not limit_per_domain and not limit_global: + return False + + due_downloads = session.query(Download).filter( + Download.status == DownloadStatus.new, + Download.frequency.is_(None), + ).all() + if not due_downloads: + return False + + global_count, domain_counts = self.daily_download_counts(session) + if limit_global and global_count >= limit_global: + return True + if limit_per_domain: + return all(domain_counts.get(self._normalize_download_domain(download), 0) >= limit_per_domain + for download in due_downloads) + return False + @staticmethod def calculate_next_download(session: Session, download: Download) -> Optional[datetime]: """ diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 0ed351d4..06c3624f 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -1859,6 +1859,51 @@ async def test_daily_limit_counts_failed_attempts(test_session, test_download_ma assert 'https://example.com/new' not in _dispatched_urls(dispatch) +@pytest.mark.asyncio +async def test_summary_daily_limit_reached(test_session, test_download_manager, test_downloader): + """get_summary reports daily_limit_reached only when every due download is blocked by a daily limit.""" + config = get_wrolpi_config() + name = test_downloader.name + + # No limits configured. + assert test_download_manager.get_summary()['daily_limit_reached'] is False + + config.download_daily_limit_per_domain = 2 + # No due downloads => not reached, even though limits are configured. + _make_processed_download(test_session, name, 'https://example.com/1') + _make_processed_download(test_session, name, 'https://example.com/2') + test_session.commit() + assert test_download_manager.get_summary()['daily_limit_reached'] is False + + # A due download for the capped domain => reached. + blocked = Download(url='https://example.com/new', downloader=name, status='new') + test_session.add(blocked) + test_session.commit() + assert test_download_manager.get_summary()['daily_limit_reached'] is True + + # A due download for another domain can still dispatch => not reached. + allowed = Download(url='https://rumble.com/new', downloader=name, status='new') + test_session.add(allowed) + test_session.commit() + assert test_download_manager.get_summary()['daily_limit_reached'] is False + + # Recurring downloads are never gated by daily limits and must not count as "due". + test_session.delete(allowed) + recurring = Download(url='https://example.com/feed', downloader=name, status='new', + frequency=DownloadFrequency.daily) + test_session.add(recurring) + test_session.commit() + assert test_download_manager.get_summary()['daily_limit_reached'] is True + + # Global limit blocks everything, even a fresh domain. + config.download_daily_limit_per_domain = None + config.download_daily_limit_global = 2 + fresh = Download(url='https://wikipedia.org/new', downloader=name, status='new') + test_session.add(fresh) + test_session.commit() + assert test_download_manager.get_summary()['daily_limit_reached'] is True + + @pytest.mark.asyncio async def test_daily_limit_global_blocks(test_session, test_download_manager, test_downloader): """The global daily limit caps downloads across all domains."""