From c26e47a55d2a15858582ea77ed49f880530feedc Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 2 Apr 2023 12:39:54 -0600 Subject: [PATCH] Ignoring worker count because each process has it's own worker. Limiting simultaneous downloads to 4. --- wrolpi/downloader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index e3bc9918..e11f9ba8 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -591,7 +591,9 @@ class DownloadManager: # Don't queue downloads when disabled. return - if len(self.data['processing_domains']) >= len(self.workers): + if (domains := len(self.data['processing_domains'])) >= 4: + self.log_debug( + f'Unable to queue downloads because there are more domains than workers: {domains} >= 4') return # Find download whose domain isn't already being downloaded.