From 9eea616b994a8ad87c64fb9b8ef06a9e440a601c Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 8 Nov 2021 14:16:19 -0700 Subject: [PATCH] Only blocking archive if all parts fail. --- modules/archive/lib.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 14916675..de898878 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -69,13 +69,16 @@ def get_new_archive_files(url: str): return ret +ARCHIVE_TIMEOUT = 5 * 60 # Wait at most 5 minutes for response. + + def request_archive(url: str): """ Send a request to the archive service to archive the URL. """ data = {'url': url} try: - resp = requests.post(f'{ARCHIVE_SERVICE}/json', json=data) + resp = requests.post(f'{ARCHIVE_SERVICE}/json', json=data, timeout=ARCHIVE_TIMEOUT) except Exception as e: logger.error('Error when requesting archive', exc_info=e) raise @@ -83,11 +86,19 @@ def request_archive(url: str): readability = resp.json()['readability'] screenshot = resp.json()['screenshot'] - if not screenshot: + if not (screenshot or singlefile or readability): raise Exception('singlefile response was empty!') - if screenshot: + if not singlefile: + logger.info(f'Failed to get singlefile for {url=}') + if not readability: + logger.info(f'Failed to get readability for {url=}') + if not screenshot: + logger.info(f'Failed to get screenshot for {url=}') + else: + # Decode screenshot to bytes. screenshot = base64.b64decode(screenshot) + return singlefile, readability, screenshot